From 576ccbf44fba411746be8aff8f34a268b7c4ad40 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 7 Sep 2026 19:23:59 +1000 Subject: [PATCH 1/7] make by compile write the whole project, not only its artefacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `by compile` wrote the generated C, the extensions and `by.h`, and nothing else. a compiled module reads its data files relative to itself, so a tree holding artefacts alone failed on the first `open`, naming a path in a directory the author never wrote anything to. carrying the rest of the project over is not enough on its own. the tree has to be one command's whole answer, because `finish` deletes what the last run wrote and this one did not — and that is only correct when this run authored the tree. so `compile` now writes everything `build` writes, the transpiled python and the sourcemap and the build record included, and the extensions besides. python's finder prefers an extension to source, so the modules that were named load natively and the rest are interpreted. three things follow, each of which was broken while the tree was artefacts alone: - `by compile one.by` produced a tree that could import `one` and nothing else written in basedpython, since the other modules arrived as `.by` - `by compile a.by` then `by compile b.by` left neither compiled: the second run pruned the first's artefact and put no python in its place - `by compile` into a directory `by build` had written deleted the sourcemap and the build record, and `by restage` then refused the tree — which took the language server's single-file re-stage down with it `Artifact` now reports what was written rather than what would have been. `--emit-c-only` used to name the extension it did not build, at a path no real build uses, so the caller found nothing there and pruned the extension a previous run had left. a build output is recognised by the `.by-manifest` it carries rather than by its name, so a tree written to any `--out` is turned away as a source. the directory is no longer created before the command knows it has anything to put in it, so a run that fails early leaves nothing behind. the default output directory is `build/` for both commands, which the standard python .gitignore already covers; `out/` never was. the editable install backend stages into the same constant and moves with it. --- crates/by_build/src/lib.rs | 23 +- crates/by_build/tests/end_to_end.rs | 35 +- crates/by_stage/src/project.rs | 126 ++++- crates/by_stage/src/staging.rs | 67 ++- crates/by_stage/src/verbatim.rs | 16 +- crates/ty/docs/cli.md | 7 +- crates/ty/src/args.rs | 13 +- crates/ty/src/by_commands.rs | 290 +++++++--- crates/ty/src/cli-reference.md | 10 +- crates/ty/src/lib.rs | 2 +- crates/ty/tests/by_e2e.rs | 508 ++++++++++++++++-- crates/ty/tests/by_restage.rs | 16 +- docs/basedpython/cli-reference.md | 73 ++- .../development/compilation/index.md | 15 +- .../development/compilation/technology.md | 2 +- docs/basedpython/development/sourcemaps.md | 4 +- docs/basedpython/features/api-lock.md | 2 +- docs/basedpython/frameworks/pytest.md | 2 +- docs/basedpython/getting-started.md | 20 +- docs/basedpython/packaging.md | 16 +- python/basedpython/build.py | 6 +- scripts/check_ecosystem_roundtrip.py | 12 +- 22 files changed, 1059 insertions(+), 206 deletions(-) diff --git a/crates/by_build/src/lib.rs b/crates/by_build/src/lib.rs index f3add74dd5..23ac643d2c 100644 --- a/crates/by_build/src/lib.rs +++ b/crates/by_build/src/lib.rs @@ -22,8 +22,18 @@ pub(crate) mod annotate; pub struct Artifact { /// the generated C, kept for `--annotate` and for debugging pub source: PathBuf, - /// the loadable extension - pub extension: PathBuf, + /// the header every generated `.c` includes, at the root of the tree + pub header: PathBuf, + /// the loadable extension, or `None` when only the C was emitted + /// + /// a caller that records what the build wrote — so that a later one can take + /// a stale file back — has to be told the difference. naming the extension an + /// `--emit-c-only` run *would* have built is not harmless: the name is + /// `.so`, which is not where any real build puts it (that is + /// `Toolchain::extension_path`, tagged with the interpreter's abi), so a + /// caller checking the file system for it finds nothing, records nothing, and + /// prunes the extension a previous real build left + pub extension: Option, /// the `--annotate` report, when one was asked for pub annotation: Option, } @@ -113,7 +123,8 @@ fn emit_verified(module: &ModuleIr, out_dir: &Path, options: &Options) -> Result } fs::create_dir_all(out_dir) .with_context(|| format!("could not create {}", out_dir.display()))?; - fs::write(out_dir.join(by_rt::BY_H_NAME), by_rt::BY_H)?; + let header = out_dir.join(by_rt::BY_H_NAME); + fs::write(&header, by_rt::BY_H)?; let source_path = out_dir.join(module.name.relative_path(".c")); create_parent(&source_path)?; @@ -123,7 +134,8 @@ fn emit_verified(module: &ModuleIr, out_dir: &Path, options: &Options) -> Result Ok(Built { artifact: Artifact { source: source_path, - extension: out_dir.join(module.name.relative_path(".so")), + header, + extension: None, annotation: write_annotation(module, out_dir, options)?, }, declined: module.declined.clone(), @@ -364,7 +376,8 @@ pub fn build_module(module: &ModuleIr, toolchain: &Toolchain, out_dir: &Path) -> Ok(Artifact { source, - extension, + header, + extension: Some(extension), annotation: None, }) } diff --git a/crates/by_build/tests/end_to_end.rs b/crates/by_build/tests/end_to_end.rs index 8c9e20da2f..d9678fbcff 100644 --- a/crates/by_build/tests/end_to_end.rs +++ b/crates/by_build/tests/end_to_end.rs @@ -81,7 +81,10 @@ fn built(module: &ModuleIr, toolchain: &Toolchain, tag: &str) -> Option let _ = std::fs::remove_dir_all(&dir); match build_module(module, toolchain, &dir) { Ok(artifact) => { - assert!(artifact.extension.exists(), "the extension was written"); + assert!( + extension_of(&artifact).exists(), + "the extension was written" + ); Some(dir) } Err(error) => { @@ -218,6 +221,18 @@ fn fib_module() -> ModuleIr { } } +/// The extension a build wrote. +/// +/// `Artifact::extension` is `None` only for an `--emit-c-only` emit, which +/// nothing in this file does — every case here goes through `build_module` or +/// `build_lowered`, both of which invoke the C compiler. +fn extension_of(artifact: &by_build::Artifact) -> &std::path::Path { + artifact + .extension + .as_deref() + .expect("this build compiled an extension, so it has to name one") +} + #[test] fn integer_arithmetic_computes_the_same_answers_as_python() { let Some((python, toolchain)) = environment() else { @@ -931,7 +946,7 @@ fn an_unchanged_module_is_not_recompiled() { .and_then(|meta| meta.modified()) .expect("the artifact exists") }; - let before = stamp(&first.artifact.extension); + let before = stamp(extension_of(&first.artifact)); let second = build_source( source, @@ -941,7 +956,11 @@ fn an_unchanged_module_is_not_recompiled() { &Options::default(), ) .expect("the toolchain already worked"); - assert_eq!(stamp(&second.artifact.extension), before, "it recompiled"); + assert_eq!( + stamp(extension_of(&second.artifact)), + before, + "it recompiled" + ); // and a real change does rebuild let changed = "def double(a: int) -> int:\n return a * 3\n"; @@ -954,7 +973,7 @@ fn an_unchanged_module_is_not_recompiled() { ) .expect("the toolchain already worked"); assert_ne!( - stamp(&third.artifact.extension), + stamp(extension_of(&third.artifact)), before, "it skipped a change" ); @@ -980,7 +999,7 @@ fn a_stale_extension_is_rebuilt_even_when_the_c_is_unchanged() { eprintln!("skipping: no working C toolchain"); return; }; - std::fs::remove_file(&built.artifact.extension).expect("the artifact exists"); + std::fs::remove_file(extension_of(&built.artifact)).expect("the artifact exists"); let again = build_source( source, "by_e2e_rebuild_stale", @@ -989,7 +1008,7 @@ fn a_stale_extension_is_rebuilt_even_when_the_c_is_unchanged() { &Options::default(), ) .expect("the toolchain already worked"); - assert!(again.artifact.extension.exists()); + assert!(extension_of(&again.artifact).exists()); } /// a class on a base from outside the module, holding storage of its own @@ -1251,10 +1270,10 @@ fn a_package_is_built_as_a_tree_and_imports_under_its_dotted_names() { return; }; assert!( - built.artifact.extension.exists(), + extension_of(&built.artifact).exists(), "{} was written to {}", name.dotted(), - built.artifact.extension.display() + extension_of(&built.artifact).display() ); } diff --git a/crates/by_stage/src/project.rs b/crates/by_stage/src/project.rs index 98877c7338..326bd1e454 100644 --- a/crates/by_stage/src/project.rs +++ b/crates/by_stage/src/project.rs @@ -14,11 +14,13 @@ //! build would have transpiled it, and the only way to guarantee that is for there //! to be one implementation of each. +use std::collections::HashMap; use std::ffi::OsStr; use std::path::{Path, PathBuf}; use anyhow::Context; -use ruff_db::system::{OsSystem, SystemPath, SystemPathBuf}; +use ruff_db::Db as _; +use ruff_db::system::{OsSystem, System, SystemPath, SystemPathBuf}; use ty_project::{Db, ProjectDatabase, ProjectMetadata}; /// Everything needed to build a project db a second time. @@ -308,7 +310,8 @@ pub(crate) fn project_sources( root: &Path, output: Option<&Path>, ) -> Vec<(PathBuf, ruff_db::files::File)> { - db.project() + let candidates: Vec<(PathBuf, ruff_db::files::File)> = db + .project() .files(db) .into_iter() .filter(|file| { @@ -328,9 +331,75 @@ pub(crate) fn project_sources( // this build is about to read, and reading those instead would build the // project into itself, one directory deeper each time .filter(|(path, _)| output.is_none_or(|output| !path.starts_with(output))) + .collect(); + // and nor is any *other* build's output. `--out` can name any directory, and + // a project can have several, so the one this run was given is not the only + // one that has to be turned away + let mut known = HashMap::new(); + candidates + .into_iter() + .filter(|(path, _)| !inside_build_output(db.system(), path, root, &mut known)) .collect() } +/// Whether `path` sits inside a tree that some build already wrote. +/// +/// A build output is self-describing: it holds a `.by-manifest` naming everything +/// that build put there. That marker is what identifies it, rather than the +/// directory's name. [`NON_SOURCE_DIRS`] already turns away the two default names, +/// but `--out` can say anything, and only the tree this run was *given* is known +/// to be an output from the arguments — a project that builds into two of them has +/// to have the other recognised on sight. +/// +/// It has to be turned away because the tree holds a *copy* of the project: every +/// hand-written `.py`, and every `.by` too when `build.sources` is on. Reading one +/// back means each of those modules is claimed twice — once where it was written +/// and once where it was copied — and the build stops, telling the author that two +/// files they never put side by side would compile to the same module. +/// +/// The project root is never itself an output, whatever it holds: a `.by-manifest` +/// left at the root would otherwise empty the project of every file it has. +fn inside_build_output( + system: &dyn System, + path: &Path, + root: &Path, + known: &mut HashMap, +) -> bool { + let Some(parent) = path.parent() else { + return false; + }; + let mut current = parent; + while current != root && current.starts_with(root) { + let is_output = match known.get(current) { + Some(answer) => *answer, + None => { + let answer = holds_manifest(system, current); + known.insert(current.to_path_buf(), answer); + answer + } + }; + if is_output { + return true; + } + match current.parent() { + Some(next) => current = next, + None => return false, + } + } + false +} + +/// Whether `directory` carries the manifest that marks it a build output. +/// +/// Asked of the db's [`System`] rather than of `std::fs`, so that it is the same +/// file system every other question in a build is asked — the language server's +/// re-stage runs this on a warm db, and an in-memory test system has to be able +/// to answer it. +pub(crate) fn holds_manifest(system: &dyn System, directory: &Path) -> bool { + SystemPath::from_std_path(directory) + .is_some_and(|path| system.is_file(&path.join(crate::staging::MANIFEST_FILENAME))) +} + /// The project's first-party module roots, longest first, as absolute paths. /// /// These are the directories a module name is resolved against — for a @@ -351,9 +420,60 @@ pub fn module_roots(db: &ProjectDatabase, cwd: &Path) -> Vec { #[cfg(test)] mod tests { - use super::is_hidden_within; + use super::{inside_build_output, is_hidden_within}; + use ruff_db::system::{OsSystem, SystemPath}; + use std::collections::HashMap; use std::path::Path; + /// `--out` can name any directory and a project can hold several, so a build + /// output is recognised by the manifest it carries rather than by its name. + #[test] + fn a_directory_holding_a_manifest_is_a_build_output() { + let directory = tempfile::tempdir().expect("tempdir"); + let root = directory.path(); + let output = root.join("anything"); + std::fs::create_dir_all(output.join("pkg")).expect("create"); + std::fs::write(output.join(".by-manifest"), "").expect("write"); + let plain = root.join("src"); + std::fs::create_dir_all(&plain).expect("create"); + + let system = + OsSystem::new(SystemPath::from_std_path(root).expect("the temp directory is utf-8")); + let mut known = HashMap::new(); + assert!(inside_build_output( + &system, + &output.join("a.by"), + root, + &mut known + )); + assert!( + inside_build_output(&system, &output.join("pkg").join("a.by"), root, &mut known), + "a file deeper inside the output is inside it too" + ); + assert!( + !inside_build_output(&system, &plain.join("a.by"), root, &mut known), + "a directory with no manifest is ordinary source" + ); + } + + /// A manifest at the root would otherwise empty the project of every file. + #[test] + fn the_project_root_is_never_itself_an_output() { + let directory = tempfile::tempdir().expect("tempdir"); + let root = directory.path(); + std::fs::write(root.join(".by-manifest"), "").expect("write"); + + let system = + OsSystem::new(SystemPath::from_std_path(root).expect("the temp directory is utf-8")); + let mut known = HashMap::new(); + assert!(!inside_build_output( + &system, + &root.join("a.by"), + root, + &mut known + )); + } + #[test] fn a_hidden_directory_is_not_project_source() { let root = Path::new("/p"); diff --git a/crates/by_stage/src/staging.rs b/crates/by_stage/src/staging.rs index 62c83a0c46..ea08eee095 100644 --- a/crates/by_stage/src/staging.rs +++ b/crates/by_stage/src/staging.rs @@ -20,13 +20,16 @@ use anyhow::Context; /// until one shadows a module that moved. The manifest is what makes the output a /// mirror rather than a pile: what the previous build wrote and this one did not /// is deleted. -const MANIFEST_FILENAME: &str = ".by-manifest"; +pub(crate) const MANIFEST_FILENAME: &str = ".by-manifest"; /// An output tree being written. pub struct Staging { out: PathBuf, /// relative destination -> the source it came from, for collision reporting written: BTreeMap>, + /// destinations another writer produced, held apart from `written` so a + /// source that would land on one is reported rather than overwriting it + recorded: BTreeSet, } impl Staging { @@ -34,6 +37,7 @@ impl Staging { Self { out: out.to_path_buf(), written: BTreeMap::new(), + recorded: BTreeSet::new(), } } @@ -78,6 +82,32 @@ impl Staging { .with_context(|| format!("could not write {}", destination.display())) } + /// Record a file another writer put into the tree at `relative`. + /// + /// `by compile` writes its extensions through `by_build`, which lays them out + /// itself rather than through a staging. They still belong in the manifest: + /// an extension left behind by a module that has since been deleted keeps + /// importing, and — because python's finder prefers an extension to source — + /// it goes on shadowing the `.py` a later `by build` writes in its place. + /// + /// A recorded path is also claimed, so a file the project keeps where an + /// artifact lands is reported rather than copied over it. The check runs in + /// both directions for whichever order the caller happens to use; today + /// `compile` records every artifact before it stages anything, so in practice + /// it is the staging side that reports. + pub fn record(&mut self, relative: &Path) -> anyhow::Result<()> { + if let Some((previous, source)) = self.written.get_key_value(relative) { + anyhow::bail!( + "the compiler wrote an artifact to `{}`, where {} is already staged \ + — one of them has to be renamed", + previous.display(), + claimant(source.as_deref()), + ); + } + self.recorded.insert(relative.to_path_buf()); + Ok(()) + } + /// Copy `source` to `relative` verbatim. pub(crate) fn copy(&mut self, relative: &Path, source: &Path) -> anyhow::Result<()> { self.claim(relative, Some(source))?; @@ -94,17 +124,27 @@ impl Staging { } fn claim(&mut self, relative: &Path, source: Option<&Path>) -> anyhow::Result<()> { + // a project that keeps a file of its own where an artifact lands — a + // hand-written `main.c` beside `main.by`, say. copying over it would + // leave a tree whose generated half is somebody else's file, and say + // nothing about it + if self.recorded.contains(relative) { + anyhow::bail!( + "{} would be carried over to `{}`, where the compiler \ + already wrote an artifact of that name — rename one of them, or \ + keep the file out of the build with `build.exclude`", + claimant(source), + relative.display(), + ); + } if let Some((previous, Some(previous_source))) = self.written.get_key_value(relative) && Some(previous_source.as_path()) != source { - let claimant = source.map_or_else( - || "the build".to_owned(), - |source| format!("`{}`", source.display()), - ); anyhow::bail!( - "`{}` and {claimant} both build to `{}` — \ + "`{}` and {} both build to `{}` — \ they are the same module, so one of them has to be renamed", previous_source.display(), + claimant(source), previous.display(), ); } @@ -118,7 +158,12 @@ impl Staging { pub fn finish(self) -> anyhow::Result<()> { let manifest = self.out.join(MANIFEST_FILENAME); let previous = read_manifest(&manifest); - let current: BTreeSet<&Path> = self.written.keys().map(PathBuf::as_path).collect(); + let current: BTreeSet<&Path> = self + .written + .keys() + .chain(self.recorded.iter()) + .map(PathBuf::as_path) + .collect(); let mut emptied: BTreeSet = BTreeSet::new(); for stale in &previous { @@ -156,6 +201,14 @@ impl Staging { } } +/// How a collision names the thing that wanted the destination. +fn claimant(source: Option<&Path>) -> String { + source.map_or_else( + || "the build".to_owned(), + |source| format!("`{}`", source.display()), + ) +} + fn create_parent(path: &Path) -> anyhow::Result<()> { if let Some(parent) = path.parent() { fs::create_dir_all(parent) diff --git a/crates/by_stage/src/verbatim.rs b/crates/by_stage/src/verbatim.rs index d87ea88308..cab6374dcd 100644 --- a/crates/by_stage/src/verbatim.rs +++ b/crates/by_stage/src/verbatim.rs @@ -16,11 +16,12 @@ use std::ffi::OsStr; use std::path::{Path, PathBuf}; +use ruff_db::Db as _; use ruff_db::system::SystemPath; use ty_project::{Db, ProjectDatabase}; use walkdir::WalkDir; -use crate::project::may_hold_build_content; +use crate::project::{holds_manifest, may_hold_build_content}; use crate::staging::{Staging, relative_destination}; /// One file carried into the output unchanged. @@ -56,6 +57,19 @@ fn verbatim_files( if entry.path() == out { return false; } + // nor is any *other* build's output. a project can have several — + // `by build --out one` beside `by build --out two` — and each holds a + // copy of every file this walk carries, so one that was not turned + // away would be copied into the next wholesale, a tree deep in a + // tree. it is recognised by the manifest it carries rather than by + // its name, because `--out` can say anything. the root is exempt: + // a manifest there would otherwise carry nothing over at all + if entry.file_type().is_dir() + && entry.path() != root + && holds_manifest(db.system(), entry.path()) + { + return false; + } if !may_hold_build_content(entry) { return false; } diff --git a/crates/ty/docs/cli.md b/crates/ty/docs/cli.md index b01c3d6d6e..d66aa93ffe 100644 --- a/crates/ty/docs/cli.md +++ b/crates/ty/docs/cli.md @@ -305,7 +305,7 @@ by build [OPTIONS]
--help, -h

Print help (see a summary with '-h')

--min-version version

minimum Python version the output must run on [default: the project's configured python version]

--no-unique-loop-bindings

leave a closure made inside a loop sharing the loop's one binding, as python does, instead of binding the values of the iteration it was made in

-
--out, -o dir

Where to write the output [default: out, or dist with --wheels]

+
--out, -o dir

Where to write the output [default: build, or dist with --wheels]

--print-manifest

Report what the build read and produced, as <kind> <value> lines.

input <path> for every file the project is made of — what a source distribution has to carry to rebuild into the same thing — and package <name> for every top-level package that came out.

--runtime-raises-checks

wrap every function with a raises clause in a runtime guard that fails when it raises something the clause does not include

@@ -367,8 +367,9 @@ by compile [OPTIONS] [FILE]...
--no-any

Fail the build when a function cannot be compiled because a type is gradual, instead of leaving it to its interpreted definition.

A contract about predictability rather than a speed switch: a gradual type is the commonest reason a function silently stays interpreted.

--no-unique-loop-bindings

leave a closure made inside a loop sharing the loop's one binding, as python does, instead of binding the values of the iteration it was made in

-
--output, -o dir

Where to write the generated C and the extension modules

-

[default: out]

--require-native

Fail the build if any function is left to its interpreted definition, whatever the reason.

+
--out, --output, -o dir

Where to write the generated C and the extension modules.

+

The same directory by build writes, and the same flag names it — --output is kept because it was the only spelling this command took.

+

[default: build]

--require-native

Fail the build if any function is left to its interpreted definition, whatever the reason.

Stricter than --no-any, and a different question: --no-any asks whether the module is fully typed, this asks whether it compiles entirely.

--runtime-raises-checks

wrap every function with a raises clause in a runtime guard that fails when it raises something the clause does not include

--soundness spec

which runtime type-soundness checks to insert: default, all (adds the opt-in parameters entry checks), none, or a comma-separated subset of generic-calls, projections, iterations, assignments, returns, arguments, parameters

diff --git a/crates/ty/src/args.rs b/crates/ty/src/args.rs index 308fe6e800..78b11e5547 100644 --- a/crates/ty/src/args.rs +++ b/crates/ty/src/args.rs @@ -146,7 +146,7 @@ pub(crate) enum Command { /// `uv`, which does the packaging. #[arg(long, conflicts_with_all = ["min_version", "print_manifest"])] wheels: bool, - /// Where to write the output [default: `out`, or `dist` with `--wheels`] + /// Where to write the output [default: `build`, or `dist` with `--wheels`] #[arg(short = 'o', long, value_name = "DIR")] out: Option, /// Report what the build read and produced, as ` ` lines. @@ -196,7 +196,16 @@ pub(crate) enum Command { #[arg(value_name = "FILE")] files: Vec, /// Where to write the generated C and the extension modules. - #[arg(short = 'o', long, value_name = "DIR", default_value = "out")] + /// + /// The same directory `by build` writes, and the same flag names it — + /// `--output` is kept because it was the only spelling this command took. + #[arg( + short = 'o', + long = "out", + visible_alias = "output", + value_name = "DIR", + default_value = "build" + )] output: PathBuf, /// Report every function that was not lowered natively, with the reason. #[arg(long)] diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index e0089b50d3..3de90be6a1 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -846,14 +846,7 @@ pub(crate) fn cmd_build( let target = config.min_version.to_string(); crate::by_stamps::fill_discovered(&mut config.stamps, &cwd, Some(&target)); - // the output directory is settled before the project is read, because it is - // the one directory the project must not be read *from*: it holds a copy of - // every source this build is about to write. canonical, because that is what - // the paths it is compared against are — creating it first is what makes - // canonicalizing it possible - let out = cwd.join(out); - fs::create_dir_all(&out).with_context(|| format!("could not create {}", out.display()))?; - let out = fs::canonicalize(&out).unwrap_or(out); + let out = settled_output_dir(&cwd, out); let (db, handles, rebuilder, root) = build_project_db(&cwd, BY_SOURCES, Some(&out))?; if handles.is_empty() { @@ -863,7 +856,7 @@ pub(crate) fn cmd_build( let file_count = handles.len(); let roots = module_roots(&db, &root); let mut staging = Staging::new(&out); - // `out/` outlives the build that wrote it — it is what a test runner, a + // `build/` outlives the build that wrote it — it is what a test runner, a // debugger or an editor plugin sees — so the sourcemap goes with it. this is // the directory where a `.by` really can be saved after the transpile, which // is what the digests beside the map are for @@ -889,12 +882,12 @@ pub(crate) fn cmd_build( }, ); // the sourcemap and the package markers describe the tree that was written, - // so they are staged whether or not something was reported — an `out/` a + // so they are staged whether or not something was reported — a `build/` a // debugger cannot read is worse than one built from a partial check stage_verbatim(&db, &root, &roots, &mut staging)?; stage_by_typed_markers(&db, &mut staging, &roots, &root)?; write_sourcemap_module(&mut staging, &entries)?; - // `out/` outlives the build that wrote it and is what a debugger, a test + // `build/` outlives the build that wrote it and is what a debugger, a test // runner or an editor plugin later reads, so it carries the same record a // `run` tree does. no entry module: a build is not pointed at one stage_build_record( @@ -946,6 +939,33 @@ fn print_build_manifest( Ok(()) } +/// Where a build's output goes, settled before the project is read. +/// +/// It is the one directory the project must not be read *from*: it holds a copy +/// of every source the build is about to write, so a second build that walked it +/// would take its own output for more of the project. Canonical, because the +/// paths it is compared against are. +/// +/// Nothing is created here. An output directory conjured before the command has +/// anything to put in it outlives every way the command can fail early, and a +/// project that has just been told it has no sources should not be left holding +/// an empty `build/` it did not ask for. A directory that does not exist yet +/// cannot be canonicalized, so its parent is, which answers the same question: +/// two paths naming this directory compare equal. +fn settled_output_dir(cwd: &Path, out: &Path) -> PathBuf { + let out = cwd.join(out); + if let Ok(canonical) = fs::canonicalize(&out) { + return canonical; + } + match (out.parent(), out.file_name()) { + (Some(parent), Some(name)) => match fs::canonicalize(parent) { + Ok(canonical) => canonical.join(name), + Err(_) => out, + }, + _ => out, + } +} + // ── compile ───────────────────────────────────────────────────────────────── /// How `by compile` was invoked. @@ -990,6 +1010,20 @@ pub(crate) fn cmd_compile( // to say which, and gets told so rather than told the wrong one crate::by_stamps::fill_discovered(&mut fallback.stamps, &cwd, None); options.fallback = Some(fallback); + // the python this writes into the tree is `by build`'s python, derived the + // way `build` derives it: from the version the *project* declares. + // + // it must not come from the interpreter the extensions are built against. + // the two commands share an output directory, so the module `build` writes + // and the module `compile` writes are the same file — and a project that + // declares `>=3.9` would get a tree lowered for 3.9 or for 3.13 depending on + // which command ran last. `build/` is also where an editable install points, + // so the losing half of that is code the project says it supports and no + // longer runs on + let mut tree_config = version_config(None, &cwd)?; + lowering.apply_for_build(&mut tree_config, &cwd)?; + let target = tree_config.min_version.to_string(); + crate::by_stamps::fill_discovered(&mut tree_config.stamps, &cwd, Some(&target)); let sources: Vec = if files.is_empty() { compilable_files(&cwd) } else { @@ -1009,17 +1043,20 @@ pub(crate) fn cmd_compile( // context that used to sit here misreported a version refusal as a missing header let toolchain = by_build::Toolchain::probe(&python)?; - let out_dir = cwd.join(output); + let out_dir = settled_output_dir(&cwd, output); let mut compiled = 0usize; let mut declined_total = 0usize; + // the output tree. `by_build` lays the artefacts out itself, so they are + // recorded rather than written through this — but they belong in the same + // manifest as everything else, or nothing ever takes a stale one back + let mut staging = Staging::new(&out_dir); // one database for the whole project, so a type imported from a sibling module // resolves. lowering each file on its own is sound — an unresolved class // degrades to the object protocol — but it makes every imported type look // gradual, and `--no-any` would then fail on noise - // `compile` embeds fallback source produced by the untyped transpile, which - // takes no db, so the rebuilder the other commands thread through is unused here - let (db, project, _rebuilder, _root) = build_project_db(&cwd, COMPILABLE_SOURCES, None)?; + let (db, project, rebuilder, root) = + build_project_db(&cwd, COMPILABLE_SOURCES, Some(&out_dir))?; // the database holds the whole project so a type imported from a sibling // resolves, but only the files that were *asked for* are checked and emitted. @@ -1076,6 +1113,47 @@ pub(crate) fn cmd_compile( render_diagnostics(&db, &diagnostics)?; } + // `compile` writes a superset of what `build` writes: the whole project as + // importable python, and native extensions for the modules that were asked + // for. two things follow from it, and neither is optional. + // + // the tree is a program rather than a heap of artefacts — a module nobody + // named still imports, from the python `by build` would have written, so + // `by compile one.by` no longer produces a tree that can import `one` and + // nothing else written in basedpython. + // + // and the manifest becomes sound. `finish` deletes what the last run wrote + // and this one did not, which is only a correct thing to do when this run + // authored the whole tree. while `compile` wrote artefacts alone, it deleted + // the sourcemap and build record a `by build` into the same directory had + // left — and `by restage` then refused that tree, which took the language + // server's single-file re-stage down with it + let roots = module_roots(&db, &root); + let transpilable: Vec<(PathBuf, ruff_db::files::File)> = project + .iter() + .filter(|(path, _)| { + path.extension() + .and_then(OsStr::to_str) + .is_some_and(|extension| BY_SOURCES.contains(&extension)) + }) + .cloned() + .collect(); + let mut entries: Vec = Vec::new(); + let mut requirements = by_transforms::RuntimeRequirements::default(); + let transpiled = render_check_and_transpile( + &db, + &transpilable, + &tree_config, + CheckGate::ParseErrorsOnly, + &rebuilder, + &mut requirements, + |emitted| { + let relative = transpiled_destination(&roots, &root, emitted.by_path); + entries.push(stage_module(&mut staging, &relative, emitted)?); + Ok(()) + }, + ); + // what each source will be compiled as, worked out before anything is written: // two sources that land on the same artefact used to leave only the second, and // nothing said so @@ -1109,63 +1187,139 @@ pub(crate) fn cmd_compile( planned.push((handle, name)); } - for ((path, file), name) in planned { - let source = fs::read_to_string(path) - .with_context(|| format!("could not read {}", path.display()))?; - - let program_file = ty_python_semantic::Db::program_file(&db, *file); - let parsed = ruff_db::parsed::parsed_module(&db, program_file.python_file(&db)).load(&db); - let model = ty_python_semantic::SemanticModel::new(&db, program_file); - // a `.py` source needs no transpiling to be its own interpreted fallback - let mut options = options.clone(); - if path.extension().is_some_and(|x| x == "py") { - options.language = by_irbuild::Language::Python; - } - let mut lowered = by_irbuild::build_module( - &db, - &model.program_environment(), - &model, - parsed.suite(), - name, - options.language, - ); - // the real path, so a `#line` in the generated C resolves for a debugger - let absolute = std::fs::canonicalize(path).unwrap_or_else(|_| path.clone()); - lowered.lines = Some(by_ir::function::LineTable::new( - absolute.display().to_string(), - &source, - )); + // the tree is finished whatever happens in here, so a run that gives up + // half way still leaves a manifest describing what is actually on disk. a + // manifest left describing the run before it would have the *next* run prune + // against a tree that no longer exists + let compiling: anyhow::Result<()> = 'compiling: { + for ((path, file), name) in planned { + let source = match fs::read_to_string(path) + .with_context(|| format!("could not read {}", path.display())) + { + Ok(source) => source, + Err(error) => break 'compiling Err(error), + }; - let built = if emit_c_only { - by_build::emit_lowered(lowered, &source, Some(&toolchain), &out_dir, &options) - } else { - by_build::build_lowered(lowered, &source, &toolchain, &out_dir, &options).inspect( - |built| { - eprintln!( - "{} -> {}", - path.display(), - built.artifact.extension.display() - ); - }, - ) + let program_file = ty_python_semantic::Db::program_file(&db, *file); + let parsed = + ruff_db::parsed::parsed_module(&db, program_file.python_file(&db)).load(&db); + let model = ty_python_semantic::SemanticModel::new(&db, program_file); + // a `.py` source needs no transpiling to be its own interpreted fallback + let mut options = options.clone(); + if path.extension().is_some_and(|x| x == "py") { + options.language = by_irbuild::Language::Python; + } + let mut lowered = by_irbuild::build_module( + &db, + &model.program_environment(), + &model, + parsed.suite(), + name, + options.language, + ); + // the real path, so a `#line` in the generated C resolves for a debugger + let absolute = std::fs::canonicalize(path).unwrap_or_else(|_| path.clone()); + lowered.lines = Some(by_ir::function::LineTable::new( + absolute.display().to_string(), + &source, + )); + + let built = match if emit_c_only { + by_build::emit_lowered(lowered, &source, Some(&toolchain), &out_dir, &options) + } else { + by_build::build_lowered(lowered, &source, &toolchain, &out_dir, &options).inspect( + |built| { + if let Some(extension) = &built.artifact.extension { + eprintln!("{} -> {}", path.display(), extension.display()); + } + }, + ) + } + .with_context(|| format!("could not compile {}", path.display())) + { + Ok(built) => built, + Err(error) => break 'compiling Err(error), + }; + + // exactly what `by_build` says it wrote — no guessing at names, and no + // asking the file system, which cannot tell an artefact this run + // produced from one a previous run left + for produced in [&built.artifact.source, &built.artifact.header] + .into_iter() + .chain(built.artifact.extension.as_ref()) + .chain(built.artifact.annotation.as_ref()) + { + if let Ok(relative) = produced.strip_prefix(&out_dir) + && let Err(error) = staging.record(relative) + { + break 'compiling Err(error); + } + } + if let Some(annotation) = &built.artifact.annotation { + eprintln!(" annotated {}", annotation.display()); + } + declined_total += built.declined.len(); + if verbose { + // a decline is the compiler's report on the code it did *not* take, + // so it points at that code the way every other diagnostic does + let diagnostics: Vec = built + .declined + .iter() + .map(|declined| declined_diagnostic(*file, declined)) + .collect(); + if let Err(error) = render_diagnostics(&db, &diagnostics) { + break 'compiling Err(error); + } + } + compiled += 1; } - .with_context(|| format!("could not compile {}", path.display()))?; - if let Some(annotation) = &built.artifact.annotation { - eprintln!(" annotated {}", annotation.display()); + // the rest of the project: the data a module reads relative to itself, + // the hand-written `.py` beside it, the markers and the record that let a + // tool read the tree back. the same calls `build` makes, in the same + // order, because this is the same tree. + // + // inside the block with the compile, because each of these can fail and + // the tree has to be finished either way — `stage_verbatim` in particular + // is where a file the project keeps on top of an artefact is reported, + // which is a failure this change deliberately introduced + if let Err(error) = stage_verbatim(&db, &root, &roots, &mut staging) { + break 'compiling Err(error); } - declined_total += built.declined.len(); - if verbose { - // a decline is the compiler's report on the code it did *not* take, so - // it points at that code the way every other diagnostic does - let diagnostics: Vec = built - .declined - .iter() - .map(|declined| declined_diagnostic(*file, declined)) - .collect(); - render_diagnostics(&db, &diagnostics)?; + if let Err(error) = stage_by_typed_markers(&db, &mut staging, &roots, &root) { + break 'compiling Err(error); + } + if let Err(error) = write_sourcemap_module(&mut staging, &entries) { + break 'compiling Err(error); } - compiled += 1; + // whether the tree actually *holds* extensions, which is the question a + // reader of the record is asking — `--emit-c-only` writes the generated C + // and stops, so its tree is as replaceable one file at a time as a + // `by build`'s. the generated `.c` beside it is then a description of the + // source at the time it was written, and a later re-stage does not + // regenerate it; nothing loads it, so the two are allowed to drift + let holds_extensions = !emit_c_only && compiled > 0; + if let Err(error) = stage_build_record( + &mut staging, + &BuildRecord::new(&root, &roots, None, holds_extensions, &tree_config), + ) { + break 'compiling Err(error); + } + Ok(()) + }; + + // whatever happened above, the manifest is brought into line with what is + // actually on disk. a manifest left describing the run before would have the + // *next* run prune against a tree that no longer exists, leaving behind files + // no source produces — an importable module nobody wrote + staging.finish()?; + + compiling?; + // the transpile's verdict is answered after the tree is finished, the way + // `build` answers it: a diagnostic must not cost the tree, because a `build/` + // a debugger cannot read is worse than one built from a partial check + if !transpiled? { + return Ok(ExitStatus::Failure); } eprintln!("\ncompiled {compiled} module(s)"); @@ -1415,7 +1569,7 @@ fn reverse_dir_converting( /// Forward-transpile every `.by` under `dir` into a `.py` next to it, using one /// shared project db so cross-module types resolve (the same path as `by -/// build`, but written in place rather than to `out/`). +/// build`, but written in place rather than to `build/`). #[allow(clippy::print_stderr)] fn forward_dir(dir: &Path, config: &Config) -> anyhow::Result { let (db, handles, rebuilder, _root) = build_project_db(dir, BY_SOURCES, None)?; diff --git a/crates/ty/src/cli-reference.md b/crates/ty/src/cli-reference.md index d939c24546..a3ec9abc58 100644 --- a/crates/ty/src/cli-reference.md +++ b/crates/ty/src/cli-reference.md @@ -24,8 +24,8 @@ by run main --min-version 3.12 ## `by build` -transpile every `.by` file in the project to `out/`, mirroring the module -layout — a src-layout project's source root is stripped, so `out/` is +transpile every `.by` file in the project to `build/`, mirroring the module +layout — a src-layout project's source root is stripped, so `build/` is importable as it stands: ```sh @@ -33,9 +33,9 @@ by build ``` ```text -main.by -> out/main.py -utils.by -> out/utils.py -src/package_name/main.by -> out/package_name/main.py +main.by -> build/main.py +utils.by -> build/utils.py +src/package_name/main.by -> build/package_name/main.py ``` generated `.py` files are ordinary Python — run them with any Python tool diff --git a/crates/ty/src/lib.rs b/crates/ty/src/lib.rs index c81874bde3..e717cc827a 100644 --- a/crates/ty/src/lib.rs +++ b/crates/ty/src/lib.rs @@ -162,7 +162,7 @@ fn run_command(command: Command) -> anyhow::Result { by_commands::cmd_build( min_version.as_deref(), &lowering, - out.as_deref().unwrap_or(Path::new("out")), + out.as_deref().unwrap_or(Path::new("build")), print_manifest, ) } diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 00993a4658..7e5a2ed99c 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -58,6 +58,398 @@ fn run_transpile(source: &str, extra_args: &[&str]) -> String { String::from_utf8(output.stdout).unwrap() } +/// A project whose program reads a data file sitting beside it. +/// +/// Written by each `compile` staging test, which then differ only in what they do +/// to the tree afterwards. +fn resource_project(name: &str) -> PathBuf { + let dir = cli_root().join(name); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(dir.join("data")).unwrap(); + fs::write( + dir.join("pyproject.toml"), + "[project]\nname=\"s\"\nversion=\"0\"\nrequires-python=\">=3.13\"\n", + ) + .unwrap(); + fs::write( + dir.join("data").join("config.json"), + "{\"greeting\": \"hi\"}\n", + ) + .unwrap(); + fs::write( + dir.join("helper.py"), + "def helper() -> int:\n return 1\n", + ) + .unwrap(); + fs::write( + dir.join("main.by"), + "from pathlib import Path\n\n\ndef go() -> str:\n \ + return (Path(__file__).parent / \"data\" / \"config.json\").read_text()\n", + ) + .unwrap(); + dir +} + +/// Whether `by` refused because this host's interpreter is below the native floor. +/// +/// `by compile` probes an interpreter before it does anything, and +/// `by_build::MINIMUM_PYTHON` refuses one older than 3.11 by name. On a host whose +/// ambient `python3` is older that is not a failure of the code under test, and a +/// test that asserted its way through it reported a wall of unrelated noise — so +/// every `compile` test here skips on it, the way `by_build`'s own suite does. +/// Pin `PYTHON` to a 3.11+ interpreter to actually run them. +#[allow( + clippy::print_stderr, + reason = "skip notices belong on the harness's stderr" +)] +fn refused_for_python_version(result: &std::process::Output) -> bool { + let refused = String::from_utf8_lossy(&result.stderr) + .contains("a native build needs python 3.11 or later"); + if refused { + eprintln!("skipping: this host's python is below the native compilation floor"); + } + refused +} + +/// Run `by compile` in `dir`, or `None` when this host cannot compile natively. +fn compile_in(dir: &Path) -> Option { + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["compile", "--emit-c-only"]) + .current_dir(dir) + .output() + .expect("failed to spawn by"); + if refused_for_python_version(&result) { + return None; + } + assert!( + result.status.success(), + "by exited with error:\n{}", + String::from_utf8_lossy(&result.stderr) + ); + Some(result) +} + +#[test] +fn compile_carries_the_rest_of_the_project_into_the_output_tree() { + // a compiled module is only half of a project. `main.by` reads its data file + // relative to itself, and the extension's `__file__` is its own place in the + // output tree — so a tree holding artefacts and nothing else fails on the + // first `open`, with a `FileNotFoundError` naming a path in a directory the + // author never wrote anything to. `compile` writes everything `build` writes + // and the extensions as well, so a compiled module finds its data exactly + // where its interpreted twin would + let dir = resource_project("by_cli_compile_resources"); + if compile_in(&dir).is_none() { + return; + } + + let out = dir.join("build"); + assert!(out.join("main.c").exists(), "the module is compiled"); + assert!( + out.join("data").join("config.json").exists(), + "a data file lands at the same relative place it had in the source" + ); + assert!( + out.join("helper.py").exists(), + "a hand-written python module beside the source is carried over too" + ); + assert!( + out.join("main.py").exists(), + "the `.by` is transpiled too, so the module imports whether or not its \ + extension was built" + ); + assert!( + out.join("_by_sourcemap.py").exists() && out.join("_by_build.json").exists(), + "the tree describes itself, the way a `by build` tree does" + ); +} + +#[test] +fn compiling_one_module_leaves_the_others_importable() { + // the docs offer `by compile app.hot` as "compile one module, leave the rest + // interpreted". while `compile` wrote artefacts alone that was not what + // happened: the modules nobody named reached the tree as `.by`, which python + // cannot import, and the second invocation's `finish` took the first's + // artefact back as well — so `compile a` then `compile b` left a tree that + // could import neither + let dir = resource_project("by_cli_compile_one_of_many"); + fs::write(dir.join("other.by"), "def other() -> int:\n return 2\n").unwrap(); + + let compile_one = |name: &str| -> bool { + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["compile", "--emit-c-only", name]) + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + if refused_for_python_version(&result) { + return false; + } + assert!( + result.status.success(), + "by exited with error:\n{}", + String::from_utf8_lossy(&result.stderr) + ); + true + }; + let out = dir.join("build"); + + if !compile_one("main.by") { + return; + } + assert!(out.join("main.c").exists(), "the named module is compiled"); + assert!( + out.join("other.py").exists(), + "a module nobody named still reaches the tree as importable python" + ); + + assert!(compile_one("other.by"), "the second compile ran"); + assert!( + out.join("other.c").exists(), + "the newly named one is compiled" + ); + assert!( + out.join("main.py").exists(), + "and the previously named one is still importable" + ); +} + +#[test] +fn a_compile_leaves_a_build_tree_readable() { + // `compile` and `build` write to the same directory by design. while + // `compile` wrote artefacts alone it took the sourcemap and the build record + // with it, and `by restage` — the language server's single-file re-stage — + // then refused the tree for having no `_by_build.json`, so a compile silently + // disabled the editor plugin against it + let dir = resource_project("by_cli_compile_then_restage"); + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("build") + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + assert!(result.status.success()); + if compile_in(&dir).is_none() { + return; + } + + let restaged = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["restage", "build", "main.by"]) + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + let answer = String::from_utf8_lossy(&restaged.stdout); + assert!( + answer.contains("\"generated\""), + "the tree is still one a re-stage can read:\n{answer}" + ); +} + +#[test] +fn a_build_with_nothing_to_write_leaves_no_output_directory() { + // `by build` created the output directory before it knew whether it had + // anything to put in it, so a project with no `.by` files — one whose sources + // are all python, say — was left holding an empty `build/` it never asked for + let dir = cli_root().join("by_cli_build_no_litter"); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + fs::write( + dir.join("pyproject.toml"), + "[project]\nname=\"s\"\nversion=\"0\"\nrequires-python=\">=3.13\"\n", + ) + .unwrap(); + fs::write(dir.join("only.py"), "x = 1\n").unwrap(); + + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("build") + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + let stderr = String::from_utf8_lossy(&result.stderr); + assert!(result.status.success(), "by build failed:\n{stderr}"); + assert!( + stderr.contains("no .by files found"), + "the build found nothing to do:\n{stderr}" + ); + assert!( + !dir.join("build").exists(), + "a build that wrote nothing left a directory behind:\n{stderr}" + ); +} + +#[test] +fn a_second_output_directory_is_not_carried_into_the_first() { + // a project can have more than one output tree — `by build --out one` beside + // `by build --out two`. only the directory *this* run was given is known to + // be an output from its arguments; the other is recognised by the + // `.by-manifest` it carries, and without that it is carried over as though it + // were source, putting a whole copy of one tree inside the other + let dir = cli_root().join("by_cli_two_outputs"); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + fs::write( + dir.join("pyproject.toml"), + "[project]\nname=\"s\"\nversion=\"0\"\nrequires-python=\">=3.13\"\n", + ) + .unwrap(); + fs::write(dir.join("main.by"), "x = 1\n").unwrap(); + fs::write(dir.join("data.json"), "{}\n").unwrap(); + + let build_into = |name: &str| { + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["build", "--out", name]) + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + assert!( + result.status.success(), + "by build failed:\n{}", + String::from_utf8_lossy(&result.stderr) + ); + }; + build_into("one"); + build_into("two"); + + assert!(dir.join("two").join("main.py").exists(), "the build ran"); + assert!( + !dir.join("two").join("one").exists(), + "the first output tree was copied into the second" + ); +} + +#[test] +fn compile_takes_back_what_the_previous_compile_wrote() { + // the output tree is a mirror rather than a pile: a resource deleted from the + // source is deleted from the tree. without the manifest it would keep being + // read, and a wheel built from the same tree would ship it + let dir = resource_project("by_cli_compile_stale"); + if compile_in(&dir).is_none() { + return; + } + let out = dir.join("build"); + assert!(out.join("data").join("config.json").exists()); + + fs::remove_file(dir.join("data").join("config.json")).unwrap(); + assert!(compile_in(&dir).is_some(), "the second compile ran"); + assert!( + !out.join("data").join("config.json").exists(), + "a resource the source no longer has is taken back out of the tree" + ); +} + +#[test] +fn a_build_takes_back_the_artifacts_a_compile_left() { + // `compile` and `build` write to the same directory by default, and python's + // finder prefers an extension to source. an artefact left behind by an + // earlier `compile` would therefore go on shadowing the `.py` this `build` + // writes in its place — which is why `compile` records what it produced even + // though `by_build` is what laid it out + let dir = resource_project("by_cli_compile_then_build"); + if compile_in(&dir).is_none() { + return; + } + let out = dir.join("build"); + assert!(out.join("main.c").exists()); + + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("build") + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + assert!( + result.status.success(), + "by exited with error:\n{}", + String::from_utf8_lossy(&result.stderr) + ); + assert!(out.join("main.py").exists(), "the build wrote the module"); + assert!( + !out.join("main.c").exists(), + "what the compile produced is taken back" + ); +} + +#[test] +fn compile_refuses_to_carry_a_file_over_an_artifact_it_wrote() { + // a project can keep a `main.c` of its own beside `main.by` — and the + // compiler writes its generated C under that same name. carrying the + // hand-written one over would leave a tree whose generated half is somebody + // else's file, with nothing said about it, so it is reported the way two + // sources claiming one module are + let dir = resource_project("by_cli_compile_artifact_collision"); + fs::write(dir.join("main.c"), "/* hand-written */\n").unwrap(); + + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["compile", "--emit-c-only"]) + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + if refused_for_python_version(&result) { + return; + } + let stderr = String::from_utf8_lossy(&result.stderr); + assert!(!result.status.success(), "by should have failed:\n{stderr}"); + assert!( + stderr.contains("already wrote an artifact of that name"), + "the collision is named:\n{stderr}" + ); + assert!( + !fs::read_to_string(dir.join("build").join("main.c")) + .unwrap() + .contains("hand-written"), + "the generated C is left as the compiler wrote it" + ); +} + +#[test] +fn compile_does_not_read_the_tree_it_writes() { + // the output holds a copy of every resource the build carried over, including + // the project's `.py` modules. a second `compile` that walked them would + // compile each module twice — once from the source and once from the copy — + // and the two would claim the same artifact + // + // the directory is deliberately *not* one of the names the project walk skips + // by default (`build`, `out`, `target`, …). those are turned away whoever + // asks, so a tree written to one of them would pass this test even if the + // build never told the database where its own output was going. + // + // two mechanisms keep `generated/` out — the output this run was given, and + // the `.by-manifest` any output carries — and either alone is enough here. + // the manifest is the only one that covers an output this run was *not* + // given, which `a_second_output_directory_is_not_carried_into_the_first` + // is for + let dir = resource_project("by_cli_compile_not_own_input"); + let compile = || -> Option { + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["compile", "--emit-c-only", "-o", "generated"]) + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + if refused_for_python_version(&result) { + return None; + } + assert!( + result.status.success(), + "by exited with error:\n{}", + String::from_utf8_lossy(&result.stderr) + ); + // the count `compile` reports, not the per-artifact lines: `--emit-c-only` + // does not print those, so counting them compares nothing to nothing + let stderr = String::from_utf8_lossy(&result.stderr); + let reported = stderr + .lines() + .find_map(|line| line.strip_prefix("compiled ")?.strip_suffix(" module(s)")) + .and_then(|count| count.parse::().ok()); + Some(reported.unwrap_or_else(|| panic!("no module count in:\n{stderr}"))) + }; + let Some(first) = compile() else { + return; + }; + assert!(first > 0, "the first compile compiled something"); + assert_eq!( + Some(first), + compile(), + "the second compile sees the same sources as the first" + ); +} + #[test] fn compile_emits_only_the_files_it_was_given_and_still_resolves_the_others() { // `by compile a.py` used to compile every source in the project and ignore the @@ -94,7 +486,7 @@ fn compile_emits_only_the_files_it_was_given_and_still_resolves_the_others() { ) .unwrap(); - let out = dir.join("out"); + let out = dir.join("build"); let result = Command::new(env!("CARGO_BIN_EXE_by")) .args(["compile", "wanted.py", "-o"]) .arg(&out) @@ -102,6 +494,9 @@ fn compile_emits_only_the_files_it_was_given_and_still_resolves_the_others() { .current_dir(&dir) .output() .expect("failed to spawn by"); + if refused_for_python_version(&result) { + return; + } assert!( result.status.success(), "by exited with error:\n{}", @@ -165,6 +560,9 @@ fn compile_writes_each_package_member_at_its_own_place_in_the_output_tree() { .current_dir(&dir) .output() .expect("failed to spawn by"); + if refused_for_python_version(&result) { + return; + } assert!( result.status.success(), "by exited with error:\n{}", @@ -221,6 +619,9 @@ fn compile_refuses_two_sources_that_would_write_the_same_artifact() { .current_dir(&dir) .output() .expect("failed to spawn by"); + if refused_for_python_version(&result) { + return; + } assert!(!result.status.success(), "the clash is refused"); let stderr = String::from_utf8_lossy(&result.stderr); assert!( @@ -259,6 +660,9 @@ fn compile_declines_a_package_body_whose_package_has_no_importable_name() { .current_dir(&dir) .output() .expect("failed to spawn by"); + if refused_for_python_version(&result) { + return; + } let stderr = String::from_utf8_lossy(&result.stderr); // declining one source is not a failed build — the rest of the project is // compiled, and what was left out is said rather than silently produced @@ -287,7 +691,7 @@ async def total(s: str, n: int) -> int: let file = dir.join("sound.by"); std::fs::write(&file, source).unwrap(); - let emitted = |spec: &str| -> String { + let emitted = |spec: &str| -> Option { let out = dir.join(spec); let status = Command::new(env!("CARGO_BIN_EXE_by")) .args(["compile"]) @@ -298,20 +702,26 @@ async def total(s: str, n: int) -> int: .current_dir(&dir) .output() .expect("failed to spawn by"); + if refused_for_python_version(&status) { + return None; + } assert!( status.status.success(), "by exited with error:\n{}", String::from_utf8_lossy(&status.stderr) ); - std::fs::read_to_string(out.join("sound.c")).expect("the C is readable") + Some(std::fs::read_to_string(out.join("sound.c")).expect("the C is readable")) }; + let Some(all) = emitted("all") else { + return; + }; assert!( - emitted("all").contains("_soundness_check"), + all.contains("_soundness_check"), "`all` puts the entry checks in the fallback" ); assert!( - !emitted("none").contains("_soundness_check"), + !emitted("none").is_some_and(|none| none.contains("_soundness_check")), "`none` leaves them out, so the flag is what made the difference" ); } @@ -1797,11 +2207,11 @@ fn build_skips_a_source_it_cannot_read() { "the skipped file must be reported:\n{stderr}" ); assert_eq!( - fs::read_to_string(dir.path().join("out/good.py")).unwrap(), + fs::read_to_string(dir.path().join("build/good.py")).unwrap(), "x = 1\n" ); assert!( - !dir.path().join("out/bad.py").exists(), + !dir.path().join("build/bad.py").exists(), "an unreadable source must not be emitted as an empty module" ); } @@ -2016,7 +2426,7 @@ fn build_renders_parse_error_and_aborts() { "stderr should include invalid-syntax diagnostic:\n{stderr}" ); assert!( - !dir.path().join("out").join("bad.py").exists(), + !dir.path().join("build").join("bad.py").exists(), "build should not emit output when parse error present" ); } @@ -2146,9 +2556,9 @@ fn build_skips_hidden_directories() { !stderr.contains("junk"), "hidden-directory file must not be checked:\n{stderr}" ); - assert!(dir.path().join("out").join("main.py").exists()); + assert!(dir.path().join("build").join("main.py").exists()); assert!( - !dir.path().join("out").join(".claude").exists(), + !dir.path().join("build").join(".claude").exists(), "hidden-directory file must not be emitted" ); } @@ -2182,7 +2592,7 @@ fn build_writes_what_the_project_exports_into_its_marker() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); assert_eq!( - fs::read_to_string(dir.path().join("out").join("my_lib").join("by.typed")).unwrap(), + fs::read_to_string(dir.path().join("build").join("my_lib").join("by.typed")).unwrap(), "exported-dependencies = [\"numpy\"]\n" ); } @@ -2204,14 +2614,14 @@ fn build_writes_a_marker_for_a_project_that_exports_nothing() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); - let marker = dir.path().join("out").join("my_lib").join("by.typed"); - assert!(marker.exists(), "expected out/my_lib/by.typed:\n{stderr}"); + let marker = dir.path().join("build").join("my_lib").join("by.typed"); + assert!(marker.exists(), "expected build/my_lib/by.typed:\n{stderr}"); assert_eq!(fs::read_to_string(marker).unwrap(), ""); } /// a src-layout project's `src/pkg/main.by` is the module `pkg.main`, so the /// emitted tree has to be rooted at `src` — mirroring the directory instead -/// emits `out/src/pkg/main.py`, whose module is `src.pkg.main`, a name nothing +/// emits `build/src/pkg/main.py`, whose module is `src.pkg.main`, a name nothing /// imports and `run.main` cannot sensibly be set to #[test] fn build_mirrors_the_module_tree_not_the_directory_tree() { @@ -2235,10 +2645,10 @@ fn build_mirrors_the_module_tree_not_the_directory_tree() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); - let out = dir.path().join("out"); + let out = dir.path().join("build"); assert!( out.join("package_name").join("main.py").exists(), - "expected out/package_name/main.py:\n{stderr}" + "expected build/package_name/main.py:\n{stderr}" ); assert!( !out.join("src").exists(), @@ -2246,7 +2656,7 @@ fn build_mirrors_the_module_tree_not_the_directory_tree() { ); } -/// `out/` outlives the build that wrote it — a test runner, a debugger or an +/// `build/` outlives the build that wrote it — a test runner, a debugger or an /// editor reads it later — so it is the tree where a `.by` really can be saved /// after the transpile, and the one that needs the digests to say so #[test] @@ -2262,7 +2672,7 @@ fn build_writes_a_sourcemap_beside_the_generated_python() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); - let out = dir.path().join("out"); + let out = dir.path().join("build"); let map = fs::read_to_string(out.join("_by_sourcemap.py")).expect("sourcemap module"); // read the keys out of the file rather than rebuilding them: the build @@ -2320,7 +2730,7 @@ fn sourcemap_table_for(source: &str) -> (Vec>, String) { String::from_utf8_lossy(&output.stderr) ); - let out = dir.path().join("out"); + let out = dir.path().join("build"); let map = fs::read_to_string(out.join("_by_sourcemap.py")).expect("sourcemap module"); let generated = fs::read_to_string(out.join("main.py")).expect("generated module"); @@ -2452,7 +2862,7 @@ fn build_targets_the_configured_python_version() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); - let emitted = fs::read_to_string(dir.path().join("out/main.py")).unwrap(); + let emitted = fs::read_to_string(dir.path().join("build/main.py")).unwrap(); assert!( !emitted.contains("typing_extensions"), "a 3.13 target needs no shim:\n{emitted}" @@ -2477,7 +2887,7 @@ fn build_emits_every_file_it_can_past_a_broken_one() { let stderr = String::from_utf8_lossy(&output.stderr); assert!( - dir.path().join("out/good.py").exists(), + dir.path().join("build/good.py").exists(), "the parseable file must still be emitted:\n{stderr}" ); assert!( @@ -2515,8 +2925,8 @@ fn build_honours_src_exclude() { !stderr.contains("bad.by"), "excluded file checked:\n{stderr}" ); - assert!(dir.path().join("out/main.py").exists()); - assert!(!dir.path().join("out/tests").exists()); + assert!(dir.path().join("build/main.py").exists()); + assert!(!dir.path().join("build/tests").exists()); } #[test] @@ -2614,7 +3024,7 @@ fn transpile_directory_round_trips_through_build() { String::from_utf8_lossy(&output.stderr) ); - let built = fs::read_to_string(root.join("out/pkg/models.py")).unwrap(); + let built = fs::read_to_string(root.join("build/pkg/models.py")).unwrap(); assert!( built.contains("x if x is not None else 0"), "coalesce lowered back to python:\n{built}" @@ -3572,7 +3982,7 @@ fn build_carries_a_python_module_into_the_output() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); assert_eq!( - fs::read_to_string(dir.path().join("out/helper.py")).unwrap(), + fs::read_to_string(dir.path().join("build/helper.py")).unwrap(), "def shout(text: str) -> str:\n return text.upper()\n", "a hand-written python module belongs in the output verbatim" ); @@ -3598,7 +4008,7 @@ fn build_carries_data_files_into_the_output() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); - let out = dir.path().join("out").join("app"); + let out = dir.path().join("build").join("app"); assert_eq!( fs::read_to_string(out.join("settings.json")).unwrap(), "{\"key\": 1}\n" @@ -3625,11 +4035,11 @@ fn build_writes_a_stub_as_a_stub() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); assert!( - dir.path().join("out/shapes.pyi").exists(), + dir.path().join("build/shapes.pyi").exists(), "a `.byi` builds to a `.pyi`:\n{stderr}" ); assert!( - !dir.path().join("out/shapes.py").exists(), + !dir.path().join("build/shapes.py").exists(), "a stub emitted as a module shadows the implementation" ); } @@ -3683,14 +4093,14 @@ fn build_deletes_output_the_project_no_longer_has() { }; build(); - assert!(dir.path().join("out/removed.py").exists()); + assert!(dir.path().join("build/removed.py").exists()); fs::remove_file(dir.path().join("removed.by")).unwrap(); build(); - assert!(dir.path().join("out/kept.py").exists()); + assert!(dir.path().join("build/kept.py").exists()); assert!( - !dir.path().join("out/removed.py").exists(), + !dir.path().join("build/removed.py").exists(), "output for a source that is gone must not survive the next build" ); } @@ -3701,8 +4111,8 @@ fn build_deletes_output_the_project_no_longer_has() { fn build_leaves_output_it_never_wrote_alone() { let dir = tempfile::tempdir().expect("tempdir"); fs::write(dir.path().join("main.by"), "x = 1\n").unwrap(); - fs::create_dir_all(dir.path().join("out")).unwrap(); - fs::write(dir.path().join("out/theirs.txt"), "hands off\n").unwrap(); + fs::create_dir_all(dir.path().join("build")).unwrap(); + fs::write(dir.path().join("build/theirs.txt"), "hands off\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) .arg("build") @@ -3715,7 +4125,7 @@ fn build_leaves_output_it_never_wrote_alone() { "by build failed:\n{}", String::from_utf8_lossy(&output.stderr) ); - assert!(dir.path().join("out/theirs.txt").exists()); + assert!(dir.path().join("build/theirs.txt").exists()); } #[test] @@ -3732,7 +4142,7 @@ fn build_writes_where_out_says() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); assert!(dir.path().join("elsewhere/main.py").exists()); - assert!(!dir.path().join("out").exists()); + assert!(!dir.path().join("build").exists()); } /// the output directory is not an input to itself, wherever it is put @@ -3847,9 +4257,9 @@ fn build_does_not_ship_what_lives_outside_the_source_root() { ); // it is still built, because it is still the project — running the tests out // of the output tree is the point of building them - assert!(dir.path().join("out/tests/test_it.py").exists()); + assert!(dir.path().join("build/tests/test_it.py").exists()); assert!( - !dir.path().join("out/tests/by.typed").exists(), + !dir.path().join("build/tests/by.typed").exists(), "a marker only speaks for what the project ships" ); } @@ -3872,7 +4282,7 @@ fn build_marks_a_package_as_carrying_its_sources() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); - let out = dir.path().join("out").join("app"); + let out = dir.path().join("build").join("app"); assert!( out.join("by.typed").exists(), "expected a marker:\n{stderr}" @@ -3905,7 +4315,7 @@ fn build_ships_python_only_when_the_project_says_so() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); - let out = dir.path().join("out").join("app"); + let out = dir.path().join("build").join("app"); assert!(out.join("__init__.py").exists()); assert!( !out.join("__init__.by").exists(), @@ -3942,9 +4352,9 @@ fn build_honours_the_configured_exclusions() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); - assert!(dir.path().join("out/public.json").exists()); + assert!(dir.path().join("build/public.json").exists()); assert!( - !dir.path().join("out/secrets.json").exists(), + !dir.path().join("build/secrets.json").exists(), "an excluded file must not reach the output" ); } @@ -3976,11 +4386,11 @@ fn build_carries_a_directory_a_negated_exclude_takes_back() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); assert!( - dir.path().join("out/dist/kept.py").exists(), + dir.path().join("build/dist/kept.py").exists(), "the re-included `.by` builds:\n{stderr}" ); assert!( - dir.path().join("out/dist/kept.json").exists(), + dir.path().join("build/dist/kept.json").exists(), "and so does everything beside it:\n{stderr}" ); } @@ -4018,7 +4428,7 @@ fn build_ships_a_source_directory_that_is_itself_a_package() { stdout.lines().any(|line| line == "package src"), "`src.mymod` is the module, so `src` is the package:\n{stdout}" ); - assert!(dir.path().join("out/src/mymod/__init__.py").exists()); + assert!(dir.path().join("build/src/mymod/__init__.py").exists()); } /// lowering for an older python can put a name in the output that only @@ -4143,7 +4553,7 @@ fn build_emitting(source: &str, settled: Option<&str>) -> String { let mut command = Command::new(env!("CARGO_BIN_EXE_by")); command - .args(["build", "--out", "out"]) + .args(["build", "--out", "build"]) .current_dir(dir.path()); match settled { Some(settled) => command.env("BY_BUILD_LOWERING", settled), @@ -4154,7 +4564,7 @@ fn build_emitting(source: &str, settled: Option<&str>) -> String { let output = command.output().expect("failed to spawn by"); let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); assert!(output.status.success(), "by build failed:\n{stderr}"); - fs::read_to_string(dir.path().join("out").join("main.py")).expect("emitted python") + fs::read_to_string(dir.path().join("build").join("main.py")).expect("emitted python") } /// the builds inside a `--wheels` release are the ones that transpile, so every @@ -4674,7 +5084,7 @@ fn build_from_a_subdirectory_builds_the_project() { assert!(output.status.success(), "by build failed:\n{stderr}"); assert!( elsewhere - .join("out") + .join("build") .join("app") .join("__init__.py") .exists(), @@ -4825,9 +5235,9 @@ fn build_does_not_carry_a_compilers_output_directory() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); - assert!(dir.path().join("out").join("main.py").exists()); + assert!(dir.path().join("build").join("main.py").exists()); assert!( - !dir.path().join("out").join("target").exists(), + !dir.path().join("build").join("target").exists(), "a build directory must not be carried into the build:\n{stderr}" ); } diff --git a/crates/ty/tests/by_restage.rs b/crates/ty/tests/by_restage.rs index 061ff2769b..954d9f613e 100644 --- a/crates/ty/tests/by_restage.rs +++ b/crates/ty/tests/by_restage.rs @@ -24,7 +24,7 @@ fn write_project(dir: &Path) { fn build(dir: &Path) { let status = Command::new(env!("CARGO_BIN_EXE_by")) - .args(["build", "--out", "out"]) + .args(["build", "--out", "build"]) .current_dir(dir) .status() .expect("`by build` should run"); @@ -33,7 +33,7 @@ fn build(dir: &Path) { fn restage(dir: &Path, file: &str) -> (bool, serde_json::Value) { let out = Command::new(env!("CARGO_BIN_EXE_by")) - .args(["restage", "out", file]) + .args(["restage", "build", file]) .current_dir(dir) .output() .expect("`by restage` should run"); @@ -78,13 +78,13 @@ build-stamps = true .unwrap(); let status = Command::new(env!("CARGO_BIN_EXE_by")) - .args(["build", "--out", "out", "--stamp", "GIT_SHA=abc123"]) + .args(["build", "--out", "build", "--stamp", "GIT_SHA=abc123"]) .current_dir(dir.path()) .status() .expect("`by build` should run"); assert!(status.success(), "`by build` failed"); - let on_disk = std::fs::read_to_string(dir.path().join("out/main.py")).unwrap(); + let on_disk = std::fs::read_to_string(dir.path().join("build/main.py")).unwrap(); assert!( on_disk.contains(r#"GIT_SHA: str = "abc123""#), "the build should have stamped the value:\n{on_disk}" @@ -111,7 +111,7 @@ fn restaging_a_file_nobody_edited_reproduces_the_build_exactly() { write_project(dir.path()); build(dir.path()); - let on_disk = std::fs::read_to_string(dir.path().join("out/main.py")).unwrap(); + let on_disk = std::fs::read_to_string(dir.path().join("build/main.py")).unwrap(); let (ok, answer) = restage(dir.path(), "main.by"); assert!(ok, "an unedited file should re-stage: {answer}"); @@ -137,7 +137,7 @@ fn the_generated_path_is_absolute_even_for_a_relative_build_directory() { generated.display() ); assert!( - generated.ends_with("out/main.py"), + generated.ends_with("build/main.py"), "{}", generated.display() ); @@ -284,7 +284,7 @@ fn a_directory_that_is_not_a_build_is_refused() { let dir = tempfile::tempdir().unwrap(); write_project(dir.path()); build(dir.path()); - std::fs::remove_file(dir.path().join("out/_by_build.json")).unwrap(); + std::fs::remove_file(dir.path().join("build/_by_build.json")).unwrap(); let (ok, answer) = restage(dir.path(), "main.by"); @@ -306,7 +306,7 @@ fn a_tree_built_by_another_by_is_refused() { write_project(dir.path()); build(dir.path()); - let record = dir.path().join("out/_by_build.json"); + let record = dir.path().join("build/_by_build.json"); let mut parsed: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(&record).unwrap()).unwrap(); parsed["byVersion"] = serde_json::Value::String("0.0.0+somethingelse".to_owned()); diff --git a/docs/basedpython/cli-reference.md b/docs/basedpython/cli-reference.md index 17573f651c..0fbd139d0f 100644 --- a/docs/basedpython/cli-reference.md +++ b/docs/basedpython/cli-reference.md @@ -14,7 +14,7 @@ in addition to the cli provided by `ty`, `by` includes: | command | what it does | | ------------------- | ------------------------------------------------------------------ | | `run` | transpile and run a module with `python -m ` | -| `build` | transpile every `.by`/`.byi` file and write to `out/` | +| `build` | transpile every `.by`/`.byi` file and write to `build/` | | `compile` | compile `.by` files to native CPython extension modules | | `generate-api-file` | write a public-api lockfile (see [api-lock](features/api-lock.md)) | | `transpile` | transpile a single file to stdout (reads stdin if no file) | @@ -104,18 +104,20 @@ reach for when one file is mid-edit artifacts and exit status answer different questions. everything that could be emitted is emitted, sourcemap and package markers included; the exit status says whether anything was *reported*. so a build that prints an error exits 1 while -still leaving a usable `out/`, and `by build && pytest out/tests` runs the tests +still leaving a usable `build/`, and `by build && pytest build/tests` runs the tests only against a tree the checker had nothing to say about -writes the transpiled python to `./out/` mirroring the *module* tree. a +writes the transpiled python to `./build/` mirroring the *module* tree. a src-layout project's `src/package_name/main.by` is the module -`package_name.main`, so it lands at `out/package_name/main.py` — `out/` is a +`package_name.main`, so it lands at `build/package_name/main.py` — `build/` is a directory you can put on `sys.path` as it stands, and `run.main` names a module -the same way an import does. the `out/` directory is **not** considered -first-party source for `by check` or `by generate-api-file` — it is regenerated -on every build +the same way an import does. `by generate-api-file` does not read `build/` as +first-party source — it is regenerated on every build. `by check` still walks it, +though, so a project that keeps its output beside its sources will see +diagnostics reported against generated python; exclude it with +`src.exclude = ["build"]` if that is in the way -alongside the python it writes `out/_by_sourcemap.py`, mapping each generated +alongside the python it writes `build/_by_sourcemap.py`, mapping each generated line back to the `.by` line it came from, with a digest of both files so a tool reading it can tell whether it still describes what is on disk — see [sourcemaps](development/sourcemaps.md) @@ -127,9 +129,9 @@ reading it can tell whether it still describes what is on disk — see > definition — see [native compilation](development/compilation/index.md) ```sh -by compile # every .by file under the project root → out/ +by compile # every .by file under the project root → build/ by compile hot.by # one file -by compile -o build hot.by # a different output directory +by compile -o native hot.by # a different output directory by compile --verbose # report every function left interpreted, and why by compile --emit-c-only # write the generated C without compiling it by compile --no-any # refuse to leave a gradual-typed function interpreted @@ -137,11 +139,56 @@ by compile --require-native # refuse to leave *any* function interpreted ``` the output directory mirrors the *module* tree, the way `by build`'s does: the -package member `pkg/sub/dup.py` lands at `out/pkg/sub/dup.cpython-313-darwin.so`, +package member `pkg/sub/dup.py` lands at `build/pkg/sub/dup.cpython-313-darwin.so`, and the package `pkg/sub/__init__.py` at -`out/pkg/sub/__init__.cpython-313-darwin.so` — so `out/` can go on `sys.path` as +`build/pkg/sub/__init__.cpython-313-darwin.so` — so `build/` can go on `sys.path` as it stands and every module imports under the dotted name it was compiled as +**`by compile` writes everything `by build` writes, and the extensions as well.** +a compiled module is not a program on its own: it reads its templates and +fixtures relative to itself, and imports the modules beside it. so the tree holds +the whole project as importable python — every `.by` transpiled, every +hand-written `.py`, `py.typed`, the data files, the sourcemap — with a native +extension beside each module that was compiled. python's finder prefers an +extension to source, so those modules load natively and the rest are interpreted, +which is what makes naming files a speed decision rather than a correctness one: + +```sh +by compile hot.by # hot is native, every other module is interpreted +``` + +naming files decides what is *compiled*, not what is written: the whole project +is checked and transpiled either way, so `by compile hot.by` costs what a +`by build` costs plus the one module's native compile. that is the price of the +tree being a program rather than a heap. + +`build/` is a mirror rather than a pile. what the previous run wrote and this one +did not is taken back, which matters most for extensions: python prefers one to +source, so an extension left behind by a deleted or renamed module would go on +shadowing the `.py` written in its place. the artefacts therefore describe the +*last* invocation — `by compile a.by` then `by compile b.by` leaves `b` native and +`a` interpreted, and `--emit-c-only`, which stops before the C compiler, leaves a +tree with no extensions at all. in every case each module still imports, so this +costs speed rather than correctness; `by compile` with no arguments compiles the +whole project. + +the directory is shared with `by build` on purpose — the same tree, built two +ways — and the two take each other's output back accordingly. it is also a name +setuptools and `python -m build` use, so a project using both writes them into one +directory; nothing is destroyed, because the manifest only ever takes back what +`by` itself wrote. + +a directory holding a `.by-manifest` is a build output, whoever wrote it, and is +never read as source or carried into another tree — a project that builds into +two directories would otherwise put a copy of each inside the other. that is by +the marker rather than by the directory's name, because `--out` can say anything. +a project that deliberately ships a directory containing one has to rename the +marker or keep the directory outside its source roots. + +> upgrading from a version that wrote `out/`: nothing migrates it, and nothing +> reads it any more. delete it — a stale `out/` holds importable python that no +> build refreshes. + `--no-any` buys no speed on its own — it is a **predictability contract**. a gradual type is the commonest reason a function silently stays interpreted, and a decline is invisible unless you look for it, so a module that means to be fully @@ -177,7 +224,7 @@ compiled functions are installed over the top ```console $ by compile hot.by --verbose -hot.by -> out/hot.cpython-313-darwin.so +hot.by -> build/hot.cpython-313-darwin.so declined describe: `list[int]` has no native representation yet compiled 1 module(s) diff --git a/docs/basedpython/development/compilation/index.md b/docs/basedpython/development/compilation/index.md index ad4b486810..3c857afac1 100644 --- a/docs/basedpython/development/compilation/index.md +++ b/docs/basedpython/development/compilation/index.md @@ -9,11 +9,22 @@ transpiled python it replaces: ```sh -by compile # compile the whole project to ./out/ +by compile # compile the whole project to ./build/ by compile app.hot # compile one module, leave the rest interpreted python -c "import app" # the extension is picked up ahead of the .py ``` +`by compile` writes everything `by build` writes and the extensions as well, so +`build/` holds the whole project as importable python — every `.by` transpiled, +every hand-written `.py`, the markers, the data files, the sourcemap — with a +native extension beside each module that was compiled. two things depend on that. +a compiled module reads its fixtures relative to itself the way its interpreted +twin does, so the twin property below is a claim about two modules in the *same* +project, not about one of them running somewhere the other's data never reached. +and the tree's manifest only means anything while one command authors the whole +tree: deleting what the last run wrote and this one did not is right for a mirror +and wrong for a heap. + the observable behaviour of a compiled module and its interpreted twin must be identical. that is not an aspiration, it is the property the entire test strategy is built on ([plan](plan.md#differential-testing)) @@ -88,7 +99,7 @@ AST and ty's inferred types │ │ │ cc + ld (platform) ▼ ▼ - out/*.py out/*.cpython-*.so + build/*.py build/*.cpython-*.so ``` ordinary python enters the same front end. `by_irbuild` lowers the `.py` AST diff --git a/docs/basedpython/development/compilation/technology.md b/docs/basedpython/development/compilation/technology.md index 6aa6ff60fa..9333b8663c 100644 --- a/docs/basedpython/development/compilation/technology.md +++ b/docs/basedpython/development/compilation/technology.md @@ -299,7 +299,7 @@ instead, three entry points over one core: ### the CLI ```sh -by compile # whole project → out/ +by compile # whole project → build/ by compile app.hot app.parse # a subset; the rest stays interpreted by compile --tier=1 # open world (see index.md) by compile --annotate # emit C next to the .by that produced it diff --git a/docs/basedpython/development/sourcemaps.md b/docs/basedpython/development/sourcemaps.md index 616f94412b..e8500b017b 100644 --- a/docs/basedpython/development/sourcemaps.md +++ b/docs/basedpython/development/sourcemaps.md @@ -33,7 +33,7 @@ a correct sourcemap is the single primitive all of these share span back to its `.by` line for a source-annotated diagnostic - `_by_sourcemap.py` carries two tables keyed by the generated `.py` path: `SOURCEMAP`, the `.by` path and its line table, and `DIGESTS`, the sha-256 of - both files that entry describes. `by build` writes it into `out/` beside the + both files that entry describes. `by build` writes it into `build/` beside the python it describes, and `by run` into the temporary tree it executes — where it lives only as long as that run. see [staleness](#staleness) below @@ -90,7 +90,7 @@ never read is invisible to them the traceback shim is the first consumer, and it shows what refusing looks like: when either digest disagrees it leaves the frame in the generated python and writes a note saying which file no longer matches. a frame pointing at -`out/main.py` is a worse answer, but a frame quoting a `.by` that has been +`build/main.py` is a worse answer, but a frame quoting a `.by` that has been rewritten since is a false one both tables are keyed by the generated path exactly as the map spells it. the diff --git a/docs/basedpython/features/api-lock.md b/docs/basedpython/features/api-lock.md index ebf577782d..f69a12d023 100644 --- a/docs/basedpython/features/api-lock.md +++ b/docs/basedpython/features/api-lock.md @@ -71,7 +71,7 @@ record. fields are colon-separated, and the records are sorted lexicographically - any symbol whose simple name starts with `_` (unless it's one of the conventional public dunders above) - stdlib, site-packages, and other non-first-party modules -- output from `by build` — the `out/` directory is not considered first-party +- output from `by build` — the `build/` directory is not considered first-party source for lockfile purposes ## determinism and stability diff --git a/docs/basedpython/frameworks/pytest.md b/docs/basedpython/frameworks/pytest.md index 5d8b56b32d..e450bdbbd9 100644 --- a/docs/basedpython/frameworks/pytest.md +++ b/docs/basedpython/frameworks/pytest.md @@ -131,7 +131,7 @@ custom collection configs in `pytest.ini` or `pyproject.toml` aren't read yet, s ```sh by build -pytest out/ +pytest build/ ``` ## see also diff --git a/docs/basedpython/getting-started.md b/docs/basedpython/getting-started.md index 956eee282d..981ec036f5 100644 --- a/docs/basedpython/getting-started.md +++ b/docs/basedpython/getting-started.md @@ -63,30 +63,30 @@ see [configuration](configuration.md) for everything that can go in there ## building -`by build` writes the project to `out/` as python: +`by build` writes the project to `build/` as python: ```sh by build ``` ```text -main.by -> out/main.py -utils.by -> out/utils.py +main.by -> build/main.py +utils.by -> build/utils.py build complete (2 files) ``` that is the whole project, not only its `.by` files — a hand-written `.py` module, a `py.typed`, a data file the program reads are all carried across to -the same place, so `out/` runs the way the source does +the same place, so `build/` runs the way the source does the generated `.py` files are ordinary python. run them with any python tool: ```sh -python out/main.py -pytest out/ -mypy out/ -ruff check out/ +python build/main.py +pytest build/ +mypy build/ +ruff check build/ ``` to ship the project rather than run it, build a wheel — see @@ -105,7 +105,7 @@ uv build by build - name: Test - run: pytest out/ + run: pytest build/ ``` ## converting python to basedpython @@ -194,7 +194,7 @@ echo 'a = b ?? 1' | by transpile output goes to stdout - redirect it to a file if you want to keep it (`by transpile hello.by > hello.py`). use `by build` to transpile a whole -project into `out/` +project into `build/` ## forward references diff --git a/docs/basedpython/packaging.md b/docs/basedpython/packaging.md index 7f17db613c..dc80217a4c 100644 --- a/docs/basedpython/packaging.md +++ b/docs/basedpython/packaging.md @@ -39,21 +39,21 @@ leave off `--lib` and you also get a `main.by` and a configured entry point, so ## what a build produces -`by build` writes the project to `out/` as python. that is the whole project, +`by build` writes the project to `build/` as python. that is the whole project, not only its `.by` files: ```text -src/app/main.by -> out/app/main.py -src/app/helper.py -> out/app/helper.py -src/app/settings.json -> out/app/settings.json -src/app/py.typed -> out/app/py.typed +src/app/main.by -> build/app/main.py +src/app/helper.py -> build/app/helper.py +src/app/settings.json -> build/app/settings.json +src/app/py.typed -> build/app/py.typed ``` a `.by` file is transpiled; everything else is carried across unchanged, to the same place. the one rearrangement is the source root — `src/app/main.by` is the module `app.main`, so it lands at `app/main.py` and not at `src/app/main.py` -`out/` is a mirror, not a pile: what a previous build wrote and this one did not +`build/` is a mirror, not a pile: what a previous build wrote and this one did not is deleted, so a module you renamed does not go on being importable a stub stays a stub. `a.byi` builds to `a.pyi`, never to `a.py` @@ -230,8 +230,8 @@ checker guessing uv sync ``` -installs the project pointing at `out/`, so `by build` is what refreshes an -editable install. run it after editing, the same way any compiled language +installs the project pointing at `build/` — `by build`'s own output directory, so +a plain `by build` is what refreshes an editable install. run it after editing, the same way any compiled language rebuilds before its changes are visible ## a single-module project diff --git a/python/basedpython/build.py b/python/basedpython/build.py index e07d880833..cb424a66d5 100644 --- a/python/basedpython/build.py +++ b/python/basedpython/build.py @@ -55,8 +55,10 @@ # where `build_editable` stages the project. it is `by build`'s own default # output directory on purpose: an editable install points python at this tree, so -# a plain `by build` is what refreshes an editable install -EDITABLE_STAGING_DIRECTORY = "out" +# a plain `by build` is what refreshes an editable install. the two have to be +# renamed together — an install left pointing at a directory `by build` no longer +# writes goes stale with nothing said about it +EDITABLE_STAGING_DIRECTORY = "build" class BuildError(Exception): diff --git a/scripts/check_ecosystem_roundtrip.py b/scripts/check_ecosystem_roundtrip.py index 24a8ba97fe..d26a68a4d5 100755 --- a/scripts/check_ecosystem_roundtrip.py +++ b/scripts/check_ecosystem_roundtrip.py @@ -14,7 +14,7 @@ `by` commands:: by transpile --reverse # python -> basedpython (in place) - by build # basedpython -> python (-> out/) + by build # basedpython -> python (-> build/) `by build` uses one shared project db, so cross-module types resolve — the same path real `.by` projects take. This is far cheaper than spawning `by` per file: @@ -219,7 +219,7 @@ class ProjectOutcome(NamedTuple): """The result of round-trip-building a project with a single binary.""" error: str | None - # relpath under out/ -> built python; empty when error is set + # relpath under build/ -> built python; empty when error is set outputs: dict[str, bytes] @@ -333,7 +333,7 @@ async def roundtrip_project( build_mem_limit_bytes: int | None, build_timeout: float | None, ) -> ProjectOutcome: - """Reverse the whole project (py->by) then build it (by->py via out/).""" + """Reverse the whole project (py->by) then build it (by->py via build/).""" rc, _, err = await _run( by, ["transpile", "--reverse", "--min-version", ROUNDTRIP_MIN_VERSION, str(root)], @@ -356,10 +356,10 @@ async def roundtrip_project( mem_limit_bytes=build_mem_limit_bytes, timeout=build_timeout, ) - outputs = collect_outputs(root / "out") + outputs = collect_outputs(root / "build") # a non-zero exit from ty diagnostics (e.g. unresolved third-party imports — # the corpus is cloned source-only, so those are expected and unavoidable) - # is not a round-trip failure: the transpile still emitted `out/`. only a + # is not a round-trip failure: the transpile still emitted `build/`. only a # build the watchdog killed (137), or one that produced nothing at all, is a # genuine failure if rc == 137 or (rc != 0 and not outputs): @@ -944,7 +944,7 @@ async def setup_and_run(name: str) -> ProjectDiff | ProjectErrors: logger.warning("project %s failed setup: %s", name, e) return skipped_result(name, f"setup failed: {e}") finally: - # free the clone (source + generated `.by` + `out/`) so disk + # free the clone (source + generated `.by` + `build/`) so disk # doesn't accumulate across the shard's projects. --checkout # is for reuse, so only clean the temp-dir mode if args.checkout is None: From a403d0eb61366aa8d921040d016ee61a8aaf89ca Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 7 Sep 2026 19:26:40 +1000 Subject: [PATCH 2/7] answer `by check` out of a running server's warm state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit a `by check` builds a project database, resolves an environment, parses and infers every file, and then exits and throws all of it away. a language server running for that project has that work done and kept current, so `by check` now asks it first and says "using project server information" when it does. on an 11,850-file project that is 3.8s down to 0.8s a server publishes a loopback port and a secret into a per-user directory as it starts, and takes the record away as it exits. a caller reads the directory, sends the project root and the configuration it resolved, and gets the diagnostics back already rendered — rendering needs the database, and relative paths need the caller's own directory, so both cross the wire the answer only crosses when it is the answer a cold check would have given, so what the two sides compare is what each actually ended up with: the merged options across every layer, the resolved python version, platform and search paths, and the two database inputs that are not options at all — `force-exclude` and whether the diagnostics explain where each rule was turned on. an environment is discovered as much as configured, out of `VIRTUAL_ENV`, a `.venv` beside the project or uv's answer, and a server started by an editor discovers it from the editor's environment: two processes that agree about every option can still be checking against different site-packages. the caller therefore asks after building its database rather than before, since resolving all of that is the database's job, and building one is cheap next to checking with it the server also refuses a different build, an open buffer the editor has not saved — every open document, notebooks included, and buffers outside the project root this database has read — and a session diagnosing only its open files; that last one is the default, so this needs `diagnosticMode: workspace`. it re-reads the file system first, because its own picture of the project is whatever the editor's watcher reported and the caller can see the disk directly, and it does so between the two passes over a request so a check that was going to be refused does not make the editor re-pull its diagnostics. `--no-server` and `BY_NO_PROJECT_SERVER` switch it off, the second on both halves the caller rules itself out for anything that is not a whole-project check: `--watch`, a fix mode, explicit paths and a memory report. a failure writing the answer is a failure rather than "no server answered", so a broken pipe no longer prints half the diagnostics and then checks the whole project again connections are served one thread each, both directions are bounded in size and time, the token is compared in constant time, and the answer repeats the token back: a record outlives a killed server, and anything that binds the freed port could otherwise have answered "All checks passed!". a build is identified by its executable as well as its version, since the version string does not change when a dirty tree is rebuilt the tests build their requests from a real cold database the way the command line does, and assert that the rendering matches that database's own — the claim the whole design rests on. `scripts/check_project_server.py` drives a real server over stdin and stdout and asserts the same thing across two processes, plus that every flag which changes a check refuses --- Cargo.lock | 2 + .../src/generate_ty_env_vars_reference.rs | 6 +- crates/ty/docs/cli.md | 3 + crates/ty/docs/environment.md | 11 + crates/ty/src/args.rs | 11 + crates/ty/src/by_project_server.rs | 192 ++++++ crates/ty/src/lib.rs | 183 +++-- crates/ty/src/printer.rs | 5 + crates/ty/tests/by_e2e.rs | 109 +++ crates/ty/tests/cli/main.rs | 6 + crates/ty/tests/cli/project_server.rs | 65 ++ crates/ty_project/src/db.rs | 5 + crates/ty_project/src/lib.rs | 4 +- crates/ty_server/Cargo.toml | 2 + crates/ty_server/src/document.rs | 8 + crates/ty_server/src/lib.rs | 22 +- crates/ty_server/src/project_server/check.rs | 320 +++++++++ crates/ty_server/src/project_server/client.rs | 158 +++++ .../ty_server/src/project_server/discovery.rs | 194 ++++++ .../ty_server/src/project_server/listener.rs | 242 +++++++ crates/ty_server/src/project_server/mod.rs | 195 ++++++ .../ty_server/src/project_server/protocol.rs | 377 ++++++++++ crates/ty_server/src/server.rs | 38 +- crates/ty_server/src/server/main_loop.rs | 95 ++- crates/ty_server/src/session.rs | 45 ++ crates/ty_server/src/session/index.rs | 10 + crates/ty_server/tests/e2e/main.rs | 40 +- crates/ty_server/tests/e2e/project_server.rs | 646 ++++++++++++++++++ crates/ty_static/src/env_vars.rs | 10 + docs/basedpython/features/index.md | 2 + docs/basedpython/features/project-server.md | 82 +++ scripts/check_project_server.py | 293 ++++++++ scripts/check_project_server.py.lock | 15 + zensical.toml | 1 + 34 files changed, 3328 insertions(+), 69 deletions(-) create mode 100644 crates/ty/src/by_project_server.rs create mode 100644 crates/ty/tests/cli/project_server.rs create mode 100644 crates/ty_server/src/project_server/check.rs create mode 100644 crates/ty_server/src/project_server/client.rs create mode 100644 crates/ty_server/src/project_server/discovery.rs create mode 100644 crates/ty_server/src/project_server/listener.rs create mode 100644 crates/ty_server/src/project_server/mod.rs create mode 100644 crates/ty_server/src/project_server/protocol.rs create mode 100644 crates/ty_server/tests/e2e/project_server.rs create mode 100644 docs/basedpython/features/project-server.md create mode 100644 scripts/check_project_server.py create mode 100644 scripts/check_project_server.py.lock diff --git a/Cargo.lock b/Cargo.lock index 5fcc5eb587..9ca868479f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5151,6 +5151,7 @@ dependencies = [ "jod-thread", "libc", "lsp-server", + "rand 0.10.2", "regex", "ruff_db", "ruff_diagnostics", @@ -5179,6 +5180,7 @@ dependencies = [ "ty_project", "ty_python_core", "ty_python_semantic", + "ty_static", ] [[package]] diff --git a/crates/ruff_dev/src/generate_ty_env_vars_reference.rs b/crates/ruff_dev/src/generate_ty_env_vars_reference.rs index a9a474fe6f..fa404d6186 100644 --- a/crates/ruff_dev/src/generate_ty_env_vars_reference.rs +++ b/crates/ruff_dev/src/generate_ty_env_vars_reference.rs @@ -91,10 +91,12 @@ fn generate() -> String { output.push_str("# Environment variables\n\n"); - // Partition and sort environment variables into TY_ and external variables. + // Partition and sort environment variables into the ones ty defines and the ones it only + // reads. `BY_` as well as `TY_`, because basedpython's own variables are as much ty's as + // the rest — listing them beside `VIRTUAL_ENV` would say ty merely observes them. let (ty_vars, external_vars): (BTreeSet<_>, BTreeSet<_>) = EnvVars::metadata() .iter() - .partition(|(var, _)| var.starts_with("TY_")); + .partition(|(var, _)| var.starts_with("TY_") || var.starts_with("BY_")); output.push_str("ty defines and respects the following environment variables:\n\n"); diff --git a/crates/ty/docs/cli.md b/crates/ty/docs/cli.md index d66aa93ffe..aa7953501a 100644 --- a/crates/ty/docs/cli.md +++ b/crates/ty/docs/cli.md @@ -76,6 +76,9 @@ over all configuration files.

--ignore rule

Disables the rule. Can be specified multiple times. Use 'all' to apply to all rules.

--no-progress

Hide all progress outputs.

For example, spinners or progress bars.

+
--no-server

Check the project from scratch instead of asking a running language server.

+

A server holding this project has already parsed and inferred it, and answers a check out of that in a fraction of the time. It only answers when the two agree about the configuration and there is nothing unsaved, so this is a way to rule the server out rather than a way to get a different answer.

+

BY_NO_PROJECT_SERVER does the same, and also stops a server from listening.

--output-format output-format

The format to use for printing diagnostic messages

May also be set with the TY_OUTPUT_FORMAT environment variable.

Possible values:

    diff --git a/crates/ty/docs/environment.md b/crates/ty/docs/environment.md index 54bead8781..fce846bba4 100644 --- a/crates/ty/docs/environment.md +++ b/crates/ty/docs/environment.md @@ -2,6 +2,17 @@ ty defines and respects the following environment variables: +### `BY_NO_PROJECT_SERVER` + +Disable the project server: the language server answering `by` command lines. + +A server started for a project holds it parsed and inferred, and a `by check` in that +project asks the server rather than doing the work again. Set this to `1` to switch +that off — on a command line, so it checks for itself, and on a server, so it does +not open the socket that makes it reachable. + +Equivalent to the `--no-server` command-line argument. + ### `TY_CONFIG_FILE` Path to a `ty.toml` configuration file to use. diff --git a/crates/ty/src/args.rs b/crates/ty/src/args.rs index 78b11e5547..838a845903 100644 --- a/crates/ty/src/args.rs +++ b/crates/ty/src/args.rs @@ -442,6 +442,17 @@ pub(crate) struct CheckCommand { #[arg(long, short = 'W')] pub(crate) watch: bool, + /// Check the project from scratch instead of asking a running language server. + /// + /// A server holding this project has already parsed and inferred it, and answers a + /// check out of that in a fraction of the time. It only answers when the two agree + /// about the configuration and there is nothing unsaved, so this is a way to rule the + /// server out rather than a way to get a different answer. + /// + /// `BY_NO_PROJECT_SERVER` does the same, and also stops a server from listening. + #[arg(long, help_heading = "Global options")] + pub(crate) no_server: bool, + /// Respect file exclusions via `.gitignore` and other standard ignore files. /// Use `--no-respect-ignore-files` to disable. #[arg( diff --git a/crates/ty/src/by_project_server.rs b/crates/ty/src/by_project_server.rs new file mode 100644 index 0000000000..b152d0dcac --- /dev/null +++ b/crates/ty/src/by_project_server.rs @@ -0,0 +1,192 @@ +//! Checking a project by asking the language server that is already holding it. +//! +//! `by check` builds a project database, resolves an environment, and parses and infers +//! every file in the project — and then exits, and throws all of it away. An editor with the +//! language server running has done that work already and has been keeping it current ever +//! since. So before doing it again, ask. +//! +//! What comes back has to be what this process would have produced on its own, which is why +//! most of what is here is the deciding rather than the asking. The server refuses whenever +//! it might answer differently — a different build, a different configuration, an unsaved +//! buffer — and this side refuses for every invocation whose answer is not simply "the +//! diagnostics for this project": a fix, a watch, a subset of paths. +//! +//! The fallback is not a failure mode, it is the normal path. Nothing here is required to +//! work, and a caller that gets nothing goes on to check for itself. + +use std::io::Write; + +use ruff_db::Db as _; +use ruff_db::diagnostic::Severity; +use ruff_db::system::SystemPath; +use ty_project::{Db as _, ProjectDatabase}; +use ty_server::project_server; +use ty_server::project_server::client; +use ty_server::project_server::protocol; +use ty_server::project_server::protocol::{Build, CheckRequest, CheckResponse, SeverityLevel}; + +use crate::printer::Printer; +use crate::{ExitStatus, exit_status_from_summary, write_summary}; + +/// Why one `by check` is not a check a server may answer. +/// +/// A server answers one question — the diagnostics for a whole project, as configured — so +/// anything that asks a narrower or a different one is ruled out here. Every one of these +/// would otherwise be a silent change in what the command does. +#[derive(Debug, Clone, Copy)] +pub(crate) enum Ineligible { + /// The caller passed `--no-server`. + /// + /// `BY_NO_PROJECT_SERVER` rules it out too, but later and separately: it is read through + /// the same [`System`](ruff_db::system::System) the server reads it through, which this + /// side does not have until it has a database. + Disabled, + + /// The check re-runs on every change, so everything after the first run is warm already. + Watch, + + /// The check rewrites the files it reports on, which a server does not do. + Fixing, + + /// The check was pointed at part of the project rather than all of it. + Paths, + + /// The caller asked about a database that this path never builds. + MemoryReport, +} + +impl std::fmt::Display for Ineligible { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Ineligible::Disabled => "the project server is disabled", + Ineligible::Watch => "the check is in watch mode", + Ineligible::Fixing => "the check rewrites the files it reports on", + Ineligible::Paths => "the check was given paths rather than the whole project", + Ineligible::MemoryReport => "a memory report was requested", + }) + } +} + +/// Asks a running server to check the project `db` was built for, and prints what it says. +/// +/// `None` when there was no answer, for any reason at all. The caller then checks the project +/// itself, which is what it would have done had none of this existed. +/// +/// `db` rather than the metadata it came from because the comparison that decides whether the +/// server may answer is over what this process *resolved* — its search paths, its interpreter, +/// its merged configuration — and resolving that is the database's job. Building one is cheap +/// next to checking with it, which is the part this exists to skip. +pub(crate) fn check( + db: &ProjectDatabase, + working_directory: &SystemPath, + printer: Printer, + ineligible: Option, +) -> Option { + if let Some(ineligible) = ineligible { + tracing::debug!("Not asking a project server: {ineligible}"); + return None; + } + + if project_server::disabled(db.system()) { + tracing::debug!("Not asking a project server: disabled by the environment"); + return None; + } + + // a platform with nowhere for a server to have announced itself + let directory = project_server::discovery::default_directory(db.system())?; + + let project = db.project(); + let options = protocol::configuration(project.metadata(db).to_merged_options().options()) + .inspect_err(|error| tracing::debug!("Failed to serialize the resolved options: {error}")) + .ok()?; + + let response = client::check( + &directory, + CheckRequest { + project_root: project.root(db).to_path_buf(), + working_directory: working_directory.to_path_buf(), + options, + environment: project_server::environment(db), + force_exclude: project.force_exclude(db), + verbose: project.verbose(db), + color: colored::control::SHOULD_COLORIZE.should_colorize(), + }, + &Build::current(ruff_db::program_version()?), + )?; + + Some(report(&response, printer)) +} + +/// Prints an answer, and says what the command should exit with. +/// +/// Separate from asking for it because from here on there is no falling back: the answer has +/// reached the caller's terminal, and checking again would print it twice. A stream that +/// cannot be written to is the same failure it is on the cold path, and is reported the same +/// way rather than being turned into "no server answered". +fn report(response: &CheckResponse, printer: Printer) -> ExitStatus { + // said before the diagnostics, and on stderr: a reader piping `--output-format=json` + // somewhere is still owed the explanation of why the answer arrived so fast, and is not + // owed it in the middle of their json + if printer.shows_general_messages() { + writeln!(std::io::stderr(), "using project server information").ok(); + } + + let written = write_answer(response, printer); + + // the project has already been checked and the answer is already partly on its way out, + // so a broken stream changes what the caller sees but not what the check found. the cold + // path reports the same failure the same way and returns the same status + if let Err(error) = written { + tracing::warn!("Failed to write the diagnostics: {error}"); + } + + exit_status_from_summary( + response.max_severity.map(severity), + response.io_error, + response.error_on_warning, + ) +} + +fn write_answer(response: &CheckResponse, printer: Printer) -> std::io::Result<()> { + { + let stdout = printer.stream_for_details().lock(); + if stdout.is_enabled() { + let mut stdout = std::io::BufWriter::new(stdout); + write!(stdout, "{}", response.rendered)?; + stdout.flush()?; + } + } + + // the cold path warns about both of these, and a caller that cannot tell which path + // answered it is a caller for whom the two paths are the same command + if response.empty_project { + tracing::warn!("No python files found under the given path(s)"); + } + if response.fatal { + tracing::warn!( + "A fatal error occurred while checking some files. \ + Not all project files were analyzed. \ + See the diagnostics list above for details." + ); + } + + // never cancelled: a check the server could not finish comes back as a refusal, and a + // refusal never reaches this far + write_summary( + printer, + response.diagnostics, + response.human_readable, + None, + false, + ) + .map_err(std::io::Error::other) +} + +fn severity(level: SeverityLevel) -> Severity { + match level { + SeverityLevel::Info => Severity::Info, + SeverityLevel::Warning => Severity::Warning, + SeverityLevel::Error => Severity::Error, + SeverityLevel::Fatal => Severity::Fatal, + } +} diff --git a/crates/ty/src/lib.rs b/crates/ty/src/lib.rs index e717cc827a..9f1a763d88 100644 --- a/crates/ty/src/lib.rs +++ b/crates/ty/src/lib.rs @@ -2,6 +2,7 @@ mod args; mod by_commands; mod by_init; mod by_lowering; +mod by_project_server; mod by_source_encoding; mod by_stamps; mod by_wheels; @@ -362,6 +363,8 @@ fn run_check(args: CheckCommand) -> anyhow::Result { .map(|path| SystemPath::absolute(path, &cwd)) .collect(); + let check_paths_given = !check_paths.is_empty(); + let mode = if args.fix { MainLoopMode::Fix(FixMode::ApplyFixes) } else if args.add_ignore { @@ -394,8 +397,12 @@ fn run_check(args: CheckCommand) -> anyhow::Result { project_metadata.apply_configuration_files(&system)?; + let no_server = args.no_server; project_metadata.apply_override_options(args.into_options()); + // the answer this command exists to produce may already exist, in a server that has been + // holding this project open. what comes back is only ever the answer this process would + // have computed — see `by_project_server` for what that costs and what it refuses let mut db = ProjectDatabase::fallible(project_metadata, system)?; // the project's django, which the type checker does not read: its templates are @@ -413,6 +420,31 @@ fn run_check(args: CheckCommand) -> anyhow::Result { project.set_included_paths(&mut db, check_paths); } + // the answer this command exists to produce may already exist, in a server that has been + // holding this project open. asked here rather than earlier because what decides whether + // the server may answer is what *this* database resolved — see `by_project_server` + let ineligible = if no_server { + Some(by_project_server::Ineligible::Disabled) + } else if watch { + Some(by_project_server::Ineligible::Watch) + } else if !matches!(mode, MainLoopMode::Check) { + Some(by_project_server::Ineligible::Fixing) + } else if check_paths_given { + Some(by_project_server::Ineligible::Paths) + } else if memory_report.is_some() { + Some(by_project_server::Ineligible::MemoryReport) + } else { + None + }; + + if let Some(exit_status) = by_project_server::check(&db, &cwd, printer, ineligible) { + return Ok(if exit_zero { + ExitStatus::Success + } else { + exit_status + }); + } + // Disabling LRU only assumes that the database is short-lived; unlike freezing below, it does // not require immutable inputs. if !watch { @@ -851,60 +883,89 @@ impl MainLoop { let terminal_settings = db.project().settings(db).terminal(); let is_human_readable = terminal_settings.output_format.is_human_readable(); - match diagnostics { - [] if is_human_readable && fixed_diagnostics.is_none_or(|fixed| fixed == 0) => { - writeln!( - self.printer.stream_for_success_summary(), + { + let stdout = self.printer.stream_for_details().lock(); + + // Only render diagnostics if they're going to be displayed, since doing + // so is expensive. + if stdout.is_enabled() { + let mut stdout = BufWriter::new(stdout); + let display_config = DisplayDiagnosticConfig::new("ty") + .format(terminal_settings.output_format.into()) + .color(colored::control::SHOULD_COLORIZE.should_colorize()) + .with_cancellation_token(Some(self.cancellation_token.clone())) + .context(0); + + write!( + stdout, "{}", - "All checks passed!".green().bold() + DisplayDiagnostics::new(db, &display_config, diagnostics) )?; + stdout.flush()?; } - diagnostics => { - let diagnostics_count = diagnostics.len(); - - let stdout = self.printer.stream_for_details().lock(); - - // Only render diagnostics if they're going to be displayed, since doing - // so is expensive. - if stdout.is_enabled() { - let mut stdout = BufWriter::new(stdout); - let display_config = DisplayDiagnosticConfig::new("ty") - .format(terminal_settings.output_format.into()) - .color(colored::control::SHOULD_COLORIZE.should_colorize()) - .with_cancellation_token(Some(self.cancellation_token.clone())) - .context(0); - - write!( - stdout, - "{}", - DisplayDiagnostics::new(db, &display_config, diagnostics) - )?; - stdout.flush()?; - } + } - if !self.cancellation_token.is_cancelled() && is_human_readable { - if let Some(fixed) = fixed_diagnostics { - let total = fixed + diagnostics_count; - writeln!( - self.printer.stream_for_failure_summary(), - "Found {total} diagnostic{} \ - ({fixed} fixed, {diagnostics_count} remaining).", - if total == 1 { "" } else { "s" } - )?; - } else { - writeln!( - self.printer.stream_for_failure_summary(), - "Found {} diagnostic{}", - diagnostics_count, - if diagnostics_count > 1 { "s" } else { "" } - )?; - } - } - } + write_summary( + self.printer, + diagnostics.len(), + is_human_readable, + fixed_diagnostics, + self.cancellation_token.is_cancelled(), + ) + } +} + +/// Writes the line that says how a check came out, after its diagnostics. +/// +/// Shared with the [project server](crate::by_project_server) path, which renders its +/// diagnostics elsewhere but has to summarize them the same way — the wording of "how did +/// that go" belongs to the command, not to whoever computed the answer. +/// +/// `cancelled` suppresses the count but not the sentence for a project with nothing wrong +/// with it: a run interrupted before it finished counting cannot say how many there were, but +/// a fix run that removed everything and was then interrupted still removed everything. +pub(crate) fn write_summary( + printer: Printer, + diagnostics: usize, + is_human_readable: bool, + fixed_diagnostics: Option, + cancelled: bool, +) -> anyhow::Result<()> { + // nothing left and nothing fixed, which is the only outcome with its own sentence + if diagnostics == 0 && fixed_diagnostics.is_none_or(|fixed| fixed == 0) { + if is_human_readable { + writeln!( + printer.stream_for_success_summary(), + "{}", + "All checks passed!".green().bold() + )?; } + return Ok(()); + } - Ok(()) + // a check that was interrupted did not finish counting, so it has no total to report + if cancelled || !is_human_readable { + return Ok(()); } + + if let Some(fixed) = fixed_diagnostics { + let total = fixed + diagnostics; + writeln!( + printer.stream_for_failure_summary(), + "Found {total} diagnostic{} \ + ({fixed} fixed, {diagnostics} remaining).", + if total == 1 { "" } else { "s" } + )?; + } else { + writeln!( + printer.stream_for_failure_summary(), + "Found {} diagnostic{}", + diagnostics, + if diagnostics > 1 { "s" } else { "" } + )?; + } + + Ok(()) } #[derive(Copy, Clone, Debug)] @@ -949,18 +1010,32 @@ fn exit_status_from_diagnostics( diagnostics: &[Diagnostic], terminal_settings: &TerminalSettings, ) -> ExitStatus { - if diagnostics.is_empty() { - return ExitStatus::Success; - } - - let mut max_severity = Severity::Info; + let mut max_severity = None; let mut io_error = false; for diagnostic in diagnostics { - max_severity = max_severity.max(diagnostic.severity()); + max_severity = max_severity.max(Some(diagnostic.severity())); io_error = io_error || matches!(diagnostic.id(), DiagnosticId::Io); } + exit_status_from_summary(max_severity, io_error, terminal_settings.error_on_warning) +} + +/// The status a check exits with, from what it found. +/// +/// Separate from the diagnostics themselves so that the [project +/// server](crate::by_project_server) path decides it here too: that path never holds the +/// diagnostics — they are rendered where they were computed — but the rules for turning them +/// into an exit code belong to the command either way. +pub(crate) fn exit_status_from_summary( + max_severity: Option, + io_error: bool, + error_on_warning: bool, +) -> ExitStatus { + let Some(max_severity) = max_severity else { + return ExitStatus::Success; + }; + if !max_severity.is_fatal() && io_error { return ExitStatus::Error; } @@ -968,7 +1043,7 @@ fn exit_status_from_diagnostics( match max_severity { Severity::Info => ExitStatus::Success, Severity::Warning => { - if terminal_settings.error_on_warning { + if error_on_warning { ExitStatus::Failure } else { ExitStatus::Success diff --git a/crates/ty/src/printer.rs b/crates/ty/src/printer.rs index d72e1ebb5a..842a9492e6 100644 --- a/crates/ty/src/printer.rs +++ b/crates/ty/src/printer.rs @@ -102,6 +102,11 @@ impl Printer { self.stdout_general() } + /// Whether messages that are neither diagnostics nor a summary should be shown. + pub(crate) fn shows_general_messages(self) -> bool { + self.stdout_general().is_enabled() + } + pub(crate) fn clear_screen() -> Result<()> { clearscreen::clear()?; Ok(()) diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 7e5a2ed99c..0fe72c3909 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -2,6 +2,7 @@ use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; +use ty_static::EnvVars; /// a temp directory of this process's own /// @@ -488,6 +489,7 @@ fn compile_emits_only_the_files_it_was_given_and_still_resolves_the_others() { let out = dir.join("build"); let result = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["compile", "wanted.py", "-o"]) .arg(&out) .arg("--emit-c-only") @@ -554,6 +556,7 @@ fn compile_writes_each_package_member_at_its_own_place_in_the_output_tree() { let out = dir.join("o"); let result = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["compile", "-o"]) .arg(&out) .arg("--emit-c-only") @@ -613,6 +616,7 @@ fn compile_refuses_two_sources_that_would_write_the_same_artifact() { let out = dir.join("o"); let result = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["compile", "-o"]) .arg(&out) .arg("--emit-c-only") @@ -654,6 +658,7 @@ fn compile_declines_a_package_body_whose_package_has_no_importable_name() { let out = dir.join("o"); let result = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["compile", "-o"]) .arg(&out) .arg("--emit-c-only") @@ -694,6 +699,7 @@ async def total(s: str, n: int) -> int: let emitted = |spec: &str| -> Option { let out = dir.join(spec); let status = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["compile"]) .arg(&file) .arg("-o") @@ -732,6 +738,7 @@ fn run_executes_module() { fs::write(dir.path().join("main.by"), "print('hello from by run')\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -767,6 +774,7 @@ fn run_names_the_by_source_for_file_and_argv() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -817,6 +825,7 @@ def main(): .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -845,6 +854,7 @@ fn run_enters_a_package_through_its_main_module() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "app"]) .current_dir(dir.path()) .output() @@ -874,6 +884,7 @@ fn run_still_rewrites_traceback_frames_to_by_lines() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -919,6 +930,7 @@ main() .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -964,6 +976,7 @@ fn run_checks_a_deeply_nested_expression() { fs::write(dir.path().join("main.by"), format!("print({terms})\n")).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -985,6 +998,7 @@ fn run_force_unwrap_yields_inner_value() { fs::write(dir.path().join("main.by"), "x = Some(5)\nprint(x! + 1)\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1010,6 +1024,7 @@ fn run_invokes_top_level_main() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1034,6 +1049,7 @@ fn run_invokes_async_main_via_asyncio() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1058,6 +1074,7 @@ fn run_uses_the_configured_entry_point() { fs::write(dir.path().join("app.by"), "print('ran the entry point')\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("run") .current_dir(dir.path()) .output() @@ -1087,6 +1104,7 @@ fn run_reads_the_entry_point_from_pyproject() { fs::write(dir.path().join("pkg/cli.by"), "print('ran pkg.cli')\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("run") .current_dir(dir.path()) .output() @@ -1114,6 +1132,7 @@ fn run_reads_the_entry_point_from_basedpython_toml() { fs::write(dir.path().join("app.by"), "print('ran the entry point')\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("run") .current_dir(dir.path()) .output() @@ -1143,6 +1162,7 @@ fn run_reads_the_entry_point_from_the_basedpython_section() { fs::write(dir.path().join("pkg/cli.by"), "print('ran pkg.cli')\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("run") .current_dir(dir.path()) .output() @@ -1167,6 +1187,7 @@ fn run_prefers_an_explicit_module_over_the_configured_entry_point() { fs::write(dir.path().join("other.by"), "print('explicit')\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "other"]) .current_dir(dir.path()) .output() @@ -1193,6 +1214,7 @@ fn run_forwards_arguments_to_the_named_entry_point() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "app", "--name", "asdf"]) .current_dir(dir.path()) .output() @@ -1212,6 +1234,7 @@ fn run_without_a_module_or_entry_point_reports_both_ways_out() { fs::write(dir.path().join("main.by"), "print('unreached')\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("run") .current_dir(dir.path()) .output() @@ -1233,6 +1256,7 @@ fn run_main_with_args(source: &str, args: &[&str]) -> (String, String, i32) { fs::write(dir.path().join("main.by"), source).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .args(args) .current_dir(dir.path()) @@ -1258,6 +1282,7 @@ fn run_main_stamped(source: &str, stamps: &[&str]) -> (String, String, i32) { fs::write(dir.path().join("basedpython.toml"), OPT_IN_TO_STAMPS).unwrap(); let mut command = Command::new(env!("CARGO_BIN_EXE_by")); + command.env(EnvVars::BY_NO_PROJECT_SERVER, "1"); command.arg("run"); for stamp in stamps { command.args(["--stamp", stamp]); @@ -1419,6 +1444,7 @@ def main(): fs::write(dir.path().join("basedpython.toml"), OPT_IN_TO_STAMPS).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "--compiled", "--python"]) .arg(&python) .args([ @@ -1595,6 +1621,7 @@ Grid()[(1, 2)] .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1624,6 +1651,7 @@ print(A.__sealed_members__ == (B, C)) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1657,6 +1685,7 @@ print(shout(\"moon\", greeting=\"good night\")) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1698,6 +1727,7 @@ print(\"quiet\".shouty) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1759,6 +1789,7 @@ print(b not in w) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1805,6 +1836,7 @@ print(Widget().kind) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1841,6 +1873,7 @@ extension str: .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1875,6 +1908,7 @@ print(Holder.value) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1912,6 +1946,7 @@ print(xs.second()) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1995,6 +2030,7 @@ print(Color.Green.name) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -2033,6 +2069,7 @@ main() .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -2107,6 +2144,7 @@ print('verified', checked) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -2151,6 +2189,7 @@ boom() .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -2192,6 +2231,7 @@ fn build_skips_a_source_it_cannot_read() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--min-version", "3.12"]) .current_dir(dir.path()) .output() @@ -2297,6 +2337,7 @@ fn transpile_renders_parse_error_with_location() { fs::write(&by_path, "a b\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("transpile") .arg(&by_path) .output() @@ -2353,6 +2394,7 @@ fn transpile_malformed_inputs_never_panic() { ]; for src in inputs { let mut child = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("transpile") .stdin(Stdio::piped()) .stdout(Stdio::piped()) @@ -2387,6 +2429,7 @@ fn run_renders_parse_error_and_aborts() { fs::write(dir.path().join("main.by"), "a b\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -2414,6 +2457,7 @@ fn build_renders_parse_error_and_aborts() { fs::write(dir.path().join("bad.by"), "a b\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -2445,6 +2489,7 @@ fn run_refuses_to_execute_on_check_errors() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -2485,6 +2530,7 @@ fn run_min_version_newer_than_interpreter_errors() { fs::write(dir.path().join("main.by"), "print(1)\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "--min-version", "3.99", "main"]) .env("PYTHON", python) .current_dir(dir.path()) @@ -2520,6 +2566,7 @@ fn run_honors_explicit_min_version() { fs::write(dir.path().join("main.by"), "print('versioned')\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "--min-version", "3.9", "main"]) .env("PYTHON", python) .current_dir(dir.path()) @@ -2545,6 +2592,7 @@ fn build_skips_hidden_directories() { fs::write(hidden.join("junk.by"), "a b\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -2584,6 +2632,7 @@ fn build_writes_what_the_project_exports_into_its_marker() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -2607,6 +2656,7 @@ fn build_writes_a_marker_for_a_project_that_exports_nothing() { fs::write(package.join("__init__.by"), "").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -2638,6 +2688,7 @@ fn build_mirrors_the_module_tree_not_the_directory_tree() { fs::write(package.join("main.by"), "print(\"src layout\")\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -2665,6 +2716,7 @@ fn build_writes_a_sourcemap_beside_the_generated_python() { fs::write(dir.path().join("main.by"), "print(\"built\")\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -2720,6 +2772,7 @@ fn sourcemap_table_for(source: &str) -> (Vec>, String) { fs::write(dir.path().join("main.by"), source).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -2827,6 +2880,7 @@ fn run_resolves_a_src_layout_entry_point() { fs::write(package.join("main.by"), "print(\"src layout\")\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("run") .current_dir(dir.path()) .output() @@ -2855,6 +2909,7 @@ fn build_targets_the_configured_python_version() { fs::write(dir.path().join("main.by"), "type X = int\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -2880,6 +2935,7 @@ fn build_emits_every_file_it_can_past_a_broken_one() { fs::write(dir.path().join("broken.by"), "x = (\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -2914,6 +2970,7 @@ fn build_honours_src_exclude() { fs::write(negative.join("bad.by"), "def f(:\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -2938,6 +2995,7 @@ fn transpile_proceeds_past_non_syntax_errors() { fs::write(&by_path, "x: int = \"string\"\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("transpile") .arg(&by_path) .output() @@ -2977,6 +3035,7 @@ fn transpile_directory_reverses_in_place() { fs::write(root.join(".venv/dep.py"), "x = 1\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("transpile") .arg("--reverse") .arg(root) @@ -3014,6 +3073,7 @@ fn transpile_directory_round_trips_through_build() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(root) .output() @@ -3138,6 +3198,7 @@ a.f() .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .env("PYTHON", &python) .current_dir(dir.path()) @@ -3209,6 +3270,7 @@ fn declared_reified_generic_runs() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .env("PYTHON", python) .current_dir(dir.path()) @@ -3250,6 +3312,7 @@ fn reified_generic_infers_specialization_from_arguments() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .env("PYTHON", python) .current_dir(dir.path()) @@ -3312,6 +3375,7 @@ Box().kind[float]() .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .env("PYTHON", python) .current_dir(dir.path()) @@ -3399,6 +3463,7 @@ print(sorted(s)) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .env("PYTHON", python) .current_dir(dir.path()) @@ -3477,6 +3542,7 @@ x(A(True)) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -3550,6 +3616,7 @@ print(h(A(\"x\"))) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .env("PYTHON", python) .current_dir(dir.path()) @@ -3617,6 +3684,7 @@ con(Con[object]()) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .env("PYTHON", python) .current_dir(dir.path()) @@ -3695,6 +3763,7 @@ print(object() is Sequence[int]) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -3741,6 +3810,7 @@ def main(): .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main", "--runtime-raises-checks"]) .current_dir(dir.path()) .output() @@ -3790,6 +3860,7 @@ def main(): .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main", "--runtime-raises-checks"]) .current_dir(dir.path()) .output() @@ -3835,6 +3906,7 @@ def main(): let check = || { Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("check") .current_dir(dir.path()) .output() @@ -3908,6 +3980,7 @@ fn a_lazy_from_import_resolves_a_submodule_and_refuses_a_missing_name_as_python_ .unwrap(); let transpiled = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["transpile", "main.by"]) .env("PYTHON", python) .current_dir(dir.path()) @@ -3974,6 +4047,7 @@ fn build_carries_a_python_module_into_the_output() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -4001,6 +4075,7 @@ fn build_carries_data_files_into_the_output() { fs::write(package.join("template.html"), "

    hi

    \n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -4027,6 +4102,7 @@ fn build_writes_a_stub_as_a_stub() { fs::write(dir.path().join("shapes.byi"), "def area() -> int: ...\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -4054,6 +4130,7 @@ fn build_refuses_two_sources_that_are_one_module() { fs::write(dir.path().join("thing.py"), "x = 2\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -4081,6 +4158,7 @@ fn build_deletes_output_the_project_no_longer_has() { let build = || { let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -4115,6 +4193,7 @@ fn build_leaves_output_it_never_wrote_alone() { fs::write(dir.path().join("build/theirs.txt"), "hands off\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -4134,6 +4213,7 @@ fn build_writes_where_out_says() { fs::write(dir.path().join("main.by"), "x = 1\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--out", "elsewhere"]) .current_dir(dir.path()) .output() @@ -4153,6 +4233,7 @@ fn build_does_not_read_its_own_output() { for _ in 0..2 { let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--out", "elsewhere"]) .current_dir(dir.path()) .output() @@ -4182,6 +4263,7 @@ fn build_reports_what_it_read_and_what_it_produced() { fs::write(package.join("helper.py"), "x = 1\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--print-manifest"]) .current_dir(dir.path()) .output() @@ -4238,6 +4320,7 @@ fn build_does_not_ship_what_lives_outside_the_source_root() { fs::write(tests.join("test_it.py"), "def test_x(): pass\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--print-manifest"]) .current_dir(dir.path()) .output() @@ -4275,6 +4358,7 @@ fn build_marks_a_package_as_carrying_its_sources() { fs::write(package.join("deep.by"), "x = 1\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -4308,6 +4392,7 @@ fn build_ships_python_only_when_the_project_says_so() { fs::write(package.join("__init__.by"), "").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -4345,6 +4430,7 @@ fn build_honours_the_configured_exclusions() { fs::write(dir.path().join("public.json"), "{}\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -4378,6 +4464,7 @@ fn build_carries_a_directory_a_negated_exclude_takes_back() { fs::write(generated.join("kept.json"), "{}\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -4413,6 +4500,7 @@ fn build_ships_a_source_directory_that_is_itself_a_package() { fs::write(package.join("__init__.by"), "").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--print-manifest"]) .current_dir(dir.path()) .output() @@ -4454,6 +4542,7 @@ fn build_reports_what_lowering_needs_at_run_time() { let manifest = |extra: &[&str]| -> String { let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--print-manifest"]) .args(extra) .current_dir(dir.path()) @@ -4498,6 +4587,7 @@ fn building_wheels_without_a_frontend_says_what_is_missing() { fs::write(package.join("__init__.by"), "").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--wheels"]) .current_dir(dir.path()) // an empty `PATH` is the only way to be sure this machine's `uv` is not @@ -4552,6 +4642,7 @@ fn build_emitting(source: &str, settled: Option<&str>) -> String { fs::write(dir.path().join("main.by"), source).unwrap(); let mut command = Command::new(env!("CARGO_BIN_EXE_by")); + command.env(EnvVars::BY_NO_PROJECT_SERVER, "1"); command .args(["build", "--out", "build"]) .current_dir(dir.path()); @@ -4644,6 +4735,7 @@ fn only_a_build_takes_its_lowering_from_the_environment() { fs::write(dir.path().join("main.by"), LOWERED_THREE_WAYS).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["transpile", "main.by"]) .current_dir(dir.path()) .env( @@ -4671,6 +4763,7 @@ fn building_wheels_refuses_a_soundness_spec_it_cannot_parse() { fs::write(dir.path().join("main.by"), "x = 1\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--wheels", "--soundness", "nonsense"]) .current_dir(dir.path()) // refused before the frontend is even looked for, so this holds on a @@ -4722,6 +4815,7 @@ fn a_release_hands_each_build_the_stamps_and_the_lowering() { fs::set_permissions(&uv, fs::Permissions::from_mode(0o755)).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--wheels", "--soundness", "none"]) .current_dir(dir.path()) // the stub is the only `uv` reachable, so this cannot accidentally @@ -4758,6 +4852,7 @@ fn building_wheels_refuses_a_single_target_version() { fs::write(dir.path().join("main.by"), "x = 1\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--wheels", "--min-version", "3.12"]) .current_dir(dir.path()) .output() @@ -4791,6 +4886,7 @@ fn run_imports_a_python_module_beside_the_transpiled_ones() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -4816,6 +4912,7 @@ fn run_reads_a_data_file_beside_the_program() { fs::write(dir.path().join("greeting.txt"), "read from disk\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -4858,6 +4955,7 @@ fn run_refuses_an_interpreter_older_than_the_project_targets() { fs::write(dir.path().join("main.by"), "print(1)\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .env_remove("PYTHON") @@ -5006,6 +5104,7 @@ fn run_uses_the_environment_the_project_configures() { fs::write(dir.path().join("main.by"), REPORTS_ITS_INTERPRETER).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .env_remove("PYTHON") @@ -5045,6 +5144,7 @@ fn run_from_a_subdirectory_is_still_the_project() { fs::create_dir_all(&elsewhere).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("run") .current_dir(&elsewhere) .env_remove("PYTHON") @@ -5075,6 +5175,7 @@ fn build_from_a_subdirectory_builds_the_project() { fs::create_dir_all(&elsewhere).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(&elsewhere) .output() @@ -5114,6 +5215,7 @@ fn run_prefers_the_project_environment_to_the_python_variable() { fs::write(dir.path().join("main.by"), REPORTS_ITS_INTERPRETER).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .env("PYTHON", named.interpreter()) @@ -5151,6 +5253,7 @@ fn run_falls_back_to_the_python_variable() { fs::write(project.join("main.by"), REPORTS_ITS_INTERPRETER).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(&project) .env("PYTHON", elsewhere.interpreter()) @@ -5182,6 +5285,7 @@ fn run_refuses_a_configured_environment_that_is_not_one() { fs::write(dir.path().join("main.by"), "print(1)\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -5205,6 +5309,7 @@ fn run_refuses_a_project_file_that_collides_with_its_shim() { fs::write(dir.path().join("_by_runner.py"), "x = 1\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -5228,6 +5333,7 @@ fn build_does_not_carry_a_compilers_output_directory() { fs::write(artifacts.join("blob"), "an enormous binary").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -5252,6 +5358,7 @@ fn init_writes_a_project_that_builds_and_runs() { let (major, minor) = running_python_version(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args([ "init", "demo", @@ -5270,6 +5377,7 @@ fn init_writes_a_project_that_builds_and_runs() { assert!(project.join("src/demo/__init__.by").exists()); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("run") .current_dir(&project) .output() @@ -5295,6 +5403,7 @@ fn init_refuses_to_write_over_a_project() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("init") .current_dir(dir.path()) .output() diff --git a/crates/ty/tests/cli/main.rs b/crates/ty/tests/cli/main.rs index d073a599a9..8cb08a2f1d 100644 --- a/crates/ty/tests/cli/main.rs +++ b/crates/ty/tests/cli/main.rs @@ -5,6 +5,7 @@ mod django; mod exit_code; mod file_selection; mod fixes; +mod project_server; mod python_environment; mod rule; mod rule_selection; @@ -23,6 +24,7 @@ use std::{ process::Command, }; use tempfile::TempDir; +use ty_static::EnvVars; #[test] fn test_quiet_output() -> anyhow::Result<()> { @@ -1097,6 +1099,9 @@ impl CliTest { user_config_directory_env_var(), self.user_config_directory(), ); + // These tests assert on exact output, and a language server running on this machine + // for a directory above the test's own would otherwise be entitled to answer them. + command.env(EnvVars::BY_NO_PROJECT_SERVER, "1"); command } @@ -1111,6 +1116,7 @@ impl CliTest { user_config_directory_env_var(), self.user_config_directory(), ); + command.env(EnvVars::BY_NO_PROJECT_SERVER, "1"); command } diff --git a/crates/ty/tests/cli/project_server.rs b/crates/ty/tests/cli/project_server.rs new file mode 100644 index 0000000000..e5c0128476 --- /dev/null +++ b/crates/ty/tests/cli/project_server.rs @@ -0,0 +1,65 @@ +//! `by check`'s side of the [project server](../../../src/by_project_server.rs). +//! +//! What a server would say, and whether it is asked at all, is covered against a real server +//! in `ty_server`'s end-to-end tests. What is left here is the command line's own half: that +//! the switch exists, that it does not change the answer, and that these tests are not +//! quietly being answered by whatever server the person running them has open. + +use anyhow::Result; +use insta_cmd::assert_cmd_snapshot; + +use crate::CliTest; + +/// The answer is the answer either way. `--no-server` chooses how it is arrived at, and a +/// flag that changed what came out would be a different command rather than a faster one. +#[test] +fn no_server_does_not_change_the_answer() -> Result<()> { + let case = CliTest::with_file( + "main.py", + r#" +def f() -> str: + return 42 +"#, + )?; + + let with_server = case.command().output()?; + assert_cmd_snapshot!(case.command().arg("--no-server"), @r" + success: false + exit_code: 1 + ----- stdout ----- + error[invalid-return-type]: Return type does not match returned value + --> main.py:3:12 + | + 2 | def f() -> str: + | --- Expected `str` because of return type + 3 | return 42 + | ^^ expected `str`, found `Literal[42]` + + Found 1 diagnostic + + ----- stderr ----- + "); + + assert_eq!( + String::from_utf8(with_server.stdout)?, + String::from_utf8(case.command().arg("--no-server").output()?.stdout)? + ); + + Ok(()) +} + +/// These tests assert on exact output, so none of them may be answered out of a language +/// server that happens to be running on this machine. `CliTest` sets the kill switch for +/// every command it builds; this is the assertion that it still does. +#[test] +fn the_tests_never_ask_a_server() -> Result<()> { + let case = CliTest::with_file("main.py", "x: int = 1\n")?; + + let output = case.command().output()?; + assert!( + !String::from_utf8(output.stderr)?.contains("using project server information"), + "a cli test was answered by a project server" + ); + + Ok(()) +} diff --git a/crates/ty_project/src/db.rs b/crates/ty_project/src/db.rs index 9f1d0d782e..ed556a03ad 100644 --- a/crates/ty_project/src/db.rs +++ b/crates/ty_project/src/db.rs @@ -267,6 +267,11 @@ impl ProjectDatabase { self.checker = Some(checker); } + /// The set of files this database checks. + pub fn check_mode(&self) -> CheckMode { + self.project().check_mode(self) + } + /// Set the check mode for the project. pub fn set_check_mode(&mut self, mode: CheckMode) { if self.project().check_mode(self) != mode { diff --git a/crates/ty_project/src/lib.rs b/crates/ty_project/src/lib.rs index e8c1992002..065013861c 100644 --- a/crates/ty_project/src/lib.rs +++ b/crates/ty_project/src/lib.rs @@ -675,7 +675,7 @@ impl Project { } } - fn verbose(self, db: &dyn Db) -> bool { + pub fn verbose(self, db: &dyn Db) -> bool { self.verbose_flag(db) } @@ -685,7 +685,7 @@ impl Project { } } - fn force_exclude(self, db: &dyn Db) -> bool { + pub fn force_exclude(self, db: &dyn Db) -> bool { self.force_exclude_flag(db) } diff --git a/crates/ty_server/Cargo.toml b/crates/ty_server/Cargo.toml index bbd5988940..38791450c7 100644 --- a/crates/ty_server/Cargo.toml +++ b/crates/ty_server/Cargo.toml @@ -32,6 +32,7 @@ ty_ide = { workspace = true } ty_module_resolver = { workspace = true } ty_project = { workspace = true } ty_python_semantic = { workspace = true } +ty_static = { workspace = true } anyhow = { workspace = true } bitflags = { workspace = true } @@ -39,6 +40,7 @@ crossbeam = { workspace = true } jod-thread = { workspace = true } lsp-server = { workspace = true } lsp-types = { workspace = true } +rand = { workspace = true, features = ["sys_rng"] } rustc-hash = { workspace = true } salsa = { workspace = true } serde = { workspace = true } diff --git a/crates/ty_server/src/document.rs b/crates/ty_server/src/document.rs index 1daccb3786..37583d26d2 100644 --- a/crates/ty_server/src/document.rs +++ b/crates/ty_server/src/document.rs @@ -86,6 +86,14 @@ impl DocumentKey { } } + /// The path this document has on disk, or `None` for one that has none. + pub(crate) const fn file_path(&self) -> Option<&SystemPathBuf> { + match self { + Self::File(path) => Some(path), + Self::Opaque(_) => None, + } + } + /// Returns the corresponding [`AnySystemPath`] for this document key. /// /// Note, calling this method on a `DocumentKey::Opaque` representing a cell document diff --git a/crates/ty_server/src/lib.rs b/crates/ty_server/src/lib.rs index da4f138448..25ae32cfc1 100644 --- a/crates/ty_server/src/lib.rs +++ b/crates/ty_server/src/lib.rs @@ -15,6 +15,7 @@ mod capabilities; mod db; mod document; mod logging; +pub mod project_server; mod server; mod session; mod system; @@ -51,9 +52,24 @@ pub fn run_server() -> anyhow::Result<()> { // This is to complement the `LSPSystem` if the document is not available in the index. let fallback_system = Arc::new(OsSystem::new(cwd)); - let server_result = Server::new(worker_threads, connection, fallback_system, false) - .context("Failed to start server")? - .run(); + // a server publishes itself for `by` command lines to find, unless the user has said + // not to. see `project_server` + let project_server_directory = if project_server::disabled(&*fallback_system) { + tracing::debug!("Not publishing this server: disabled by the environment"); + None + } else { + project_server::discovery::default_directory(&*fallback_system) + }; + + let server_result = Server::new( + worker_threads, + connection, + fallback_system, + false, + project_server_directory, + ) + .context("Failed to start server")? + .run(); let io_result = io_threads.join(); diff --git a/crates/ty_server/src/project_server/check.rs b/crates/ty_server/src/project_server/check.rs new file mode 100644 index 0000000000..207ce80c36 --- /dev/null +++ b/crates/ty_server/src/project_server/check.rs @@ -0,0 +1,320 @@ +//! Answering a command line's check out of the session the editor is already keeping warm. +//! +//! The whole of the saving is that this runs on a database somebody else built and has been +//! feeding changes into. What is left is the part a cold run cannot skip either: deciding +//! whether this database is allowed to answer for this caller, and rendering. + +use std::path::{Path, PathBuf}; + +use ruff_db::Db as _; +use ruff_db::diagnostic::{ + DiagnosticId, DisplayDiagnosticConfig, DisplayDiagnostics, FileResolver, Input, Severity, + UnifiedFile, +}; +use ruff_db::files::File; +use ruff_db::system::SystemPath; +use ruff_notebook::NotebookIndex; +use ty_project::{CheckMode, CollectReporter, Db as _, ProjectDatabase}; + +use super::Outcome; +use super::protocol; +use super::protocol::{CheckRequest, CheckResponse, Refusal, Response, SeverityLevel}; +use crate::session::{OpenDocument, SessionSnapshot}; + +/// Runs `request` against `snapshot`, or explains why it did not. +/// +/// A [`Refusal`] is a normal outcome, not an error. The caller falls back to checking for +/// itself, which is slower and always correct, so every uncertainty here resolves to one. +pub(super) fn run(snapshot: &SessionSnapshot, request: &CheckRequest, rescanned: bool) -> Outcome { + let db = match gate(snapshot, request) { + Ok(db) => db, + Err(refusal) => return Outcome::Answered(Response::Refused { reason: refusal }), + }; + + // everything that could have refused has agreed, so this request is going to be answered + // — and now it is worth going to the file system, which is the one thing that decides + // whether the answer is about the project as it is now + if !rescanned { + return Outcome::NeedsRescan; + } + + Outcome::Answered(answer(db, request)) +} + +/// Everything that has to be true before this session may answer for `request`. +/// +/// Separate from [`answer`] because it is the cheap half, and because the main loop runs it +/// first on its own: re-reading the file system is expensive and interrupts the editor, and +/// there is no reason to do either for a request that was never going to be answered. +fn gate<'a>( + snapshot: &'a SessionSnapshot, + request: &CheckRequest, +) -> Result<&'a ProjectDatabase, Refusal> { + let Some(db) = project_database(snapshot, request) else { + return Err(Refusal::UnknownProject); + }; + + // a server that has been told to diagnose only what is open checks only what is open, + // and would answer a whole project's check with a handful of files' diagnostics. the + // editor's `diagnostic-mode` decides this, and nothing here may quietly override it: it + // is a salsa input on a project the editor is also using + if db.check_mode() != CheckMode::AllFiles { + return Err(Refusal::CheckMode); + } + + if let Some(refusal) = disagreement(db, request) { + return Err(refusal); + } + + // this database reads an open file out of the editor's buffer, so for as long as a + // buffer differs from the file, the two sides are checking different programs + let unsaved = unsaved_documents(db, snapshot); + if !unsaved.is_empty() { + return Err(Refusal::Unsaved { files: unsaved }); + } + + Ok(db) +} + +/// Checks the project and renders what it found. +fn answer(db: &ProjectDatabase, request: &CheckRequest) -> Response { + let diagnostics = match salsa::Cancelled::catch(|| { + let mut reporter = CollectReporter::default(); + db.check_with_reporter(&mut reporter); + reporter.into_sorted(db) + }) { + Ok(diagnostics) => diagnostics, + Err(cancelled) => { + tracing::debug!("A command-line check was cancelled: {cancelled:?}"); + return Response::Refused { + reason: Refusal::Cancelled, + }; + } + }; + + let terminal = db.project().settings(db).terminal(); + let config = DisplayDiagnosticConfig::new("ty") + .format(terminal.output_format.into()) + .color(request.color) + .context(0); + let resolver = RelativeTo { + db, + working_directory: working_directory(db, request), + }; + + let mut max_severity = None; + let mut io_error = false; + for diagnostic in &diagnostics { + max_severity = max_severity.max(Some(diagnostic.severity())); + io_error = io_error || matches!(diagnostic.id(), DiagnosticId::Io); + } + + Response::Check(CheckResponse { + rendered: DisplayDiagnostics::new(&resolver, &config, &diagnostics).to_string(), + diagnostics: diagnostics.len(), + human_readable: terminal.output_format.is_human_readable(), + max_severity: max_severity.map(SeverityLevel::from), + io_error, + error_on_warning: terminal.error_on_warning, + empty_project: db.project().files(db).is_empty(), + fatal: max_severity == Some(Severity::Fatal), + }) +} + +/// Where the caller is standing, spelled the way this database spells it. +/// +/// Both sides reach the same tree, and neither necessarily by the same name — see +/// [`super::canonical`]. Paths are relativized by stripping a prefix, so the caller's own +/// spelling of its directory would strip nothing off the files this database holds. What +/// makes it strip is putting the caller's position *inside* the project onto the root this +/// database uses. +/// +/// A caller standing outside the project — `by check --project ../elsewhere` — has no +/// position inside it, and gets its own directory back. Nothing is stripped then, which +/// leaves absolute paths, which is what a cold check prints from there too. +fn working_directory(db: &ProjectDatabase, request: &CheckRequest) -> PathBuf { + let root = db.project().root(db); + let relative = super::canonical(&request.working_directory) + .strip_prefix(super::canonical(root)) + .map(SystemPath::to_path_buf); + + match relative { + Ok(relative) => root.join(relative).as_std_path().to_path_buf(), + Err(_) => request.working_directory.as_std_path().to_path_buf(), + } +} + +/// The database, rendering paths from where the caller is standing rather than from where the +/// server is. +/// +/// Everything a rendering needs comes from the database, except the one thing that does not +/// belong to it: which directory the reader will read the answer in. A server's own is +/// wherever its editor was started, which is nowhere in particular. +struct RelativeTo<'db> { + db: &'db ProjectDatabase, + working_directory: PathBuf, +} + +impl FileResolver for RelativeTo<'_> { + fn path(&self, file: File) -> &str { + self.db.path(file) + } + + fn input(&self, file: File) -> Input { + self.db.input(file) + } + + fn notebook_index(&self, file: &UnifiedFile) -> Option { + self.db.notebook_index(file) + } + + fn is_notebook(&self, file: &UnifiedFile) -> bool { + self.db.is_notebook(file) + } + + fn current_directory(&self) -> &Path { + &self.working_directory + } +} + +impl From for SeverityLevel { + fn from(severity: Severity) -> Self { + match severity { + Severity::Info => SeverityLevel::Info, + Severity::Warning => SeverityLevel::Warning, + Severity::Error => SeverityLevel::Error, + Severity::Fatal => SeverityLevel::Fatal, + } + } +} + +/// Whatever this database and the caller disagree about, if anything. +/// +/// Both sides resolved the same project from the same files on disk, and they have to have +/// resolved it to the same thing before one can answer for the other. What is compared is +/// what each side actually ended up with rather than what it was told, because most of the +/// ways they can differ are not written down anywhere: a flag on the command line, a setting +/// the editor contributed, an interpreter discovered out of one process's environment and not +/// the other's. +fn disagreement(db: &ProjectDatabase, request: &CheckRequest) -> Option { + let options = + match protocol::configuration(db.project().metadata(db).to_merged_options().options()) { + Ok(options) => options, + Err(error) => { + tracing::debug!("Failed to serialize the project's options: {error}"); + return Some(Refusal::UnknownProject); + } + }; + if options != request.options { + return Some(Refusal::Options { + server: Box::new(options), + }); + } + + let environment = environment(db); + if environment != request.environment { + return Some(Refusal::Environment { + server: environment, + }); + } + + if request.verbose { + return Some(Refusal::Verbose); + } + + let force_exclude = db.project().force_exclude(db); + if force_exclude != request.force_exclude { + return Some(Refusal::ForceExclude { + server: force_exclude, + }); + } + + None +} + +/// A project's resolved python version, platform and search paths, as one comparable string. +/// +/// The `Debug` rendering rather than a hand-built one because it is already the curated set: +/// it prints the four kinds of search path and deliberately leaves out the typeshed version +/// map, which is thousands of lines and identical wherever the typeshed path is. +/// +/// The project's own directory is written out of it first. Two processes reach the same tree +/// by different names — a `by check` takes its root from the working directory, which the +/// operating system hands over with symlinks resolved, while an editor sends whatever it was +/// opened with — and every first-party search path, and every path under the project such as +/// a `.venv` beside it, inherits that difference. What is left is what the two would actually +/// disagree about: a different interpreter, a different typeshed, a different version. +pub fn environment(db: &ProjectDatabase) -> String { + let rendered = format!("{:?}", db.project().program_settings(db)); + let root = db.project().root(db); + + // both spellings, because which one this side happens to hold is the whole problem + let canonical = super::canonical(root); + let mut rendered = rendered.replace(root.as_str(), PROJECT_ROOT); + if canonical.as_str() != root.as_str() { + rendered = rendered.replace(canonical.as_str(), PROJECT_ROOT); + } + rendered +} + +/// What the project's own directory is called in an [`environment`] rendering. +const PROJECT_ROOT: &str = ""; + +/// The database rooted exactly at the project the caller resolved, if this session holds one. +/// +/// Exactly, not enclosing: a caller inside a workspace member resolved that member as its +/// project, and a database rooted at the repository above it checks a different set of files +/// under different settings. +fn project_database<'a>( + snapshot: &'a SessionSnapshot, + request: &CheckRequest, +) -> Option<&'a ProjectDatabase> { + let wanted = super::canonical(&request.project_root); + + snapshot + .projects() + .iter() + .find(|db| super::canonical(db.project().root(*db)) == wanted) +} + +/// The open documents this project might read whose content is not what is on disk. +/// +/// `db` first because everything about the question is the database's: which paths are in +/// scope, and what the file system says a path holds. +/// +/// In scope is the project's own tree, plus anything this database has already interned a +/// file for — an editable install, a stub package, a file on an extra search path. The second +/// half matters because [`LSPSystem`](crate::system::LSPSystem) lays the editor's buffers +/// over the whole file system, not over the project: a check that resolves an import into a +/// file the editor happens to be holding reads the buffer. What is deliberately *not* in +/// scope is a document from some unrelated workspace in the same editor window, which this +/// project would never read and which should not cost a caller its answer. +fn unsaved_documents(db: &ProjectDatabase, snapshot: &SessionSnapshot) -> Vec { + let root = super::canonical(db.project().root(db)); + + snapshot + .open_documents() + .filter(|(path, _)| { + super::canonical(path).starts_with(&root) || db.files().try_system(db, path).is_some() + }) + .filter(|(path, document)| match *document { + // a file the file system cannot produce at all — one the editor holds for a file + // that has since been deleted — differs like any other + OpenDocument::Text(contents) => { + snapshot.read_from_disk(path).as_deref() != Some(contents) + } + OpenDocument::Notebook => true, + }) + .map(|(path, _)| { + // the prefix to strip is the canonical root, so what it is stripped from has to + // be the canonical path — a file in scope only because the database interned it + // may still sit outside the root, and keeps its own spelling + let canonical = super::canonical(path); + canonical + .strip_prefix(&root) + .unwrap_or(path) + .as_str() + .to_string() + }) + .collect() +} diff --git a/crates/ty_server/src/project_server/client.rs b/crates/ty_server/src/project_server/client.rs new file mode 100644 index 0000000000..4fb02b66ed --- /dev/null +++ b/crates/ty_server/src/project_server/client.rs @@ -0,0 +1,158 @@ +//! Asking a running server for an answer, from the `by` command line. +//! +//! Everything in here fails quietly. A caller uses this the way it would use a cache: it +//! asks, and if it does not get an answer it does the work. So there is no error type — a +//! failure is `None` and a line in the log, and the caller carries on into the check it was +//! always able to run. + +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::{Ipv4Addr, SocketAddr, TcpStream}; +use std::time::Duration; + +use ruff_db::system::SystemPath; + +use super::discovery; +use super::protocol::{ + Answer, Build, CheckRequest, CheckResponse, PROTOCOL, Payload, Refusal, Request, Response, +}; + +/// How long the server has to accept a connection. +/// +/// It is on this machine, and it is either listening or it is a record left behind by a +/// process that has died. There is no third case worth waiting on. +const CONNECT_TIMEOUT: Duration = Duration::from_millis(500); + +/// How long the server has to answer once it has accepted. +/// +/// Long, because the answer is a real check: the usual case is a database that has the result +/// already, but a server that has just started, or one whose project changed under it, does +/// the same work the caller would have done. Not unbounded, because a server whose main loop +/// has wedged would otherwise leave the caller waiting for something that is never coming, +/// with a check it could have run itself sitting there the whole time. +const RESPONSE_TIMEOUT: Duration = Duration::from_secs(120); + +const SEND_TIMEOUT: Duration = Duration::from_secs(10); + +/// The most an answer may be. +/// +/// A project's rendered diagnostics can genuinely be megabytes, so this is generous — but the +/// port a record names is free for anything to bind once its server dies, and a reply from +/// something that is not a server should not be able to fill memory before the token that +/// would have caught it is read. +const MAX_RESPONSE: u64 = 256 * 1024 * 1024; + +/// Asks a server holding `request.project_root` to check it. +/// +/// `None` when there is no server to ask, when the ones there are will not answer for this +/// caller, or when anything at all goes wrong on the way. +pub fn check( + directory: &SystemPath, + request: CheckRequest, + build: &Build, +) -> Option { + let project_root = request.project_root.clone(); + let payload = Payload::Check(request); + + // two editors can hold the same project, and one of them may be in a state that stops it + // answering while the other is not — so a refusal moves on to the next rather than ending + // the search. every refusal is a server that did nothing, so the request is still ours to + // send again + for candidate in discovery::candidates(directory, &project_root) { + match ask(&candidate, &payload, build) { + Some(Response::Check(response)) => return Some(response), + Some(Response::Refused { reason }) => { + tracing::debug!("A project server did not answer: {reason}"); + if let Refusal::Options { server } = &reason { + tracing::debug!("It resolved: {server}"); + } + } + None => {} + } + } + + tracing::debug!("No project server answered for `{project_root}`"); + None +} + +/// One round trip, or `None` if there wasn't one. +/// +/// `payload` is borrowed rather than moved because every outcome here leaves the request +/// still ours to make: a server that refuses did nothing, and a server that could not be +/// reached never saw it. +fn ask(candidate: &discovery::Candidate, payload: &Payload, build: &Build) -> Option { + let record = &candidate.record; + + // both of these are in the record, so they are settled before a connection is opened + // rather than after a round trip + if record.protocol != PROTOCOL { + tracing::debug!( + "Ignoring a project server speaking protocol {} (this build speaks {PROTOCOL})", + record.protocol + ); + return None; + } + if !build.is(&record.build) { + tracing::debug!( + "Ignoring a project server from a different build ({})", + record.build.version + ); + return None; + } + + let address = SocketAddr::from((Ipv4Addr::LOCALHOST, record.port)); + let Ok(mut connection) = TcpStream::connect_timeout(&address, CONNECT_TIMEOUT) else { + // a listening server always accepts, so this is a record whose server is gone + tracing::debug!("No project server is listening on {address}; forgetting its record"); + discovery::forget(candidate); + return None; + }; + + let request = Request { + protocol: PROTOCOL, + token: record.token.clone(), + client: build.clone(), + payload: serde_json::to_value(payload) + .inspect_err(|error| tracing::debug!("Failed to serialize the request: {error}")) + .ok()?, + }; + + match round_trip(&mut connection, &request, &record.token) { + Ok(response) => Some(response), + Err(error) => { + tracing::debug!("The project server on {address} did not answer: {error}"); + None + } + } +} + +fn round_trip( + connection: &mut TcpStream, + request: &Request, + token: &str, +) -> anyhow::Result { + connection.set_read_timeout(Some(RESPONSE_TIMEOUT))?; + connection.set_write_timeout(Some(SEND_TIMEOUT))?; + + let mut line = serde_json::to_vec(request)?; + line.push(b'\n'); + connection.write_all(&line)?; + connection.flush()?; + + let mut response = String::new(); + BufReader::new(connection) + .take(MAX_RESPONSE) + .read_line(&mut response)?; + if response.is_empty() { + anyhow::bail!("the server closed the connection without answering"); + } + + let answer: Answer = serde_json::from_str(&response)?; + + // the record's port was free to be taken by anything once the server that published it + // died, so an answer only counts if it came from something that had read the record + if answer.token != token { + anyhow::bail!("something other than the project server answered on its port"); + } + + Ok(answer.response) +} diff --git a/crates/ty_server/src/project_server/discovery.rs b/crates/ty_server/src/project_server/discovery.rs new file mode 100644 index 0000000000..b334de7e2c --- /dev/null +++ b/crates/ty_server/src/project_server/discovery.rs @@ -0,0 +1,194 @@ +//! How a command line finds a server that is already holding this project. +//! +//! A server is started by an editor, over stdin and stdout, and nothing about that says +//! where it is or that it exists. So a listening server leaves a small record of itself in +//! a per-user directory, and a command line reads the directory. +//! +//! The record is the capability: it carries the port *and* the token, so being able to +//! reach a server means being able to read a file that only this user can read. + +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; + +use std::io::Write; +use std::path::{Path, PathBuf}; + +use ruff_db::system::{System, SystemPath, SystemPathBuf}; + +use super::protocol::Build; + +/// What one listening server publishes about itself. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Record { + pub protocol: u32, + pub build: Build, + pub port: u16, + pub token: String, + + /// The workspace roots this server was started for. + /// + /// A caller keeps a record whose roots contain the project it resolved. That is a + /// filter, not a promise — whether a database is actually rooted at that project is + /// something only the server can answer, and it does. + pub roots: Vec, +} + +/// Where records live for this user, or `None` where this platform has no home to put them. +/// +/// Under ty's own cache directory rather than a directory of this module's choosing, so that +/// there is one answer to "where does ty keep per-user state" and it is the same answer on +/// every platform. +/// +/// Callers pass the directory in rather than reaching for this, so that a server started for +/// a test publishes somewhere a real `by check` will never look. +pub fn default_directory(system: &dyn System) -> Option { + Some(system.cache_dir()?.join("project-servers")) +} + +/// A record on disk, removed when the server that wrote it lets go of this. +#[derive(Debug)] +pub(crate) struct Registration { + path: PathBuf, +} + +impl Drop for Registration { + fn drop(&mut self) { + if let Err(error) = std::fs::remove_file(&self.path) { + tracing::debug!( + "Failed to remove the project server record `{}`: {error}", + self.path.display() + ); + } + } +} + +/// Publishes `record`, replacing whatever this process published before. +/// +/// Keyed by process id, so a server that died without cleaning up is overwritten by +/// whatever the operating system next gives that number to, and in the meantime is +/// recognised as dead by [`live_records`] failing to connect to it. +pub(crate) fn publish(directory: &SystemPath, record: &Record) -> std::io::Result { + let directory = directory.as_std_path(); + std::fs::create_dir_all(directory)?; + + let path = directory.join(format!("{}.json", std::process::id())); + let mut file = std::fs::File::create(&path)?; + restrict_to_owner(&file)?; + file.write_all(&serde_json::to_vec(record)?)?; + file.flush()?; + + Ok(Registration { path }) +} + +/// The token is the whole of the access control, so nobody else may read the file. +#[cfg(unix)] +fn restrict_to_owner(file: &std::fs::File) -> std::io::Result<()> { + file.set_permissions(std::fs::Permissions::from_mode(0o600)) +} + +/// Windows has no mode bits; the containing directory is already per-user. +#[cfg(not(unix))] +fn restrict_to_owner(_file: &std::fs::File) -> std::io::Result<()> { + Ok(()) +} + +/// A published record, and the file it was published in. +/// +/// The path travels with it so that a caller who finds nothing listening can take the record +/// away — see `forget`. +#[derive(Debug, Clone)] +pub struct Candidate { + pub record: Record, + path: PathBuf, +} + +/// Removes a record whose server is not there any more. +/// +/// A server takes its own record away as it exits, but a server that was killed never gets +/// to. Nothing else would ever remove those, and each one costs every later `by check` a +/// connection attempt. +/// +/// The file is re-read first. Records are named by process id, so between reading one and +/// failing to reach it a new server can have been given that id and published over it — +/// deleting that one would leave a live server undiscoverable for the rest of its life. +pub(super) fn forget(candidate: &Candidate) { + match read_record(&candidate.path) { + Some(current) if current.port != candidate.record.port => { + tracing::debug!( + "Leaving `{}` alone: another server has published over it", + candidate.path.display() + ); + return; + } + _ => {} + } + + if let Err(error) = std::fs::remove_file(&candidate.path) { + tracing::debug!( + "Failed to remove the stale project server record `{}`: {error}", + candidate.path.display() + ); + } +} + +/// Every published record whose roots contain `project_root`, nearest root first. +/// +/// Both `project_root` and each record's roots go through `super::canonical` before they are +/// compared: the two reach the same tree from different processes, and neither necessarily by +/// the same name. +/// +/// Nearest first because a workspace opened at a repository root and one opened at the +/// package inside it are both candidates, and the inner one is the one whose settings were +/// resolved for this project. +pub fn candidates(directory: &SystemPath, project_root: &SystemPath) -> Vec { + let Ok(entries) = std::fs::read_dir(directory.as_std_path()) else { + return Vec::new(); + }; + + let project_root = super::canonical(project_root); + let mut candidates: Vec<(usize, Candidate)> = entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter_map(|path| Some((read_record(&path)?, path))) + .filter_map(|(record, path)| { + let depth = record + .roots + .iter() + // both sides, because a root reaches this file from whichever process wrote + // it and a project root reaches this function from whichever asked — see + // [`super::canonical`] + .map(|root| super::canonical(root)) + .filter(|root| project_root.starts_with(root)) + .map(|root| root.components().count()) + .max()?; + Some((depth, Candidate { record, path })) + }) + .collect(); + + candidates.sort_by(|(left, _), (right, _)| right.cmp(left)); + candidates + .into_iter() + .map(|(_, candidate)| candidate) + .collect() +} + +fn read_record(path: &Path) -> Option { + if path.extension()? != "json" { + return None; + } + let contents = std::fs::read(path).ok()?; + match serde_json::from_slice(&contents) { + Ok(record) => Some(record), + // a record written by a build that has since changed the shape. it is not this + // process's to delete — the server that wrote it still owns the file, and will + // remove it when it exits + Err(error) => { + tracing::debug!( + "Ignoring unreadable project server record `{}`: {error}", + path.display() + ); + None + } + } +} diff --git a/crates/ty_server/src/project_server/listener.rs b/crates/ty_server/src/project_server/listener.rs new file mode 100644 index 0000000000..7ac58a1257 --- /dev/null +++ b/crates/ty_server/src/project_server/listener.rs @@ -0,0 +1,242 @@ +//! The side channel a `by` command line reaches the server on. +//! +//! The language server's own connection is stdin and stdout, and it belongs to the editor +//! that started it. So requests from a command line arrive somewhere else: a loopback socket, +//! opened once at startup, announced through a [record](super::discovery::Record) only this +//! user can read. +//! +//! Nothing here touches the session. A connection is read, checked against the two things +//! that can be settled without it — the protocol and the build — and then handed to the main +//! loop, which is the only place a database may be looked at. + +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::{Ipv4Addr, SocketAddr, TcpListener, TcpStream}; +use std::time::Duration; + +use ruff_db::system::{SystemPath, SystemPathBuf}; + +use super::discovery::{self, Record, Registration}; +use super::protocol::{Answer, Build, PROTOCOL, Payload, Refusal, Request, Response}; +use crate::server::{Event, MainLoopSender}; + +/// How long a client is given to send its request once it has connected. +/// +/// A connection that opens and says nothing holds a thread, and the port is reachable by +/// anything running on this machine. +const READ_TIMEOUT: Duration = Duration::from_secs(5); + +/// How long the server will spend trying to hand an answer back. +/// +/// A client that stops reading part way through would otherwise pin a worker thread for as +/// long as it cared to. +const WRITE_TIMEOUT: Duration = Duration::from_secs(30); + +/// The most a request may be. +/// +/// Nothing legitimate comes close: the largest field is a project's merged configuration, +/// which is kilobytes. The cap is here because reading a line from a socket that anyone on +/// this machine can connect to is otherwise an invitation to fill memory, and the token that +/// would stop them is not read until after the line is. +const MAX_REQUEST: u64 = 4 * 1024 * 1024; + +/// A request that reached the main loop, and the connection waiting for its answer. +#[derive(Debug)] +pub(crate) struct Incoming { + pub(crate) payload: Payload, + pub(crate) connection: TcpStream, + + /// The secret this request arrived with, to be repeated in the answer. + pub(crate) token: String, + + /// Whether the file system has been re-read for this request yet. + /// + /// The re-read is the expensive part and it disrupts the editor, so it happens only once + /// everything cheap about the request has been agreed — see the main loop. + pub(crate) rescanned: bool, + + /// How many times this has been through the main loop. + /// + /// A check runs on a snapshot, and an edit in the editor cancels it. Re-queueing takes a + /// fresh snapshot, which is the only way to retry; giving up after a few is what keeps a + /// caller from waiting on a session that is being typed into. + pub(crate) attempts: u8, +} + +/// Everything the server holds open for as long as it accepts command-line requests. +/// +/// Dropping it takes the record off disk. The listener thread ends when the process does: it +/// is blocked in `accept`, and there is nothing worth waking it for. +#[derive(Debug)] +pub(crate) struct Listener { + _registration: Registration, +} + +impl Listener { + /// Starts accepting command-line requests for `roots`, announcing itself in `directory`. + /// + /// `Ok(None)` when there is nothing to announce: a server with no workspace folders holds + /// no project any caller could ask about, so a record for it could only ever be a port to + /// connect to and be refused by. + /// + /// An `Err` is the listener failing to start, which is not fatal: a server that cannot be + /// reached this way is a server every command line checks for itself, which is what they + /// all did before. + pub(crate) fn spawn( + sender: MainLoopSender, + directory: &SystemPath, + roots: Vec, + version: &str, + ) -> anyhow::Result> { + if roots.is_empty() { + tracing::debug!("Not accepting command-line requests: this server has no workspaces"); + return Ok(None); + } + + let listener = TcpListener::bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0)))?; + let port = listener.local_addr()?.port(); + let token = super::token()?; + + let build = Build::current(version); + let registration = discovery::publish( + directory, + &Record { + protocol: PROTOCOL, + build: build.clone(), + port, + token: token.clone(), + roots, + }, + )?; + + std::thread::Builder::new() + .name("ty:project-server".to_owned()) + .spawn(move || accept_loop(&listener, &sender, &token, &build))?; + + tracing::info!("Accepting command-line requests on 127.0.0.1:{port}"); + + Ok(Some(Self { + _registration: registration, + })) + } +} + +fn accept_loop(listener: &TcpListener, sender: &MainLoopSender, token: &str, build: &Build) { + std::thread::scope(|scope| { + for connection in listener.incoming() { + let Ok(connection) = connection.inspect_err(|error| { + tracing::debug!("Failed to accept a command-line connection: {error}"); + }) else { + continue; + }; + + // one thread per connection, because reading one is the only part of this that + // waits on somebody else. a single slow caller — or a local process that connects + // and says nothing until its five seconds are up — would otherwise be enough to + // stop every other caller being heard + let spawned = std::thread::Builder::new() + .name("ty:project-server-request".to_owned()) + .spawn_scoped(scope, || { + if let Err(error) = accept(connection, sender, token, build) { + tracing::debug!("Failed to read a command-line request: {error}"); + } + }); + + if let Err(error) = spawned { + tracing::debug!("Failed to start a thread for a command-line request: {error}"); + } + } + }); +} + +fn accept( + mut connection: TcpStream, + sender: &MainLoopSender, + token: &str, + build: &Build, +) -> anyhow::Result<()> { + connection.set_read_timeout(Some(READ_TIMEOUT))?; + connection.set_write_timeout(Some(WRITE_TIMEOUT))?; + + let mut line = String::new(); + BufReader::new(connection.try_clone()?) + .take(MAX_REQUEST) + .read_line(&mut line)?; + let request: Request = serde_json::from_str(&line)?; + + // an unreadable request and a wrong token get the same treatment, which is none: the + // caller learns whether it guessed the token by whether it is answered at all + if !constant_time_eq(&request.token, token) { + tracing::warn!("Rejecting a command-line request with the wrong token"); + return Ok(()); + } + + if request.protocol != PROTOCOL { + return respond( + &mut connection, + token, + &Response::Refused { + reason: Refusal::Protocol { server: PROTOCOL }, + }, + ); + } + + // same protocol is not the same checker. two builds of `by` disagree about what this + // project's diagnostics are as readily as they disagree about anything else, and the + // caller asked for its own answer + if !build.is(&request.client) { + return respond( + &mut connection, + token, + &Response::Refused { + reason: Refusal::Version { + server: Box::new(build.clone()), + }, + }, + ); + } + + // the payload is only parsed once the two agree on how to read it. a newer client's + // request would fail here, having already been told that its build is not this one + let payload: Payload = serde_json::from_value(request.payload)?; + + sender + .send(Event::ProjectServer(Box::new(Incoming { + payload, + connection, + token: request.token, + rescanned: false, + attempts: 0, + }))) + .map_err(|_| anyhow::anyhow!("the main loop is gone")) +} + +pub(crate) fn respond( + connection: &mut TcpStream, + token: &str, + response: &Response, +) -> anyhow::Result<()> { + let mut line = serde_json::to_vec(&Answer { + token: token.to_owned(), + response: response.clone(), + })?; + line.push(b'\n'); + connection.write_all(&line)?; + connection.flush()?; + Ok(()) +} + +/// Whether two secrets are equal, in time that does not depend on where they first differ. +/// +/// The channel is loopback and the attack is not a practical one, but a comparison that +/// leaks is not cheaper to write than one that does not. +fn constant_time_eq(left: &str, right: &str) -> bool { + let (left, right) = (left.as_bytes(), right.as_bytes()); + if left.len() != right.len() { + return false; + } + + left.iter() + .zip(right) + .fold(0u8, |difference, (left, right)| difference | (left ^ right)) + == 0 +} diff --git a/crates/ty_server/src/project_server/mod.rs b/crates/ty_server/src/project_server/mod.rs new file mode 100644 index 0000000000..499bdd328f --- /dev/null +++ b/crates/ty_server/src/project_server/mod.rs @@ -0,0 +1,195 @@ +//! Answering `by` command lines out of a server that is already holding the project. +//! +//! A `by check` builds a project database, resolves an environment, parses every file and +//! infers every one of them, and then exits and throws all of it away. An editor with the +//! language server running has that same work already done and kept up to date. This is the +//! way across: a loopback socket the server announces in a per-user +//! [record](discovery::Record), a [request](protocol::Request) that names the project and +//! the configuration the caller resolved, and an answer rendered out of the warm database. +//! +//! Two rules shape everything here. +//! +//! The hot answer has to be the cold answer. A server checks what the editor is holding, +//! under settings the editor contributed to, in a build that may be months older than the +//! `by` on the `PATH` — so the answer only crosses when none of that is true, and every +//! other case is a [refusal](protocol::Refusal) the caller silently checks past. A refusal +//! costs the time it would have cost anyway. A wrong answer costs trust in the command. +//! +//! And it is never the only way to get an answer. Nothing here is required to work: no +//! server, a stale record, a refused connection and a garbled reply all lead to the same +//! place, which is the check the caller would have run if none of this existed. + +use std::fmt::Write; + +use rand::TryRng; +use ruff_db::system::{System, SystemPath, SystemPathBuf}; +use ty_static::EnvVars; + +mod check; +pub mod client; +pub mod discovery; +mod listener; +pub mod protocol; + +pub use check::environment; +pub(crate) use listener::{Incoming, Listener, respond}; + +/// Whether [`EnvVars::BY_NO_PROJECT_SERVER`] has switched the project server off. +/// +/// One switch for both halves, because a user who does not want a command line reaching into +/// their editor's process does not want the port open either. +/// +/// Read through `system` rather than the process environment, which is what the rest of the +/// server does: tests run concurrently in one process, so a test that wanted to exercise this +/// could not set the variable without setting it for every other test at the same time. +pub fn disabled(system: &dyn System) -> bool { + system + .env_var(EnvVars::BY_NO_PROJECT_SERVER) + .is_ok_and(|value| { + !matches!( + value.trim().to_ascii_lowercase().as_str(), + "" | "0" | "false" | "no" | "n" | "off" + ) + }) +} + +/// `path` with every symlink resolved, or `path` itself where it cannot be. +/// +/// The two sides reach the same project by different names. A `by check` takes its project +/// root from the working directory, which the operating system hands over already resolved; +/// an editor sends whatever it was opened with. On macOS that alone is enough to disagree — +/// a temporary directory is `/var/…` to one and `/private/var/…` to the other — and a user +/// whose project is behind a symlink disagrees everywhere. +/// +/// So both roots are put through this before either is compared to the other, and a path +/// that cannot be resolved at all is left alone rather than dropped: it still compares equal +/// to itself, which is the case where both sides spell it the same way. +fn canonical(path: &SystemPath) -> SystemPathBuf { + // Two things make a resolved path stop comparing against an unresolved one. Resolving + // only succeeds for a path that exists, so a directory and a file under it that has not + // been written yet would come back spelled differently; and on Windows the answer is a + // verbatim `\\?\C:\…` path, which no path the editor or the caller sends is spelled + // like. So the longest ancestor that does exist is resolved, the rest is put back on, + // and the verbatim prefix is dropped the way `System::canonicalize_path` drops it — + // everything under one root then comes back in one spelling, created or not. + let mut unresolved = Vec::new(); + let mut current = path; + loop { + if let Ok(resolved) = std::fs::canonicalize(current.as_std_path()) + && let Ok(resolved) = SystemPathBuf::from_path_buf(resolved) + { + let mut resolved = resolved.simplified().to_path_buf(); + for component in unresolved.iter().rev() { + resolved.push(component); + } + return resolved; + } + let (Some(parent), Some(name)) = (current.parent(), current.file_name()) else { + return path.to_path_buf(); + }; + unresolved.push(name); + current = parent; + } +} + +/// A fresh secret for one server's lifetime. +/// +/// The listener is on loopback, which every user on this machine can reach; the token is +/// what makes reaching it depend on being able to read a file that only this user can. +fn token() -> anyhow::Result { + let mut bytes = [0u8; 32]; + rand::rngs::SysRng + .try_fill_bytes(&mut bytes) + .map_err(|error| anyhow::anyhow!("failed to read random bytes: {error}"))?; + + let mut token = String::with_capacity(bytes.len() * 2); + for byte in bytes { + write!(token, "{byte:02x}")?; + } + Ok(token) +} + +/// What a pass over a request produced. +#[derive(Debug)] +pub(crate) enum Outcome { + /// Nothing stands in the way, and the file system has not been re-read yet. + /// + /// The re-read has to happen on the main loop, because it mutates the session — so a + /// request that gets this far goes back there and comes round once more. + NeedsRescan, + + Answered(protocol::Response), +} + +/// Runs a request against `snapshot`. +/// +/// `rescanned` says whether the file system has already been re-read for this request. Until +/// it has, the most this can do is agree that the request is answerable. +/// +/// A panic in the checker is caught here rather than left to the request-handling hook, +/// because the caller on the other end of the socket is waiting for one of two shapes and a +/// server that says nothing at all leaves it there until its own timeout. +pub(crate) fn run( + snapshot: &crate::session::SessionSnapshot, + payload: &protocol::Payload, + rescanned: bool, +) -> Outcome { + // nothing borrowed here is read again after an unwind: the outcome is built from scratch + // on either path, so asserting unwind safety costs nothing + let checked = ruff_db::panic::catch_unwind(std::panic::AssertUnwindSafe(|| match payload { + protocol::Payload::Check(request) => check::run(snapshot, request, rescanned), + })); + + match checked { + Ok(outcome) => outcome, + Err(panic) => { + tracing::error!("A command-line request panicked: {panic}"); + Outcome::Answered(protocol::Response::Refused { + reason: protocol::Refusal::Panicked, + }) + } + } +} + +/// Whether a fresh snapshot could turn `response` into an answer. +/// +/// A check runs against a snapshot, and typing in the editor cancels it. Retrying means +/// going back through the main loop for a snapshot of what the session is now. +pub(crate) fn is_retryable(response: &protocol::Response) -> bool { + matches!( + response, + protocol::Response::Refused { + reason: protocol::Refusal::Cancelled + } + ) +} + +#[cfg(test)] +mod tests { + use super::canonical; + use ruff_db::system::SystemPath; + use tempfile::TempDir; + + /// The two sides compare a root against a path under it, and a path is only ever + /// compared to another that went through [`canonical`] too. What breaks that is one + /// spelling for a path that exists and another for one that does not: a temporary + /// directory reached through a symlink resolves elsewhere, and on Windows a resolved + /// path carries a verbatim prefix an unresolved one never has. + #[test] + fn a_path_that_does_not_exist_yet_is_spelled_like_the_root_it_is_under() { + let directory = TempDir::new().expect("a temporary directory"); + let root = SystemPath::from_std_path(directory.path()).expect("a UTF-8 path"); + + let canonical_root = canonical(root); + let nested = canonical(&root.join("not-written-yet")); + + assert!( + nested.starts_with(&canonical_root), + "`{nested}` should be inside `{canonical_root}`" + ); + assert_eq!( + nested.strip_prefix(&canonical_root).map(SystemPath::as_str), + Ok("not-written-yet") + ); + } +} diff --git a/crates/ty_server/src/project_server/protocol.rs b/crates/ty_server/src/project_server/protocol.rs new file mode 100644 index 0000000000..f7a2c6b35d --- /dev/null +++ b/crates/ty_server/src/project_server/protocol.rs @@ -0,0 +1,377 @@ +//! What a `by` command line and a running server say to each other. +//! +//! One JSON object per message, terminated by a newline. `serde_json` escapes newlines +//! inside strings, so a rendered diagnostic — which is full of them — still arrives as a +//! single line. + +use ruff_db::system::SystemPathBuf; + +/// Bumped whenever the shape below changes. +/// +/// A client and a server that disagree on it stop talking, which matters because the two +/// are separate builds of separate binaries: the server in the editor may be months older +/// than the `by` on the `PATH`. +pub const PROTOCOL: u32 = 1; + +/// A project's merged configuration, in the one shape both sides will produce for it. +/// +/// Serializing the options is not enough on its own. A command line builds its layer of them +/// out of its arguments, and does so by filling in every group whether or not anything went +/// into it — so a `by check` with no flags at all resolves `{"environment": {}, "terminal": +/// {}, …}` where a server resolves `{}`. Those say the same thing, and a comparison that +/// called them different would refuse every invocation there is. +/// +/// So a group with nothing in it is removed, recursively. An empty *list* is left alone: it +/// is a real setting, and `exclude = []` does not mean the same as no `exclude` at all. +pub fn configuration( + options: &T, +) -> Result { + fn prune(value: serde_json::Value) -> serde_json::Value { + match value { + serde_json::Value::Object(fields) => serde_json::Value::Object( + fields + .into_iter() + .map(|(name, value)| (name, prune(value))) + .filter(|(_, value)| match value { + serde_json::Value::Null => false, + serde_json::Value::Object(fields) => !fields.is_empty(), + _ => true, + }) + .collect(), + ), + serde_json::Value::Array(items) => { + serde_json::Value::Array(items.into_iter().map(prune).collect()) + } + value => value, + } + } + + serde_json::to_value(options).map(prune) +} + +/// The part of a request that is read before anything is decided. +/// +/// Deliberately lenient about its payload, which stays JSON until the protocol and the build +/// have been agreed. A stricter envelope would fail to parse a newer client's request and +/// close the connection without saying why, which is the one case the version fields exist to +/// explain. +#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Request { + pub protocol: u32, + + /// The secret from the server's own discovery record. + /// + /// The listener is on loopback, so anything running as any user on this machine can + /// reach the port. The record file it has to read first is not readable by them. + pub token: String, + + pub client: Build, + + pub payload: serde_json::Value, +} + +/// Enough about a build of `by` to tell it apart from another one. +/// +/// The version alone is not enough, and the case it misses is the one that matters most: two +/// builds of the same commit with different uncommitted changes report the same version, and +/// that is the ordinary state of an editor holding a server while its `by` is rebuilt. So the +/// executable is identified as well — the same file, unmodified, is the same build, which is +/// the only direction this needs to be sure about. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Build { + pub version: String, + + /// The executable, and what the file system last said about it. + /// + /// `None` where the executable cannot be identified at all, which never compares equal to + /// anything — including another `None`. + pub executable: Option, +} + +impl Build { + /// This process's build. + pub fn current(version: &str) -> Self { + Self { + version: version.to_owned(), + executable: Executable::current(), + } + } + + /// Whether `other` is known to be the same build as this one. + /// + /// Never assumes: two builds it cannot tell apart are reported as different, because the + /// cost of that is a check the caller was always able to run. + pub(crate) fn is(&self, other: &Self) -> bool { + self.version == other.version + && match (&self.executable, &other.executable) { + (Some(left), Some(right)) => left == right, + _ => false, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Executable { + pub path: String, + pub modified: u64, + pub size: u64, +} + +impl Executable { + fn current() -> Option { + let path = std::env::current_exe().ok()?; + let metadata = std::fs::metadata(&path).ok()?; + Some(Self { + path: path.to_str()?.to_owned(), + modified: metadata + .modified() + .ok()? + .duration_since(std::time::UNIX_EPOCH) + .ok()? + .as_secs(), + size: metadata.len(), + }) + } +} + +#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase", tag = "kind")] +pub enum Payload { + Check(CheckRequest), +} + +#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CheckRequest { + /// The project the command line resolved for itself. + /// + /// A server holds a database per workspace, and a workspace is not always a project — + /// so the root is matched against the databases rather than assumed. + pub project_root: SystemPathBuf, + + /// The command line's configuration, merged across every layer, as JSON. + /// + /// The two sides must agree about what checking this project *means* before one can + /// answer for the other. Merged, because the layers are where the disagreements are: the + /// caller's flags and the editor's own settings both arrive as layers over the same + /// configuration file, and comparing the file alone compares the one thing that could + /// never have differed. + /// + /// Sending the whole thing rather than a hash of it costs a few kilobytes once and buys a + /// diffable explanation when the two disagree. + pub options: serde_json::Value, + + /// The python version, platform and search paths the command line resolved. + /// + /// Not derivable from the options: an environment is discovered as much as configured, + /// out of `VIRTUAL_ENV`, `CONDA_PREFIX`, a `.venv` beside the project, uv's answer, or the + /// interpreter the running executable sits next to — and a server started by an editor + /// discovers it from the editor's environment, not from the caller's shell. Two processes + /// that agree about every option can still be checking against different site-packages. + /// + /// A rendering rather than the settings themselves, because what has to cross is an + /// identity to compare and a difference to print. + pub environment: String, + + /// Whether exclusions apply to paths named on the command line. + /// + /// A database input rather than an option, so it is not in the merged configuration and + /// has to travel on its own. + pub force_exclude: bool, + + /// Whether every diagnostic should say where its rule was turned on. + /// + /// Also a database input, and one a server never sets: an editor has its own way of + /// showing where a rule came from. So a verbose check is always refused — which is the + /// point of comparing it rather than declining to ask, because a refusal says so in the + /// log where a decision not to ask would say nothing. + pub verbose: bool, + + /// The directory the command was run from. + /// + /// Diagnostics name their file relative to where the reader is standing, and the server + /// is standing wherever the editor started it. Without this every path in the answer + /// would be absolute, which is a visible difference from the same check run cold. + pub working_directory: SystemPathBuf, + + /// Whether the command line's output is going somewhere that renders ANSI colour. + /// + /// The server cannot see the caller's terminal, and its own stdout is the LSP + /// connection. + pub color: bool, +} + +/// A response, and the proof that it came from the server the caller meant. +/// +/// The token travels back because a record outlives the process that wrote it: a server that +/// was killed leaves its port behind, and any local process that binds that port next +/// receives the request. Without this, such a process could answer "All checks passed!" and +/// the caller would exit zero on a project it never looked at. +#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Answer { + pub token: String, + pub response: Response, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase", tag = "kind")] +pub enum Response { + Check(CheckResponse), + + /// The server will not answer, and the caller should check for itself. + /// + /// Never an error: every refusal is a case where the hot answer might not have been + /// the answer a cold run would give. + Refused { + reason: Refusal, + }, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +// four flags, because a rendering is the one thing that crosses and everything the caller +// still has to decide has to cross beside it. they are not a state machine; they are four +// independent facts about one check +#[expect(clippy::struct_excessive_bools)] +pub struct CheckResponse { + /// The diagnostics, already rendered in the project's configured output format. + /// + /// Rendering happens on the server because it needs the database — the source text + /// behind every span, and the roots that make a path relative. Sending the diagnostics + /// themselves would mean the caller reopening every file they point into. + pub rendered: String, + + pub diagnostics: usize, + + /// Whether the configured output format is one a summary line belongs under. + pub human_readable: bool, + + /// The worst severity reported, or `None` when nothing was. + /// + /// The exit status is the caller's to decide — it is the process that carries it — so + /// what crosses is what the decision reads rather than the decision. + pub max_severity: Option, + + pub io_error: bool, + pub error_on_warning: bool, + + /// Whether the project turned out to hold no python at all. + /// + /// The cold path warns about this, and about [`Self::fatal`], on stderr. Neither is a + /// diagnostic, so neither is in the rendering, and a caller that did not hear them would + /// be quietly getting less than the command normally gives it. + pub empty_project: bool, + + /// Whether anything failed so badly that the check is incomplete. + pub fatal: bool, +} + +/// [`ruff_db::diagnostic::Severity`], in a shape that survives the wire. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum SeverityLevel { + Info, + Warning, + Error, + Fatal, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase", tag = "kind")] +pub enum Refusal { + Protocol { + server: u32, + }, + + /// The two are different builds of `by`. + /// + /// Same protocol, different checker: the answers could differ for any reason at all. + Version { + server: Box, + }, + + UnknownProject, + + /// The server resolved a different configuration for this project than the caller did. + Options { + server: Box, + }, + + /// The server resolved a different environment for this project than the caller did. + Environment { + server: String, + }, + + /// The two disagree about whether exclusions apply to paths named on the command line. + ForceExclude { + server: bool, + }, + + /// The caller asked for diagnostics that say where each rule was turned on. + Verbose, + + /// The server is only checking the files the editor has open. + /// + /// Its answer would be the diagnostics for those files, and a `by check` asked for the + /// project's. + CheckMode, + + /// A file open in the editor does not match what is on disk. + /// + /// The server's view of an open file is the editor's buffer, so answering from it would + /// report on source the caller cannot see. + Unsaved { + files: Vec, + }, + + /// The database changed under the check often enough that it never finished. + Cancelled, + + /// Already logged on the server. + Panicked, +} + +impl std::fmt::Display for Refusal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Refusal::Protocol { server } => { + write!( + f, + "the server speaks protocol {server}, this build speaks {PROTOCOL}" + ) + } + Refusal::Version { server } => { + write!(f, "the server is a different build ({})", server.version) + } + Refusal::UnknownProject => f.write_str("the server has no database for this project"), + Refusal::CheckMode => f.write_str( + "the server is only checking open files — set the editor's diagnostic mode to \ + `workspace` for it to answer for the whole project", + ), + Refusal::Options { .. } => { + f.write_str("the server resolved a different configuration for this project") + } + Refusal::Environment { .. } => { + f.write_str("the server resolved a different environment for this project") + } + Refusal::Verbose => f.write_str( + "a verbose check explains where each rule was turned on, and the server's \ + diagnostics do not", + ), + Refusal::ForceExclude { server } => write!( + f, + "the server checks with `force-exclude` {}", + if *server { "on" } else { "off" } + ), + Refusal::Unsaved { files } => { + write!(f, "unsaved editor changes in {}", files.join(", ")) + } + Refusal::Cancelled => f.write_str("the server was too busy to finish the check"), + Refusal::Panicked => f.write_str("the check panicked on the server"), + } + } +} diff --git a/crates/ty_server/src/server.rs b/crates/ty_server/src/server.rs index 10930cebec..bc977dbf66 100644 --- a/crates/ty_server/src/server.rs +++ b/crates/ty_server/src/server.rs @@ -10,7 +10,7 @@ use lsp_types::{ ClientCapabilities, InitializeParams, MessageType, Uri, WorkspaceFolders, WorkspaceFoldersInitializeParams, }; -use ruff_db::system::System; +use ruff_db::system::{System, SystemPathBuf}; use std::num::NonZeroUsize; use std::panic::{PanicHookInfo, RefUnwindSafe}; use std::sync::Arc; @@ -21,6 +21,7 @@ mod main_loop; mod schedule; mod script_progress; +use crate::project_server::Listener as ProjectServerListener; use crate::session::client::Client; pub(crate) use api::Error; pub(crate) use api::{ @@ -39,6 +40,12 @@ pub struct Server { main_loop_receiver: MainLoopReceiver, main_loop_sender: MainLoopSender, session: Session, + + /// The side channel `by` command lines reach this session on. + /// + /// `None` when it is switched off or failed to start. Held rather than used: dropping + /// it takes the server's discovery record off disk. + _project_server_listener: Option, } impl Server { @@ -47,6 +54,7 @@ impl Server { connection: Connection, native_system: Arc, in_test: bool, + project_server_directory: Option, ) -> crate::Result { let (id, init_value) = connection.initialize_start()?; @@ -157,11 +165,28 @@ impl Server { ) })?; + let project_server_listener = project_server_directory.and_then(|directory| { + ProjectServerListener::spawn( + main_loop_sender.clone(), + &directory, + workspace_roots(&workspace_urls), + version, + ) + // not being reachable from a command line is not a reason to fail to start: it + // costs `by check` the warm answer, which it was never entitled to + .inspect_err(|error| { + tracing::warn!("Failed to listen for command-line requests: {error}"); + }) + .ok() + .flatten() + }); + Ok(Self { connection, worker_threads, main_loop_receiver, main_loop_sender, + _project_server_listener: project_server_listener, session: Session::new( resolved_client_capabilities, position_encoding, @@ -200,6 +225,17 @@ impl Server { } } +/// The workspace URIs as paths, dropping any that does not name one. +/// +/// Only for the discovery record, which is a filter on which servers a command line bothers +/// to ask. A URI that is not a file path names nothing a command line could be running in. +fn workspace_roots(workspace_urls: &[Uri]) -> Vec { + workspace_urls + .iter() + .filter_map(|uri| SystemPathBuf::from_path_buf(uri.to_file_path().ok()?).ok()) + .collect() +} + type PanicHook = Box) + 'static + Sync + Send>; struct ServerPanicHookHandler { diff --git a/crates/ty_server/src/server/main_loop.rs b/crates/ty_server/src/server/main_loop.rs index 814eecd39d..2413ae3c3c 100644 --- a/crates/ty_server/src/server/main_loop.rs +++ b/crates/ty_server/src/server/main_loop.rs @@ -1,4 +1,6 @@ -use crate::server::schedule::Scheduler; +use crate::Session; +use crate::project_server; +use crate::server::schedule::{BackgroundSchedule, Scheduler, Task}; use crate::server::{Server, api}; use crate::session::client::{Client, ClientResponseHandler}; use crate::session::{ClientOptions, SuspendedWorkspaceDiagnosticRequest}; @@ -7,6 +9,16 @@ use lsp_server::Message; use lsp_types::Notification; use lsp_types::Uri; use ruff_db::system::SystemPathBuf; +use std::panic::AssertUnwindSafe; +use ty_project::watch::ChangeEvent; + +/// How many fresh snapshots a command-line request gets before it is told to check for +/// itself. +/// +/// Every retry is a check that ran and was cancelled, so this is bounded by patience rather +/// than by correctness: a session being typed into cancels checks faster than they finish, +/// and the caller has a cold path that always works. +const RETRY_LIMIT: u8 = 3; pub(crate) type ConnectionSender = crossbeam::channel::Sender; pub(crate) type MainLoopSender = crossbeam::channel::Sender; @@ -165,6 +177,9 @@ impl Server { // self.try_register_file_watcher(&client); } }, + Event::ProjectServer(incoming) => { + self.answer_project_server_request(incoming, &mut scheduler, client); + } Event::PollUvEnvironments { project_root } => { self.session.poll_uv_sync(&client, &project_root); } @@ -200,12 +215,87 @@ impl Server { uv_sync.select(&self.connection.receiver, &self.main_loop_receiver) } + /// Schedules a `by` command line's request against a snapshot of this session. + /// + /// Twice, in the ordinary case. The first pass decides only whether this session may + /// answer at all, and the second produces the answer — with a re-read of the file system + /// in between, on this thread, because that is what makes the answer the caller's own: + /// this session's picture of the file system is whatever the editor's watcher reported, + /// and the caller is a process that can see the disk directly, so anything the watcher + /// missed — a `git checkout` most of all — would otherwise be answered out of a tree that + /// is no longer there. + /// + /// The order is the point. Re-reading walks every project in the session and makes the + /// editor re-pull its diagnostics, which is far too much to spend on a request that was + /// going to be refused for a configuration the two never shared. + fn answer_project_server_request( + &mut self, + mut incoming: Box, + scheduler: &mut Scheduler, + client: Client, + ) { + if incoming.rescanned { + api::changes::apply(&mut self.session, &client, &[ChangeEvent::Rescan]); + } + + let sender = self.main_loop_sender.clone(); + let task = Task::background(BackgroundSchedule::Worker, move |session: &Session| { + // the snapshot is not read after an unwind: `project_server::run` catches the + // panic and builds its answer without it + let snapshot = AssertUnwindSafe(session.snapshot_session()); + + Box::new(move |_client: &Client| { + let _span = tracing::debug_span!("project server request").entered(); + let snapshot = snapshot.0; + let outcome = project_server::run(&snapshot, &incoming.payload, incoming.rescanned); + + let response = match outcome { + // back to the main loop to be re-read and then answered. the flag is set + // here rather than there so that the pass which set it is the pass that + // agreed the request was worth it + project_server::Outcome::NeedsRescan => { + incoming.rescanned = true; + requeue(&sender, incoming); + return; + } + project_server::Outcome::Answered(response) => response, + }; + + if project_server::is_retryable(&response) && incoming.attempts < RETRY_LIMIT { + incoming.attempts += 1; + requeue(&sender, incoming); + return; + } + + let token = incoming.token.clone(); + if let Err(error) = + project_server::respond(&mut incoming.connection, &token, &response) + { + tracing::debug!("Failed to answer a command-line request: {error}"); + } + }) + }); + + scheduler.dispatch(task, &mut self.session, client); + } + fn initialize(&mut self, client: &Client) { self.session .request_uninitialized_workspace_folder_configurations(client); } } +/// Sends a request back to the main loop for another pass. +/// +/// A failure means the main loop is gone and the process is on its way out; the caller sees +/// the connection close and checks for itself, which is the same thing every other failure +/// here leads to. +fn requeue(sender: &MainLoopSender, incoming: Box) { + if sender.send(Event::ProjectServer(incoming)).is_err() { + tracing::debug!("Dropping a command-line request: the main loop is gone"); + } +} + /// An action that should be performed on the main loop. #[derive(Debug)] pub(crate) enum Action { @@ -235,6 +325,9 @@ pub(crate) enum Event { Action(Action), + /// A request from a `by` command line, over the server's side channel. + ProjectServer(Box), + PollUvEnvironments { project_root: SystemPathBuf, }, diff --git a/crates/ty_server/src/session.rs b/crates/ty_server/src/session.rs index c4eb70d62d..663cf211ce 100644 --- a/crates/ty_server/src/session.rs +++ b/crates/ty_server/src/session.rs @@ -1349,6 +1349,7 @@ impl Session { resolved_client_capabilities: self.resolved_client_capabilities, revision: self.revision, client_name: self.client_name, + native_system: self.native_system.clone(), } } @@ -1675,6 +1676,12 @@ pub(crate) struct SessionSnapshot { revision: u64, client_name: ClientName, + /// The file system underneath the editor's buffers. + /// + /// Held so that a snapshot can ask what a file says on disk, which is a different + /// question from what [`Self::open_documents`] answers. + native_system: Arc, + /// IMPORTANT: It's important that the databases come last, or at least, /// after any `Arc` that we try to extract or mutate in-place using `Arc::into_inner` /// and that relies on Salsa's cancellation to guarantee that there's now only a @@ -1696,6 +1703,32 @@ impl SessionSnapshot { &self.index } + /// Every file-backed document the editor has open, with the path it stands for. + /// + /// Notebooks are included, because the whole point of asking is whether the editor is + /// holding something the file system is not. Notebook *cells* are not: a cell's + /// file-level representation is the notebook that contains it, which is already here. + /// + /// A document behind a non-`file` URI has no path on disk at all, and is left out — there + /// is nothing to compare it to, and a check that reads it is reading something a caller on + /// the command line could never have seen either. + pub(crate) fn open_documents(&self) -> impl Iterator)> { + self.index + .keyed_file_documents() + .filter_map(|(key, document)| { + let document = match document { + Document::Text(text) => OpenDocument::Text(text.contents()), + Document::Notebook(_) => OpenDocument::Notebook, + }; + Some((key.file_path()?.as_path(), document)) + }) + } + + /// What `path` says on disk, past whatever the editor is holding for it. + pub(crate) fn read_from_disk(&self, path: &SystemPath) -> Option { + self.native_system.read_to_string(path).ok() + } + pub(crate) fn global_settings(&self) -> &GlobalSettings { &self.global_settings } @@ -1721,6 +1754,18 @@ impl SessionSnapshot { } } +/// What the editor is holding for one file. +/// +/// A notebook carries no text to compare: the editor holds it as cells, and what is on disk +/// is a serialization of them whose formatting is the writer's choice. So the two are only +/// ever known to be the same when neither has been touched, and this says which case it is +/// rather than pretending to answer. +#[derive(Debug, Clone, Copy)] +pub(crate) enum OpenDocument<'a> { + Text(&'a str), + Notebook, +} + /// Represents the client (editor) that's connected to the language server. #[derive(Debug, Clone, Copy)] pub(crate) enum ClientName { diff --git a/crates/ty_server/src/session/index.rs b/crates/ty_server/src/session/index.rs index 81815a929a..b41730dd37 100644 --- a/crates/ty_server/src/session/index.rs +++ b/crates/ty_server/src/session/index.rs @@ -43,6 +43,16 @@ impl Index { }) } + /// The same documents as [`Self::file_documents`], each with the key it is stored under. + pub(super) fn keyed_file_documents(&self) -> impl Iterator { + self.documents + .iter() + .filter(|(_, document)| match document { + Document::Text(text_document) => text_document.notebook().is_none(), + Document::Notebook(_) => true, + }) + } + pub(crate) fn document_handle( &self, uri: &lsp_types::Uri, diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index 9fd0f83d54..5f366522c8 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -42,6 +42,7 @@ mod initialize; mod injections; mod inlay_hints; mod notebook; +mod project_server; mod publish_diagnostics; mod pull_diagnostics; mod rename; @@ -226,6 +227,7 @@ impl TestServer { capabilities: ClientCapabilities, initialization_options: Option, env_vars: Vec<(String, Option)>, + project_server_directory: Option, ) -> Self { setup_tracing(); @@ -252,7 +254,13 @@ impl TestServer { // TODO: This should probably be configurable to test concurrency issues let worker_threads = NonZeroUsize::new(1).unwrap(); - match Server::new(worker_threads, server_connection, test_system, true) { + match Server::new( + worker_threads, + server_connection, + test_system, + true, + project_server_directory, + ) { Ok(server) => { if let Err(err) = server.run() { panic!("Server stopped with error: {err:?}"); @@ -1279,9 +1287,18 @@ pub(crate) struct TestServerBuilder { initialization_options: Option, client_capabilities: ClientCapabilities, env_vars: Vec<(String, Option)>, + project_server_directory: Option, } impl TestServerBuilder { + /// The environment variables a test server is started without. + /// + /// Removed so that a test's answer does not depend on the machine it runs on — most of + /// all on which interpreter happens to be on the `PATH`. A test that builds a database of + /// its own alongside the server has to remove them too, or the two resolve different + /// environments and the server rightly refuses to answer for it. + pub(crate) const CLEARED_ENV_VARS: &'static [&'static str] = &["HOME", "PATH", "VIRTUAL_ENV"]; + /// Create a new builder pub(crate) fn new() -> Result { // Default client capabilities for the test server: @@ -1311,11 +1328,11 @@ impl TestServerBuilder { test_context: TestContext::new()?, initialization_options: None, client_capabilities, - env_vars: vec![ - ("HOME".into(), None), - ("PATH".into(), None), - ("VIRTUAL_ENV".into(), None), - ], + env_vars: Self::CLEARED_ENV_VARS + .iter() + .map(|name| ((*name).to_string(), None)) + .collect(), + project_server_directory: None, }) } @@ -1611,6 +1628,16 @@ impl TestServerBuilder { Ok(self) } + /// Let `by` command lines reach this server, announcing it in `directory`. + /// + /// Off by default, and never the user's own directory: a test server that published + /// itself there would be found and asked for answers by whatever `by check` the person + /// running the tests has going in another terminal. + pub(crate) fn with_project_server(mut self, directory: &SystemPath) -> Self { + self.project_server_directory = Some(directory.to_path_buf()); + self + } + /// Build the test server pub(crate) fn build(self) -> TestServer { TestServer::new( @@ -1619,6 +1646,7 @@ impl TestServerBuilder { self.client_capabilities, self.initialization_options, self.env_vars, + self.project_server_directory, ) } } diff --git a/crates/ty_server/tests/e2e/project_server.rs b/crates/ty_server/tests/e2e/project_server.rs new file mode 100644 index 0000000000..56021ecd1a --- /dev/null +++ b/crates/ty_server/tests/e2e/project_server.rs @@ -0,0 +1,646 @@ +//! The side channel a `by` command line reaches a running server on. +//! +//! Exercised through a real server, because everything worth testing about it is about the +//! session: which database answers, what the editor's open buffers do to the answer, and +//! whether the configuration and environment the caller resolved are the ones the server did. +//! None of that exists in the library the handler calls. +//! +//! Requests are built out of a real cold [`ProjectDatabase`], the way the `by` command line +//! builds them, rather than by hand. That is the whole point of [`answers_what_a_cold_check +//!_would_have`]: a request assembled to suit the server would agree with the server about +//! anything, including the things it gets wrong. +//! +//! The server under test publishes itself into a temporary directory rather than the user's +//! own, so that a `by check` running elsewhere on this machine never finds it. + +use std::io::{BufRead, BufReader, Write}; +use std::net::{Ipv4Addr, SocketAddr, TcpStream}; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::Result; +use ruff_db::diagnostic::{DisplayDiagnosticConfig, DisplayDiagnostics}; +use ruff_db::system::{OsSystem, SystemPath, TestSystem}; +use ruff_ranged_value::{ValueSource, ValueSourceGuard}; +use tempfile::TempDir; +use ty_project::{Db as _, ProjectDatabase, ProjectMetadata}; +use ty_server::project_server::protocol; +use ty_server::project_server::protocol::{ + Answer, Build, CheckRequest, PROTOCOL, Payload, Refusal, Request, Response, +}; +use ty_server::project_server::{discovery, environment}; +use ty_server::{ClientOptions, DiagnosticMode}; + +use crate::{TestServer, TestServerBuilder}; + +/// A file whose one error is easy to recognise in rendered output. +const MAIN: &str = "def f() -> str:\n return 42\n"; + +/// How long a test will wait for an answer. +/// +/// Bounded so that a change which stops the server answering — a dropped request, a wedged +/// main loop — fails the suite rather than hanging it. +const TIMEOUT: Duration = Duration::from_secs(60); + +/// Where a test server publishes itself: never the directory a real `by check` reads. +fn published(directory: &TempDir) -> &SystemPath { + SystemPath::from_std_path(directory.path()).expect("a temporary directory to be utf-8") +} + +/// A server holding `src`, reachable by a command line, checking the whole project. +/// +/// Workspace diagnostics because that is the precondition: a server diagnosing only what is +/// open has not checked the project and refuses outright — see +/// [`refuses_while_checking_only_open_files`]. +fn server(directory: &TempDir, files: &[(&SystemPath, &str)]) -> Result { + let mut builder = TestServerBuilder::new()? + .with_project_server(published(directory)) + .with_workspace( + SystemPath::new("src"), + Some(ClientOptions::default().with_diagnostic_mode(DiagnosticMode::Workspace)), + )?; + for (path, content) in files { + builder = builder.with_file(path, content)?; + } + Ok(builder.build().wait_until_workspaces_are_initialized()) +} + +/// The database a `by check` in `project_root` would build for itself. +fn cold(project_root: &SystemPath) -> Result { + cold_with(project_root, None) +} + +/// The same, for a `by check` that was given flags. +/// +/// The flags arrive as a layer over the configuration file, the way `--python-version` and +/// the rest do, rather than by editing the file: the layering is exactly what this has to +/// exercise. +fn cold_with(project_root: &SystemPath, overrides: Option<&str>) -> Result { + // the same environment the server under test was given. an interpreter discovered out of + // one process's environment and not the other's is a real disagreement, and one this + // refuses over — see [`refuses_an_environment_it_does_not_share`] — so a test that wants + // an answer has to stand where the server stands + let system = TestSystem::new(OsSystem::new(project_root)); + for name in TestServerBuilder::CLEARED_ENV_VARS { + system.remove_env_var(*name); + } + + let mut metadata = ProjectMetadata::discover(project_root, &system)?; + metadata.apply_configuration_files(&system)?; + if let Some(overrides) = overrides { + // the same source a flag's value carries, which is what decides whether a range is + // expected alongside it + let _guard = ValueSourceGuard::new(ValueSource::Cli, false); + metadata.apply_override_options(serde_json::from_str(overrides)?); + } + + let mut db = ProjectDatabase::fallible(metadata, system)?; + db.set_checker(Arc::new(ty_ide::DjangoChecker)); + Ok(db) +} + +/// The request that command line would send, resolved from its own database. +fn request_for(db: &ProjectDatabase) -> Result { + let project = db.project(); + Ok(CheckRequest { + project_root: project.root(db).to_path_buf(), + working_directory: project.root(db).to_path_buf(), + options: protocol::configuration(project.metadata(db).to_merged_options().options())?, + environment: environment(db), + force_exclude: project.force_exclude(db), + verbose: project.verbose(db), + color: false, + }) +} + +/// One round trip on the side channel, bypassing the client so that a test can send something +/// a client never would. +fn ask( + directory: &TempDir, + project_root: &SystemPath, + request: impl FnOnce(&discovery::Record) -> Result, +) -> Result> { + let candidates = discovery::candidates(published(directory), project_root); + let Some(record) = candidates.first().map(|candidate| &candidate.record) else { + anyhow::bail!("the server published no record covering `{project_root}`"); + }; + + let mut connection = TcpStream::connect(SocketAddr::from((Ipv4Addr::LOCALHOST, record.port)))?; + connection.set_read_timeout(Some(TIMEOUT))?; + connection.set_write_timeout(Some(TIMEOUT))?; + + let mut line = serde_json::to_vec(&request(record)?)?; + line.push(b'\n'); + connection.write_all(&line)?; + connection.flush()?; + + let mut response = String::new(); + BufReader::new(&connection).read_line(&mut response)?; + if response.is_empty() { + return Ok(None); + } + + let answer: Answer = serde_json::from_str(&response)?; + // the caller has no way to know it is talking to the server unless the server proves it + // read the record, so every test asserts it along the way + assert_eq!(answer.token, record.token); + Ok(Some(answer.response)) +} + +/// A well-formed request, which the tests below then spoil one field at a time. +fn check_request(record: &discovery::Record, db: &ProjectDatabase) -> Result { + Ok(Request { + protocol: PROTOCOL, + token: record.token.clone(), + client: record.build.clone(), + payload: serde_json::to_value(Payload::Check(request_for(db)?))?, + }) +} + +/// Asks the usual way, from a database built the way the command line builds one. +fn check(directory: &TempDir, db: &ProjectDatabase) -> Result> { + ask(directory, db.project().root(db), |record| { + check_request(record, db) + }) +} + +/// The claim the whole design rests on: what the server answers is what this process would +/// have produced on its own, character for character. +#[test] +fn answers_what_a_cold_check_would_have() -> Result<()> { + let directory = TempDir::new()?; + let server = server( + &directory, + &[ + (SystemPath::new("src/main.py"), MAIN), + ( + SystemPath::new("src/other.py"), + "import collections\n\nx: int = collections\n", + ), + ], + )?; + + let db = cold(&server.file_path(SystemPath::new("src")))?; + let Some(Response::Check(response)) = check(&directory, &db)? else { + panic!("the server did not answer"); + }; + + let diagnostics = db.check(); + let config = DisplayDiagnosticConfig::new("ty") + .format(db.project().settings(&db).terminal().output_format.into()) + .color(false) + .context(0); + let expected = DisplayDiagnostics::new(&db, &config, &diagnostics).to_string(); + + assert_eq!(response.rendered, expected); + assert_eq!(response.diagnostics, diagnostics.len()); + assert!(response.human_readable); + assert!(!response.empty_project); + + Ok(()) +} + +/// A change made behind the server's back is a change the answer has to include. The editor's +/// watcher is what feeds this session, and nothing tells it about a file written by a `git +/// checkout`, a code generator, or this test. +#[test] +fn answers_from_the_file_system_rather_than_from_the_watcher() -> Result<()> { + let directory = TempDir::new()?; + let server = server(&directory, &[(SystemPath::new("src/main.py"), MAIN)])?; + let project_root = server.file_path(SystemPath::new("src")); + + // no `didChangeWatchedFiles`, because the test client is not watching anything + std::fs::write(project_root.join("added.py").as_std_path(), "y: str = 1\n")?; + + let db = cold(&project_root)?; + let Some(Response::Check(response)) = check(&directory, &db)? else { + panic!("the server did not answer"); + }; + + assert!( + response.rendered.contains("added.py"), + "the answer did not see a file written behind the server's back:\n{}", + response.rendered + ); + assert_eq!(response.diagnostics, db.check().len()); + + Ok(()) +} + +/// A flag on the command line lands in the merged configuration, and moves what the check +/// reports. Comparing the configuration file alone would compare the one thing that could +/// never have differed. +#[test] +fn refuses_a_flag_that_changes_the_answer() -> Result<()> { + let directory = TempDir::new()?; + let server = server(&directory, &[(SystemPath::new("src/main.py"), MAIN)])?; + + let db = cold_with( + &server.file_path(SystemPath::new("src")), + Some(r#"{"rules": {"invalid-return-type": "ignore"}}"#), + )?; + + let response = check(&directory, &db)?; + assert!( + matches!( + response, + Some(Response::Refused { + reason: Refusal::Options { .. } + }) + ), + "expected a refusal over the configuration, got {response:?}" + ); + + Ok(()) +} + +/// Two processes that agree about every option can still be checking against different +/// site-packages: an environment is discovered as much as configured, and a server discovers +/// it from the editor's environment rather than the caller's shell. +#[test] +fn refuses_an_environment_it_does_not_share() -> Result<()> { + let directory = TempDir::new()?; + let server = server(&directory, &[(SystemPath::new("src/main.py"), MAIN)])?; + let db = cold(&server.file_path(SystemPath::new("src")))?; + + let response = ask(&directory, db.project().root(&db), |record| { + let mut request = request_for(&db)?; + request.environment.push_str(" (somewhere else)"); + Ok(Request { + protocol: PROTOCOL, + token: record.token.clone(), + client: record.build.clone(), + payload: serde_json::to_value(Payload::Check(request))?, + }) + })?; + + assert!( + matches!( + response, + Some(Response::Refused { + reason: Refusal::Environment { .. } + }) + ), + "expected a refusal over the environment, got {response:?}" + ); + + Ok(()) +} + +/// A verbose check adds a note to every diagnostic saying where its rule was turned on, and a +/// server's database never does. Refused rather than declined to ask, so that the reason is +/// in the log where somebody looking for it can find it. +#[test] +fn refuses_a_verbose_check() -> Result<()> { + let directory = TempDir::new()?; + let server = server(&directory, &[(SystemPath::new("src/main.py"), MAIN)])?; + let db = cold(&server.file_path(SystemPath::new("src")))?; + + let response = ask(&directory, db.project().root(&db), |record| { + let mut request = request_for(&db)?; + request.verbose = true; + Ok(Request { + protocol: PROTOCOL, + token: record.token.clone(), + client: record.build.clone(), + payload: serde_json::to_value(Payload::Check(request))?, + }) + })?; + + assert!( + matches!( + response, + Some(Response::Refused { + reason: Refusal::Verbose + }) + ), + "expected a refusal over verbosity, got {response:?}" + ); + + Ok(()) +} + +/// The editor is holding text that is not in the file, so the server is checking a program +/// the caller cannot see. +#[test] +fn refuses_while_a_buffer_is_unsaved() -> Result<()> { + let directory = TempDir::new()?; + let main = SystemPath::new("src/main.py"); + let mut server = server(&directory, &[(main, MAIN)])?; + + server.open_text_document(main, "x: int = 1\n", 1); + + let db = cold(&server.file_path(SystemPath::new("src")))?; + match check(&directory, &db)? { + Some(Response::Refused { + reason: Refusal::Unsaved { files }, + }) => assert_eq!(files, vec!["main.py".to_string()]), + other => panic!("expected a refusal over the open buffer, got {other:?}"), + } + + Ok(()) +} + +/// A notebook the editor is holding carries no text to compare against the file, because what +/// is on disk is a serialization whose formatting is the writer's choice. Refused, rather than +/// skipped for want of a way to compare it. +#[test] +fn refuses_while_a_notebook_is_open() -> Result<()> { + let directory = TempDir::new()?; + let notebook = SystemPath::new("src/notes.ipynb"); + let mut server = server( + &directory, + &[ + (SystemPath::new("src/main.py"), MAIN), + (notebook, EMPTY_NOTEBOOK), + ], + )?; + + server.send_notification::( + lsp_types::DidOpenNotebookDocumentParams { + notebook_document: lsp_types::NotebookDocument { + uri: server.file_uri(notebook), + notebook_type: "jupyter-notebook".to_string(), + version: 0, + metadata: None, + cells: Vec::new(), + }, + cell_text_documents: Vec::new(), + }, + ); + + let db = cold(&server.file_path(SystemPath::new("src")))?; + match check(&directory, &db)? { + Some(Response::Refused { + reason: Refusal::Unsaved { files }, + }) => assert_eq!(files, vec!["notes.ipynb".to_string()]), + other => panic!("expected a refusal over the open notebook, got {other:?}"), + } + + Ok(()) +} + +/// An open document that still matches its file is not a reason to refuse. Otherwise the +/// feature would be unavailable to anyone who had the project open, which is everyone it is +/// for. +#[test] +fn answers_while_a_buffer_is_open_and_saved() -> Result<()> { + let directory = TempDir::new()?; + let main = SystemPath::new("src/main.py"); + let mut server = server(&directory, &[(main, MAIN)])?; + + server.open_text_document(main, MAIN, 1); + + let db = cold(&server.file_path(SystemPath::new("src")))?; + let response = check(&directory, &db)?; + assert!( + matches!(response, Some(Response::Check(_))), + "an open, saved buffer should not stop the server answering, got {response:?}\nclient: {}", + environment(&db) + ); + + Ok(()) +} + +/// The default editor setting diagnoses only what is open. A server in that mode has not +/// checked the project and must not answer as though it had. +#[test] +fn refuses_while_checking_only_open_files() -> Result<()> { + let directory = TempDir::new()?; + let server = TestServerBuilder::new()? + .with_project_server(published(&directory)) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/main.py"), MAIN)? + .build() + .wait_until_workspaces_are_initialized(); + + let db = cold(&server.file_path(SystemPath::new("src")))?; + let response = check(&directory, &db)?; + assert!( + matches!( + response, + Some(Response::Refused { + reason: Refusal::CheckMode + }) + ), + "expected a refusal over the check mode, got {response:?}" + ); + + Ok(()) +} + +/// A server holds a database per workspace, and is asked about a project rather than about a +/// workspace. One it does not hold, it does not answer for. +#[test] +fn refuses_a_project_it_does_not_hold() -> Result<()> { + let directory = TempDir::new()?; + let server = server(&directory, &[(SystemPath::new("src/main.py"), MAIN)])?; + let db = cold(&server.file_path(SystemPath::new("src")))?; + + let response = ask(&directory, db.project().root(&db), |record| { + let mut request = request_for(&db)?; + request.project_root = request.project_root.join("nested"); + Ok(Request { + protocol: PROTOCOL, + token: record.token.clone(), + client: record.build.clone(), + payload: serde_json::to_value(Payload::Check(request))?, + }) + })?; + + assert!( + matches!( + response, + Some(Response::Refused { + reason: Refusal::UnknownProject + }) + ), + "expected a refusal over the project, got {response:?}" + ); + + Ok(()) +} + +/// Two builds of `by` disagree about a project's diagnostics as readily as they disagree +/// about anything else, and the caller asked for its own build's answer. +#[test] +fn refuses_a_different_build() -> Result<()> { + let directory = TempDir::new()?; + let server = server(&directory, &[(SystemPath::new("src/main.py"), MAIN)])?; + let db = cold(&server.file_path(SystemPath::new("src")))?; + + let response = ask(&directory, db.project().root(&db), |record| { + let mut request = check_request(record, &db)?; + request.client.version = "0.0.0-not-this-one".to_owned(); + Ok(request) + })?; + + assert!( + matches!( + response, + Some(Response::Refused { + reason: Refusal::Version { .. } + }) + ), + "expected a refusal over the build, got {response:?}" + ); + + Ok(()) +} + +/// Two builds of the same commit report the same version — which is the ordinary state of an +/// editor holding a server while its `by` is rebuilt — so the executable settles it. +#[test] +fn refuses_a_rebuild_of_the_same_version() -> Result<()> { + let directory = TempDir::new()?; + let server = server(&directory, &[(SystemPath::new("src/main.py"), MAIN)])?; + let db = cold(&server.file_path(SystemPath::new("src")))?; + + let response = ask(&directory, db.project().root(&db), |record| { + let mut request = check_request(record, &db)?; + if let Some(executable) = request.client.executable.as_mut() { + executable.modified += 1; + } + Ok(request) + })?; + + assert!( + matches!( + response, + Some(Response::Refused { + reason: Refusal::Version { .. } + }) + ), + "expected a refusal over the executable, got {response:?}" + ); + + Ok(()) +} + +/// A protocol the server does not speak is said so, rather than left to fail to parse. +#[test] +fn refuses_a_protocol_it_does_not_speak() -> Result<()> { + let directory = TempDir::new()?; + let server = server(&directory, &[(SystemPath::new("src/main.py"), MAIN)])?; + let db = cold(&server.file_path(SystemPath::new("src")))?; + + let response = ask(&directory, db.project().root(&db), |record| { + let mut request = check_request(record, &db)?; + request.protocol = PROTOCOL + 1; + // a later protocol's payload is not this one's, and must not have to be + request.payload = serde_json::json!({ "kind": "SomethingLater" }); + Ok(request) + })?; + + assert!( + matches!( + response, + Some(Response::Refused { + reason: Refusal::Protocol { .. } + }) + ), + "expected a refusal over the protocol, got {response:?}" + ); + + Ok(()) +} + +/// The port is on loopback, which every user on this machine can reach; the token is what +/// makes reaching it depend on being able to read a file that only this user can. +#[test] +fn says_nothing_without_the_token() -> Result<()> { + let directory = TempDir::new()?; + let server = server(&directory, &[(SystemPath::new("src/main.py"), MAIN)])?; + let db = cold(&server.file_path(SystemPath::new("src")))?; + + let response = ask(&directory, db.project().root(&db), |record| { + let mut request = check_request(record, &db)?; + request.token = "not the token".to_owned(); + Ok(request) + })?; + + assert!( + response.is_none(), + "a request without the token was answered with {response:?}" + ); + + Ok(()) +} + +/// A record only rules a server *in*. Whether the database is really rooted there is the +/// server's to answer, but a server for an unrelated tree is not worth connecting to at all. +#[test] +fn a_record_is_found_only_from_inside_its_roots() -> Result<()> { + let directory = TempDir::new()?; + let server = server(&directory, &[(SystemPath::new("src/main.py"), MAIN)])?; + let published = published(&directory); + let project_root = server.file_path(SystemPath::new("src")); + + assert_eq!(discovery::candidates(published, &project_root).len(), 1); + assert_eq!( + discovery::candidates(published, &project_root.join("nested")).len(), + 1, + "a directory inside a root is inside it" + ); + assert_eq!( + discovery::candidates(published, SystemPath::new("/somewhere/else")).len(), + 0 + ); + + Ok(()) +} + +/// A record whose server has gone is taken away by whoever discovers that, or it would cost +/// every later `by check` a connection attempt for the life of the machine. +/// +/// No server here at all: a record is a file, and this is about what happens to a file whose +/// process is not there any more. +#[test] +fn a_record_outlives_a_server_only_until_somebody_tries_it() -> Result<()> { + let project = TempDir::new()?; + let project_root = SystemPath::from_std_path(project.path()) + .unwrap() + .to_path_buf(); + std::fs::write(project_root.join("main.py").as_std_path(), MAIN)?; + + let directory = TempDir::new()?; + let published = published(&directory); + let build = Build::current(ruff_db::program_version().unwrap_or("test")); + + // a port nothing is listening on: bound to learn a free number, then let go + let port = std::net::TcpListener::bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0)))? + .local_addr()? + .port(); + std::fs::write( + directory.path().join("stale.json"), + serde_json::to_vec(&discovery::Record { + protocol: PROTOCOL, + build: build.clone(), + port, + token: "irrelevant".to_owned(), + roots: vec![project_root.clone()], + })?, + )?; + + let db = cold(&project_root)?; + assert_eq!(discovery::candidates(published, &project_root).len(), 1); + assert!( + ty_server::project_server::client::check(published, request_for(&db)?, &build).is_none(), + "a record whose server is gone should not produce an answer" + ); + assert_eq!( + discovery::candidates(published, &project_root).len(), + 0, + "the stale record should have been taken away" + ); + + Ok(()) +} + +/// The smallest notebook the server will accept as one. +const EMPTY_NOTEBOOK: &str = r#"{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +}"#; diff --git a/crates/ty_static/src/env_vars.rs b/crates/ty_static/src/env_vars.rs index 88c6b4110f..e67a9723e5 100644 --- a/crates/ty_static/src/env_vars.rs +++ b/crates/ty_static/src/env_vars.rs @@ -61,6 +61,16 @@ impl EnvVars { /// Accepts the same values as the `--output-format` command-line argument. pub const TY_OUTPUT_FORMAT: &'static str = "TY_OUTPUT_FORMAT"; + /// Disable the project server: the language server answering `by` command lines. + /// + /// A server started for a project holds it parsed and inferred, and a `by check` in that + /// project asks the server rather than doing the work again. Set this to `1` to switch + /// that off — on a command line, so it checks for itself, and on a server, so it does + /// not open the socket that makes it reachable. + /// + /// Equivalent to the `--no-server` command-line argument. + pub const BY_NO_PROJECT_SERVER: &'static str = "BY_NO_PROJECT_SERVER"; + /// Enable uv integration. /// /// When set to `"1"` or `"true"`, ty invokes `uv workspace metadata` to discover the workspace diff --git a/docs/basedpython/features/index.md b/docs/basedpython/features/index.md index 42b8001d20..aacec24383 100644 --- a/docs/basedpython/features/index.md +++ b/docs/basedpython/features/index.md @@ -45,6 +45,8 @@ features that apply to a project rather than to a file language is written inside it - [linting](linter.md) — the `BY` rules, and how ruff's own rules read `.by` source +- [the project server](project-server.md) — `by check` answered out of the + language server's warm state diff --git a/docs/basedpython/features/project-server.md b/docs/basedpython/features/project-server.md new file mode 100644 index 0000000000..9bad89ec83 --- /dev/null +++ b/docs/basedpython/features/project-server.md @@ -0,0 +1,82 @@ +# the project server + +`by check` asks a running [language server](editor.md) for the project's diagnostics before +checking for itself, and says so when it does: + +```console +$ by check +using project server information +All checks passed! +``` + +a server that has the project open has already parsed and inferred it, and has been keeping +that current ever since. on a project of about twelve thousand files that is the difference +between four seconds and under one + +## what it needs + +the server has to be checking the whole project, which is not what an editor asks for by +default — the default is only the files you have open. a server in that mode has never looked +at the rest of the project, so it has no answer to give and does not pretend to: + +```json +{ "ty.diagnosticMode": "workspace" } +``` + +this is the same setting that decides whether errors in files you have not opened appear in +your editor's problems list, so it is one you want anyway if you want this + +## it is the same answer, or it is no answer + +what comes back is what your own process would have computed. the server refuses whenever +that might not be true, and a refusal is not a failure — the check simply runs the way it +always did, and you see no message + +it refuses when the two are different builds of `by`, when they resolved different +configuration, when they resolved different environments, when a file open in the editor has +unsaved changes, and when the session is being typed into so fast that the check keeps being +cancelled + +**any flag that changes the check is a different check.** `--python-version`, +`--error-on-warning`, `--output-format` and the rest all land in the configuration the two +compare, so a `by check` carrying one of them is always run in full. so is a `-v` check, whose +diagnostics carry an explanation of where each rule was turned on that a server's do not. and +`by check` does not ask at all when the invocation is not a whole-project check: `--watch`, +`--fix`, `--add-ignore`, or a check pointed at particular paths + +the answer describes the project as it is on disk, not as your editor last noticed it — a +`git checkout` your editor missed does not go unseen + +### finding out which one it was + +every refusal is logged. `-v` will not show you, because a verbose check is itself a reason to +refuse; ask for the log directly instead: + +```console +$ TY_LOG=ty=debug by check +``` + +## turning it off + +`--no-server` checks from scratch: + +```console +$ by check --no-server +``` + +`BY_NO_PROJECT_SERVER=1` does the same, and does it on both halves: a command line with it set +checks for itself, and a server started with it set does not open the socket that makes it +reachable at all + +```console +$ BY_NO_PROJECT_SERVER=1 by check +``` + +## what it opens + +a server listens on a loopback port and writes a record of itself — the port and a random +secret — into a per-user directory under ty's cache, readable only by you. a request without +the secret is not answered, and an answer that does not repeat the secret back is not believed + +the record is removed when the server shuts down. one whose server was killed outright stays +behind until the next `by check` finds nothing listening and clears it away diff --git a/scripts/check_project_server.py b/scripts/check_project_server.py new file mode 100644 index 0000000000..c35ccce258 --- /dev/null +++ b/scripts/check_project_server.py @@ -0,0 +1,293 @@ +# /// script +# requires-python = ">=3.11" +# /// +"""Check that `by check` gives the same answer through a running server as it does alone. + +Everything else that covers the project server runs the server and the caller in one +process, where the two share a build, a working directory and a view of the file system by +construction. This runs them the way a user does — a real `by server` over stdin and stdout, +a real `by check` finding it — and compares the two answers byte for byte. + + uv run --no-project scripts/check_project_server.py path/to/by [--modules N] + +`--modules` sizes the generated project, which is also what makes the timings mean anything: +the point of the feature is a project big enough that checking it twice is worth avoiding. +""" + +from __future__ import annotations + +import argparse +import json +import os +import queue +import shutil +import subprocess +import sys +import tempfile +import threading +import time +from pathlib import Path +from typing import Any + + +class Server: + """A `by server` on the other end of a pipe, with its output drained. + + Draining is not optional. This asks for pushed diagnostics over a project of a few + thousand files, which is more than a pipe holds — a client that only reads when it wants + something blocks the server inside a write and never gets there. + """ + + def __init__(self, by: Path, root: Path): + self.process = subprocess.Popen( + [by, "server"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + cwd=root, + ) + # `Popen` types the pipes as optional because asking for them is optional; they + # were asked for just above, so bind them once rather than at every use + stdin, stdout = self.process.stdin, self.process.stdout + if stdin is None or stdout is None: + raise RuntimeError("the server started without the pipes it was given") + self.stdin, self.stdout = stdin, stdout + self.messages: queue.Queue[dict[str, Any]] = queue.Queue() + self.reader = threading.Thread(target=self._read_forever, daemon=True) + self.reader.start() + + def _read_forever(self): + while True: + length = None + while True: + line = self.stdout.readline() + if not line: + return + line = line.strip() + if not line: + break + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":")[1]) + if length is None: + return + self.messages.put(json.loads(self.stdout.read(length))) + + def send(self, message: dict[str, Any]): + body = json.dumps(message).encode() + self.stdin.write(f"Content-Length: {len(body)}\r\n\r\n".encode() + body) + self.stdin.flush() + + def await_message(self, matches, timeout: float) -> dict[str, Any] | None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + message = self.messages.get(timeout=deadline - time.monotonic()) + except queue.Empty: + return None + # the server asks the client for things during startup, and a request it never + # gets an answer to leaves it waiting rather than working + if message.get("method") == "workspace/configuration": + self.send( + { + "jsonrpc": "2.0", + "id": message["id"], + "result": [None] * len(message["params"]["items"]), + } + ) + elif "id" in message and "method" in message: + self.send({"jsonrpc": "2.0", "id": message["id"], "result": None}) + if matches(message): + return message + return None + + +def write_project(root: Path, modules: int): + """A project with one error in it, and `modules` files of work around that error. + + The work has to be work. A few thousand files that each declare a dataclass and add up a + list infer almost instantly, and a measurement over those measures process startup — the + part this feature cannot save. What makes a real check slow is inference that has + somewhere to go: generics that get solved, overloads that get picked between, and long + chains where each step's type depends on the last. + """ + (root / "pyproject.toml").write_text('[project]\nname = "sample"\nversion = "0"\n') + (root / "main.py").write_text("def f() -> str:\n return 42\n") + for index in range(modules): + previous = ( + f"from module_{index - 1} import pipeline as previous\n" if index else "" + ) + seed = "previous(rows)" if index else "rows" + (root / f"module_{index}.py").write_text( + f"from collections.abc import Callable, Iterable, Mapping, Sequence\n" + f"from dataclasses import dataclass, field\n" + f"from typing import Generic, TypeVar, overload\n" + f"{previous}\n" + f"T = TypeVar('T')\n" + f"U = TypeVar('U')\n" + f"\n" + f"@dataclass\n" + f"class Row{index}(Generic[T]):\n" + f" key: str\n" + f" value: T\n" + f" tags: dict[str, list[tuple[int, str]]] = field(default_factory=dict)\n" + f"\n" + f"class Box{index}(Generic[T]):\n" + f" def __init__(self, inner: T) -> None:\n" + f" self.inner = inner\n" + f"\n" + f" def map(self, f: Callable[[T], U]) -> 'Box{index}[U]':\n" + f" return Box{index}(f(self.inner))\n" + f"\n" + f"@overload\n" + f"def widen(value: int) -> float: ...\n" + f"@overload\n" + f"def widen(value: str) -> str: ...\n" + f"def widen(value: int | str) -> float | str:\n" + f" return value + 0 if isinstance(value, int) else value\n" + f"\n" + f"def pipeline(rows: Sequence[Row{index}[int]]) -> Sequence[Row{index}[int]]:\n" + f" seeded = {seed}\n" + f" boxed = [Box{index}(row.value).map(widen).map(str).map(len) for row in seeded]\n" + f" grouped: Mapping[str, list[int]] = {{\n" + f" row.key: [box.inner for box in boxed] for row in seeded\n" + f" }}\n" + f" ordered: Iterable[tuple[str, list[int]]] = sorted(\n" + f" grouped.items(), key=lambda pair: (len(pair[1]), pair[0])\n" + f" )\n" + f" return [Row{index}(key, sum(values)) for key, values in ordered]\n" + ) + + +def run_check( + by: Path, root: Path, *args: str, env: dict[str, str] | None = None +) -> tuple[float, subprocess.CompletedProcess[str]]: + # a clean switch position rather than whatever the caller exported, so that a developer + # with the kill switch set in their shell does not get a confusing failure here + environment = {**os.environ, "BY_NO_PROJECT_SERVER": "0", **(env or {})} + started = time.monotonic() + done = subprocess.run( + [by, "check", *args], cwd=root, capture_output=True, text=True, env=environment + ) + return time.monotonic() - started, done + + +USED_THE_SERVER = "using project server information" + +# every one of these changes what a check reports or how it reports it, and none of them is +# something a server was asked to do — so each has to send the caller back to checking for +# itself rather than being quietly ignored +FLAGS_THAT_MUST_REFUSE = [ + ("--python-version", "3.9"), + ("--error-on-warning",), + ("--output-format", "concise"), + ("--no-server",), + ("-vv",), +] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("by", type=Path, help="the `by` binary to exercise") + parser.add_argument("--modules", type=int, default=200) + parser.add_argument( + "--project", + type=Path, + help="an existing project to measure instead of a generated one. a real project is " + "the only corpus that says anything about the timings: what this feature saves is " + "inference, and generated files have almost none to save", + ) + args = parser.parse_args() + + by = args.by.resolve() + generated = args.project is None + if generated: + root = Path(tempfile.mkdtemp(prefix="by-project-server-")) + write_project(root, args.modules) + else: + root = args.project.resolve() + + server = Server(by, root) + try: + server.send( + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "processId": os.getpid(), + # pull diagnostics, which is how an editor in workspace mode asks a + # server to check a project. without it the server checks nothing, and a + # measurement against it measures the server doing the whole check inside + # the request — which is the thing this is supposed to avoid + "capabilities": { + "textDocument": {"diagnostic": {}}, + "workspace": {"diagnostics": {"refreshSupport": False}}, + }, + "workspaceFolders": [{"uri": root.as_uri(), "name": "sample"}], + # the server has to be checking the whole project before it can answer for + # one, and this is the setting that decides that + "initializationOptions": {"diagnosticMode": "workspace"}, + }, + } + ) + _ = server.await_message(lambda message: message.get("id") == 1, timeout=30) + server.send({"jsonrpc": "2.0", "method": "initialized", "params": {}}) + + warming = time.monotonic() + server.send( + { + "jsonrpc": "2.0", + "id": 3, + "method": "workspace/diagnostic", + "params": {"previousResultIds": []}, + } + ) + if server.await_message(lambda m: m.get("id") == 3, timeout=900) is None: + print("the server never answered the workspace diagnostic", file=sys.stderr) + return 1 + print(f"warming the server took {time.monotonic() - warming:.2f}s") + + hot_elapsed, hot = run_check(by, root) + cold_elapsed, cold = run_check(by, root, env={"BY_NO_PROJECT_SERVER": "1"}) + flagged = { + flags: run_check(by, root, *flags)[1] for flags in FLAGS_THAT_MUST_REFUSE + } + finally: + server.send({"jsonrpc": "2.0", "id": 2, "method": "shutdown", "params": None}) + _ = server.await_message(lambda message: message.get("id") == 2, timeout=30) + server.send({"jsonrpc": "2.0", "method": "exit", "params": None}) + server.process.wait(timeout=10) + if generated: + shutil.rmtree(root, ignore_errors=True) + + print(f"hot: {hot_elapsed:6.2f}s exit {hot.returncode}") + print(f"cold: {cold_elapsed:6.2f}s exit {cold.returncode}") + + failures = [] + if USED_THE_SERVER not in hot.stderr: + failures.append( + "the check did not use the server. every reason it might have refused is logged, " + "so run `TY_LOG=ty=debug by check` in the project to see which one it was — not " + f"`-v`, which is itself a reason to refuse.\nstderr was:\n{hot.stderr}" + ) + for flags, done in flagged.items(): + if USED_THE_SERVER in done.stderr: + failures.append( + f"`by check {' '.join(flags)}` was answered by the server, which cannot have " + "resolved the same check" + ) + if hot.stdout != cold.stdout: + failures.append( + f"the answers differ.\nhot:\n{hot.stdout}\ncold:\n{cold.stdout}" + ) + if hot.returncode != cold.returncode: + failures.append( + f"the exit statuses differ: {hot.returncode} hot, {cold.returncode} cold" + ) + + for failure in failures: + print(f"\nFAILED: {failure}", file=sys.stderr) + return 1 if failures else 0 + + +sys.exit(main()) diff --git a/scripts/check_project_server.py.lock b/scripts/check_project_server.py.lock new file mode 100644 index 0000000000..5951180dc2 --- /dev/null +++ b/scripts/check_project_server.py.lock @@ -0,0 +1,15 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[options] +exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. +exclude-newer-span = "P7D" + +[options.exclude-newer-package] +astral-dev-toolchain-hyperfine = "2026-08-27T00:00:00Z" +astral-dev-toolchain-cargo-insta = "2026-08-27T00:00:00Z" +astral-dev-toolchain-cargo-shear = "2026-08-27T00:00:00Z" +astral-dev-toolchain-cargo-codspeed = "2026-08-27T00:00:00Z" +astral-dev-toolchain-cargo-nextest = "2026-08-27T00:00:00Z" +astral-dev-toolchain-cargo-fuzz = "2026-09-01T00:00:00Z" diff --git a/zensical.toml b/zensical.toml index 51f1db1b39..ab4acdcda4 100644 --- a/zensical.toml +++ b/zensical.toml @@ -93,6 +93,7 @@ features = [ { "editor support" = "features/editor.md" }, { "language injection" = "features/language-injection.md" }, { "linting" = "features/linter.md" }, + { "the project server" = "features/project-server.md" }, ] }, { "standard library" = [ { "typeshed improvements" = "features/typeshed.md" }, From 735ebaaf1aacc19d073360d8a1d9178131dd97a9 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 7 Sep 2026 19:28:31 +1000 Subject: [PATCH 3/7] stop buff reporting basedpython type syntax as python code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `or` and `and` inside a type expression spell union and intersection, so the boolean rules have nothing to say about them: `type A = "foo" or "bar"` was SIM222 "use `"foo"` instead of `"foo" or ...`". A bare string in a type position is the literal type, not a forward reference, so UP037 offered to strip the quotes off `c: "c"` and leave a name. A match type's `case` arms are types, so their `:` opened no suite for E701 to report. `is` is a parametric type test rather than identity, and `===` is the spelling that always compares identity — F632 read the two as one, panicked on `1 === 1` because it could not find that operator's token, and offered to rewrite the type test `1 is T` as `1 == T`. E721 told a `.by` file to reach for python's two operators rather than basedpython's. A destructuring binder always binds, so `for Point(x, y) in points` and `def f(Point(x, y): Point)` now bind their captures the way the equivalent python unpacking does instead of as plain assignments F841 reports. A type parameter is not a constant, however the one-letter convention spells it, so `assert T == int` is no longer a yoda condition. ty spelled a `.by` file's types in python syntax whenever a `.py` file that imports it was checked first: the diagnostic that inference cached said `Literal["b"]` rather than `"b"`. The spelling now comes from the file being inferred rather than from whoever asked for the inference, which is what keeps it out of the salsa cache's blind spot. --- .../flake8_bugbear/B007_basedpython.by | 31 +++++++ .../flake8_simplify/SIM222_basedpython.by | 20 +++++ .../test/fixtures/flake8_simplify/SIM300.py | 15 ++++ .../flake8_simplify/SIM300_basedpython.by | 16 ++++ .../fixtures/pycodestyle/E70_basedpython.by | 52 +++++++++++ .../fixtures/pycodestyle/E721_basedpython.by | 11 +++ .../fixtures/pyflakes/F632_basedpython.by | 19 ++++ .../fixtures/pyflakes/F841_basedpython.by | 36 ++++++++ .../fixtures/pyupgrade/UP037_basedpython.by | 20 +++++ .../test/fixtures/ruff/RUF021_basedpython.by | 21 +++++ .../src/checkers/ast/analyze/expression.rs | 6 ++ crates/ruff_linter/src/checkers/ast/mod.rs | 90 ++++++++++++++++--- .../src/rules/flake8_bugbear/mod.rs | 1 + .../rules/unused_loop_control_variable.rs | 63 ++++++++++--- ...-control-variable_B007_basedpython.by.snap | 73 +++++++++++++++ .../src/rules/flake8_simplify/mod.rs | 3 + .../flake8_simplify/rules/yoda_conditions.rs | 40 ++++++--- ..._expr-and-false_SIM222_basedpython.by.snap | 20 +++++ ...s__expr-or-true_SIM222_basedpython.by.snap | 21 +++++ ...yoda-conditions_SIM300_basedpython.by.snap | 19 ++++ .../ruff_linter/src/rules/pycodestyle/mod.rs | 1 + .../pycodestyle/rules/compound_statements.rs | 22 ++++- .../pycodestyle/rules/type_comparison.rs | 21 ++++- ...-on-one-line-colon_E70_basedpython.by.snap | 54 +++++++++++ ...__type-comparison_E721_basedpython.by.snap | 30 +++++++ crates/ruff_linter/src/rules/pyflakes/mod.rs | 1 + .../rules/invalid_literal_comparisons.rs | 69 +++++++++++--- ...tests__is-literal_F632_basedpython.by.snap | 65 ++++++++++++++ ...__unused-variable_F841_basedpython.by.snap | 24 +++++ crates/ruff_linter/src/rules/pyupgrade/mod.rs | 1 + ...yupgrade__tests__UP037_basedpython.by.snap | 4 + crates/ruff_linter/src/rules/ruff/mod.rs | 1 + ...ained-operators_RUF021_basedpython.by.snap | 16 ++++ crates/ruff_python_semantic/src/model.rs | 12 +++ crates/ty_project/src/lib.rs | 37 ++++++++ crates/ty_python_semantic/src/lib.rs | 6 +- .../ty_python_semantic/src/types/display.rs | 29 ++++-- .../src/types/infer/builder.rs | 9 ++ docs/basedpython/features/linter.md | 32 +++++-- 39 files changed, 940 insertions(+), 71 deletions(-) create mode 100644 crates/ruff_linter/resources/test/fixtures/flake8_bugbear/B007_basedpython.by create mode 100644 crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM222_basedpython.by create mode 100644 crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM300_basedpython.by create mode 100644 crates/ruff_linter/resources/test/fixtures/pycodestyle/E721_basedpython.by create mode 100644 crates/ruff_linter/resources/test/fixtures/pyflakes/F632_basedpython.by create mode 100644 crates/ruff_linter/resources/test/fixtures/pyupgrade/UP037_basedpython.by create mode 100644 crates/ruff_linter/resources/test/fixtures/ruff/RUF021_basedpython.by create mode 100644 crates/ruff_linter/src/rules/flake8_bugbear/snapshots/ruff_linter__rules__flake8_bugbear__tests__unused-loop-control-variable_B007_basedpython.by.snap create mode 100644 crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__expr-and-false_SIM222_basedpython.by.snap create mode 100644 crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__expr-or-true_SIM222_basedpython.by.snap create mode 100644 crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__yoda-conditions_SIM300_basedpython.by.snap create mode 100644 crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__type-comparison_E721_basedpython.by.snap create mode 100644 crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__is-literal_F632_basedpython.by.snap create mode 100644 crates/ruff_linter/src/rules/pyupgrade/snapshots/ruff_linter__rules__pyupgrade__tests__UP037_basedpython.by.snap create mode 100644 crates/ruff_linter/src/rules/ruff/snapshots/ruff_linter__rules__ruff__tests__parenthesize-chained-operators_RUF021_basedpython.by.snap diff --git a/crates/ruff_linter/resources/test/fixtures/flake8_bugbear/B007_basedpython.by b/crates/ruff_linter/resources/test/fixtures/flake8_bugbear/B007_basedpython.by new file mode 100644 index 0000000000..df94bfc0c5 --- /dev/null +++ b/crates/ruff_linter/resources/test/fixtures/flake8_bugbear/B007_basedpython.by @@ -0,0 +1,31 @@ +"""a destructuring loop binds its control variables in the header's pattern rather +than in its target, which holds only the synthetic binder the pattern takes apart""" + + +class Point: + __match_args__ = ("x", "y") + x: int + y: int + + +def unused_capture(points: list[Point]) -> None: + for Point(x, y) in points: # B007 on `y` + print(x) + + +def both_used(points: list[Point]) -> None: + for Point(x, y) in points: + print(x, y) + + +# an underscore-prefixed capture says the omission is deliberate, as it does for a +# tuple target +def deliberately_unused(points: list[Point]) -> None: + for Point(x, _y) in points: + print(x) + + +# nesting reaches the captures at every depth +def nested(pairs: list[tuple[Point, Point]]) -> None: + for (Point(a, b), Point(c, d)) in pairs: # B007 on `b`, `c` and `d` + print(a) diff --git a/crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM222_basedpython.by b/crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM222_basedpython.by new file mode 100644 index 0000000000..15ce8cea79 --- /dev/null +++ b/crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM222_basedpython.by @@ -0,0 +1,20 @@ +# `or` and `and` inside a type expression are basedpython's keyword spellings of +# union and intersection, so what looks like boolean logic there is type syntax: +# `A or B` is the type `A | B`, not an expression whose value is always `A` +type Names = "foo" or "bar" + +type Both = int and str + + +def annotated(a: str or bytes, b: "x" or "y") -> int or None: ... + + +def nested(handlers: list[int or str]) -> None: ... + + +# a boolean operator in a value expression is untouched +def values(a: int, b: int) -> None: + if a or True: # SIM222 + pass + if b and False: # SIM223 + pass diff --git a/crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM300.py b/crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM300.py index 0745cc6108..b6c12cdd86 100644 --- a/crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM300.py +++ b/crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM300.py @@ -44,3 +44,18 @@ # https://github.com/astral-sh/ruff/issues/14761 {"": print(1)} == print(2) {0: 1, **print(2)} == print(4) + + +# A type parameter is a name the caller chooses, not a constant, however the +# one-letter convention spells it +def compare_type_param[T](value: T) -> None: + if T == int: + pass + if T > value: + pass + + +class Box[T]: + def check(self) -> None: + if T == int: + pass diff --git a/crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM300_basedpython.by b/crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM300_basedpython.by new file mode 100644 index 0000000000..603b283e4d --- /dev/null +++ b/crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM300_basedpython.by @@ -0,0 +1,16 @@ +# a `type def` computes a type from its type parameters, so its body compares +# those parameters against types — none of which is a constant on the left +type def Widen[X]: + if X <= int: + return int + return str + + +def reified_cell[reified T](x: T) -> None: + assert T == int + + +# a genuine yoda condition still fires +def yoda(age: int) -> None: + if 42 == age: # SIM300 + pass diff --git a/crates/ruff_linter/resources/test/fixtures/pycodestyle/E70_basedpython.by b/crates/ruff_linter/resources/test/fixtures/pycodestyle/E70_basedpython.by index 44ef59d12f..ca0d0fe3b0 100644 --- a/crates/ruff_linter/resources/test/fixtures/pycodestyle/E70_basedpython.by +++ b/crates/ruff_linter/resources/test/fixtures/pycodestyle/E70_basedpython.by @@ -14,3 +14,55 @@ class Members: # a class definition written on one line is still reported class OneLiner: x = 1 + + +# a match type's `case` arms are type expressions, so the `:` in one separates a +# pattern from a type and opens no suite +class A: ... + + +class B: ... + + +type Swap[*Ts] = match *Ts: + case (A, B): (B, A) + case _: () + + +def nested_alias[T]() -> None: + type Widen[X] = match X: + case int: float + case _: X + + # a `match` statement after the alias is a compound statement again + match T: + case int(): pass + case _: pass + + +# a statement-expression `match` really does hold statements +def statement_expression(v: object) -> int: + result = match v: + case int(): 1 + case _: 0 + return result + + +class Holder: + type Inner[X] = match X: + case int: float + case _: X + + def method(self, v: object) -> None: + match v: + case int(): pass + + +def guarded(flag: bool) -> None: + if flag: + type Guarded[X] = match X: + case int: float + case _: X + + match flag: + case True: pass diff --git a/crates/ruff_linter/resources/test/fixtures/pycodestyle/E721_basedpython.by b/crates/ruff_linter/resources/test/fixtures/pycodestyle/E721_basedpython.by new file mode 100644 index 0000000000..8f5f5953c6 --- /dev/null +++ b/crates/ruff_linter/resources/test/fixtures/pycodestyle/E721_basedpython.by @@ -0,0 +1,11 @@ +# `is` is basedpython's isinstance check and `===` its identity comparison, so +# the two operators this rule names are not the ones python's message names +def compare(a: object, b: object) -> None: + if type(a) == type(b): + pass + if type(a) == int: + pass + + +def reified[reified T](x: T) -> None: + assert T == int diff --git a/crates/ruff_linter/resources/test/fixtures/pyflakes/F632_basedpython.by b/crates/ruff_linter/resources/test/fixtures/pyflakes/F632_basedpython.by new file mode 100644 index 0000000000..cb69fb5138 --- /dev/null +++ b/crates/ruff_linter/resources/test/fixtures/pyflakes/F632_basedpython.by @@ -0,0 +1,19 @@ +# basedpython spells identity `===` / `!==`; a plain `is` is a parametric type +# test, which the parser flattens to the same operator. Only the identity +# spellings, and an `is` whose right-hand side is a literal, compare identity +def identity(x: object) -> None: + print(1 === 1) # F632 + print(x !== "a") # F632 + + +def literal_right_hand_side(x: object) -> None: + print(1 is 1) # F632 + print(x is not "a") # F632 + + +def type_tests[T](x: T, xs: list[int]) -> None: + print(1 is T) + print(xs is list[int]) + # this one lowers to `isinstance([], [])`, which raises at runtime — but the + # mistake in it is the target, not the operator, so it is ty's to report + print([] is []) diff --git a/crates/ruff_linter/resources/test/fixtures/pyflakes/F841_basedpython.by b/crates/ruff_linter/resources/test/fixtures/pyflakes/F841_basedpython.by index 57e96f2e30..2e4bd0ebf2 100644 --- a/crates/ruff_linter/resources/test/fixtures/pyflakes/F841_basedpython.by +++ b/crates/ruff_linter/resources/test/fixtures/pyflakes/F841_basedpython.by @@ -45,3 +45,39 @@ def ordinary_capture(value: object) -> int: return 1 case _: return 0 + + +# a destructuring binder always binds, so its captures are the ones the +# equivalent python spelling makes: `for Point(x, y) in points` binds like +# `for x, y in points`, and `let Point(x, y) := p` like `x, y = p`. None of +# those are the plain assignment this rule reports +class Point: + __match_args__ = ("x", "y") + x: int + y: int + + +def destructuring_binders(points: list[Point], p: Point) -> None: + let Point(a, b) := p + print(a) + + for Point(c, d) in points: + print(c) + + with open("f") as Point(e, g): + print(e) + + +def destructuring_parameter(Point(h, i): Point) -> int: + return h + + +# a `match` case and an `if let` bind only when the pattern matches, which is +# what python's own capture pattern does, so an unused capture still fires +def matching_binders(p: Point) -> int: + if let Point(j, k) := p: # F841 on `k` + return j + match p: + case Point(l, m): # F841 on `m` + return l + return 0 diff --git a/crates/ruff_linter/resources/test/fixtures/pyupgrade/UP037_basedpython.by b/crates/ruff_linter/resources/test/fixtures/pyupgrade/UP037_basedpython.by new file mode 100644 index 0000000000..3e394d6bc4 --- /dev/null +++ b/crates/ruff_linter/resources/test/fixtures/pyupgrade/UP037_basedpython.by @@ -0,0 +1,20 @@ +from __future__ import annotations + + +# basedpython reads a bare string in a type position as the literal type +# `Literal["c"]` rather than as a forward reference, so its quotes carry the +# meaning and removing them would name a variable instead +c: "c" = "b" + + +def parameter(mode: "read") -> "write": ... + + +def body() -> None: + local: "held" = "held" + print(local) + + +# a forward reference is written the way every other type is +class Node: + parent: Node | None diff --git a/crates/ruff_linter/resources/test/fixtures/ruff/RUF021_basedpython.by b/crates/ruff_linter/resources/test/fixtures/ruff/RUF021_basedpython.by new file mode 100644 index 0000000000..6f5b554ff6 --- /dev/null +++ b/crates/ruff_linter/resources/test/fixtures/ruff/RUF021_basedpython.by @@ -0,0 +1,21 @@ +# `A and B or C` in a type expression is the intersection of `A` and `B` unioned +# with `C`, which the keywords already spell unambiguously — there is no python +# precedence to make clear with parentheses +class A: ... + + +class B: ... + + +class C: ... + + +def annotated(x: A and B or C) -> None: ... + + +type Alias = A and B or C + + +# a boolean expression in a value position still wants the parentheses +def values(a: bool, b: bool, c: bool) -> bool: + return a and b or c diff --git a/crates/ruff_linter/src/checkers/ast/analyze/expression.rs b/crates/ruff_linter/src/checkers/ast/analyze/expression.rs index 33c3cf36ed..e0a31180b1 100644 --- a/crates/ruff_linter/src/checkers/ast/analyze/expression.rs +++ b/crates/ruff_linter/src/checkers/ast/analyze/expression.rs @@ -1966,6 +1966,12 @@ pub(crate) fn expression(expr: &Expr, checker: &Checker) { } } Expr::BoolOp(bool_op) => { + // basedpython spells union and intersection with `or` and `and` inside a + // type expression, so `A or B` there is the type `A | B` rather than a + // boolean expression. None of the rules below describe that operator + if checker.semantic().in_basedpython_type_expression() { + return; + } if checker.is_rule_enabled(Rule::BooleanChainedComparison) { pylint::rules::boolean_chained_comparison(checker, bool_op); } diff --git a/crates/ruff_linter/src/checkers/ast/mod.rs b/crates/ruff_linter/src/checkers/ast/mod.rs index 40a469e076..87868f759a 100644 --- a/crates/ruff_linter/src/checkers/ast/mod.rs +++ b/crates/ruff_linter/src/checkers/ast/mod.rs @@ -39,7 +39,7 @@ use ruff_python_ast::identifier::Identifier; use ruff_python_ast::name::QualifiedName; use ruff_python_ast::str::Quote; use ruff_python_ast::token::Tokens; -use ruff_python_ast::visitor::{Visitor, walk_except_handler, walk_pattern}; +use ruff_python_ast::visitor::{Visitor, walk_except_handler, walk_pattern, walk_with_item}; use ruff_python_ast::{ self as ast, AnyParameterRef, ArgOrKeyword, Comprehension, ElifElseClause, ExceptHandler, Expr, ExprContext, ExprFString, ExprTString, InterpolatedStringElement, Keyword, MatchCase, @@ -194,6 +194,26 @@ impl ExpectedDocstringKind { } } +/// basedpython: how a pattern's captures are bound, which follows the binder the +/// pattern sits in rather than the pattern itself +/// +/// a `match` case or an `if let` binds only when the pattern matches, which is what +/// python's own capture pattern does. the destructuring binders bind unconditionally, +/// and each one has a python spelling whose bindings they should match: `for Point(x, +/// y) in points` binds `x` and `y` the way `for x, y in points` does, and `let Point(x, +/// y) := p` the way `x, y = p` does +#[derive(Copy, Clone, Debug, Default)] +enum PatternBinder { + /// `case :`, `if let := ...` — a conditional capture + #[default] + Match, + /// `for in ...` — a loop variable + Loop, + /// `let := ...`, `with ... as `, `def f(: T)` — the + /// captures come out of unpacking a value that is always there + Unpacking, +} + pub(crate) struct Checker<'a> { /// The [`Parsed`] output for the source code. parsed: &'a Parsed, @@ -250,6 +270,8 @@ pub(crate) struct Checker<'a> { semantic_checker: SemanticSyntaxChecker, /// Errors collected by the `semantic_checker`. semantic_errors: RefCell>, + /// basedpython: the binder whose pattern is currently being visited + pattern_binder: PatternBinder, context: &'a LintContext<'a>, } @@ -307,6 +329,7 @@ impl<'a> Checker<'a> { target_version, semantic_checker: SemanticSyntaxChecker::new(), semantic_errors: RefCell::default(), + pattern_binder: PatternBinder::default(), context, } } @@ -1703,6 +1726,19 @@ impl<'a> Visitor<'a> for Checker<'a> { self.visit_body(orelse); self.semantic.flags = flags_snapshot; } + // basedpython: `let := ` unpacks the subject, and falls + // through to the `else` block when the pattern does not match + Stmt::Let(ast::StmtLet { + pattern, + value, + orelse, + range: _, + node_index: _, + }) => { + self.visit_expr(value); + self.visit_pattern_binder(PatternBinder::Unpacking, pattern); + self.visit_body(orelse); + } Stmt::For(ast::StmtFor { node_index: _, range: _, @@ -1717,7 +1753,7 @@ impl<'a> Visitor<'a> for Checker<'a> { self.visit_expr(target); // basedpython: a destructuring loop binds the pattern's captures if let Some(pattern) = pattern { - self.visit_pattern(pattern); + self.visit_pattern_binder(PatternBinder::Loop, pattern); } self.visit_body(body); let flags_snapshot = self.semantic.flags; @@ -1739,7 +1775,7 @@ impl<'a> Visitor<'a> for Checker<'a> { // its subject against the pattern rather than testing it if let Some(pattern) = pattern { self.visit_expr(test); - self.visit_pattern(pattern); + self.visit_pattern_binder(PatternBinder::Match, pattern); } else { self.visit_boolean_test(test); } @@ -1822,15 +1858,21 @@ impl<'a> Visitor<'a> for Checker<'a> { && (self.semantic.in_annotation() || self.source_type.is_stub()) { if let Expr::StringLiteral(string_literal) = expr { - self.visit - .string_type_definitions - .push((string_literal, self.semantic.snapshot())); + // basedpython promotes bare string literals in type positions to + // `Literal["..."]` rather than treating them as forward references, + // so there is no annotation inside the quotes to defer + if !self.source_type.is_basedpython() { + self.visit + .string_type_definitions + .push((string_literal, self.semantic.snapshot())); + return; + } } else { self.visit .future_type_definitions .push((expr, self.semantic.snapshot())); + return; } - return; } self.semantic.push_node(expr); @@ -2515,13 +2557,21 @@ impl<'a> Visitor<'a> for Checker<'a> { // basedpython: a destructuring parameter binds its pattern's captures in // the same scope the parameter itself is bound in if let Some(pattern) = parameter.pattern.as_deref() { - self.visit_pattern(pattern); + self.visit_pattern_binder(PatternBinder::Unpacking, pattern); } // Step 4: Analysis analyze::parameter(parameter, self); } + fn visit_with_item(&mut self, with_item: &'a ast::WithItem) { + // basedpython: `with ctx() as Point(x, y):` unpacks the bound value. a with item + // holds no statements, so the binder can stand for the whole walk + let snapshot = std::mem::replace(&mut self.pattern_binder, PatternBinder::Unpacking); + walk_with_item(self, with_item); + self.pattern_binder = snapshot; + } + fn visit_pattern(&mut self, pattern: &'a Pattern) { // Step 1: Binding if let Pattern::MatchAs(ast::PatternMatchAs { @@ -2543,12 +2593,20 @@ impl<'a> Visitor<'a> for Checker<'a> { // `is_basedpython_transpile_resolved_name` makes applies: a name that // matches no variant at all stays an ordinary capture, and ty reports // one whose subject does not accept it - let flags = if self.semantic.is_based_enum_case_name(name.as_str()) { + let mut flags = if self.semantic.is_based_enum_case_name(name.as_str()) { BindingFlags::BASED_ENUM_CASE_NAME } else { BindingFlags::empty() }; - self.add_binding(name, name.range(), BindingKind::Assignment, flags); + let kind = match self.pattern_binder { + PatternBinder::Match => BindingKind::Assignment, + PatternBinder::Loop => BindingKind::LoopVar, + PatternBinder::Unpacking => { + flags |= BindingFlags::UNPACKED_ASSIGNMENT; + BindingKind::Assignment + } + }; + self.add_binding(name, name.range(), kind, flags); } // Step 2: Traversal @@ -2569,7 +2627,7 @@ impl<'a> Visitor<'a> for Checker<'a> { } fn visit_match_case(&mut self, match_case: &'a MatchCase) { - self.visit_pattern(&match_case.pattern); + self.visit_pattern_binder(PatternBinder::Match, &match_case.pattern); if let Some(expr) = &match_case.guard { self.visit_boolean_test(expr); } @@ -2870,12 +2928,20 @@ impl<'a> Checker<'a> { self.semantic.flags = snapshot; } + /// basedpython: visit a [`Pattern`] as the given binder's, so that its captures are + /// bound the way that binder's python spelling binds them + fn visit_pattern_binder(&mut self, binder: PatternBinder, pattern: &'a Pattern) { + let snapshot = std::mem::replace(&mut self.pattern_binder, binder); + self.visit_pattern(pattern); + self.pattern_binder = snapshot; + } + /// Visit an [`ElifElseClause`] fn visit_elif_else_clause(&mut self, clause: &'a ElifElseClause) { if let Some(test) = &clause.test { if let Some(pattern) = &clause.pattern { self.visit_expr(test); - self.visit_pattern(pattern); + self.visit_pattern_binder(PatternBinder::Match, pattern); } else { self.visit_boolean_test(test); } diff --git a/crates/ruff_linter/src/rules/flake8_bugbear/mod.rs b/crates/ruff_linter/src/rules/flake8_bugbear/mod.rs index 7a2cc0413b..06e879914b 100644 --- a/crates/ruff_linter/src/rules/flake8_bugbear/mod.rs +++ b/crates/ruff_linter/src/rules/flake8_bugbear/mod.rs @@ -77,6 +77,7 @@ mod tests { #[test_case(Rule::UnintentionalTypeAnnotation, Path::new("B032.py"))] #[test_case(Rule::UnreliableCallableCheck, Path::new("B004.py"))] #[test_case(Rule::UnusedLoopControlVariable, Path::new("B007.py"))] + #[test_case(Rule::UnusedLoopControlVariable, Path::new("B007_basedpython.by"))] #[test_case(Rule::UselessComparison, Path::new("B015.ipynb"))] #[test_case(Rule::UselessComparison, Path::new("B015.py"))] #[test_case(Rule::UselessContextlibSuppress, Path::new("B022.py"))] diff --git a/crates/ruff_linter/src/rules/flake8_bugbear/rules/unused_loop_control_variable.rs b/crates/ruff_linter/src/rules/flake8_bugbear/rules/unused_loop_control_variable.rs index c896454d8d..20398c64bf 100644 --- a/crates/ruff_linter/src/rules/flake8_bugbear/rules/unused_loop_control_variable.rs +++ b/crates/ruff_linter/src/rules/flake8_bugbear/rules/unused_loop_control_variable.rs @@ -2,9 +2,10 @@ use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast as ast; use ruff_python_ast::helpers; use ruff_python_ast::helpers::{NameFinder, StoredNameFinder}; -use ruff_python_ast::visitor::Visitor; +use ruff_python_ast::visitor::{Visitor, walk_pattern}; use ruff_python_semantic::Binding; -use ruff_text_size::Ranged; +use ruff_text_size::{Ranged, TextRange}; +use rustc_hash::FxHashMap; use crate::checkers::ast::Checker; use crate::codes::Category; @@ -83,10 +84,26 @@ impl Violation for UnusedLoopControlVariable { /// B007 pub(crate) fn unused_loop_control_variable(checker: &Checker, stmt_for: &ast::StmtFor) { - let control_names = { + let control_names: FxHashMap<&str, TextRange> = { let mut finder = StoredNameFinder::default(); finder.visit_expr(stmt_for.target.as_ref()); - finder.names + let mut names: FxHashMap<&str, TextRange> = finder + .names + .into_iter() + .map(|(name, expr)| (name, expr.range())) + .collect(); + + // basedpython: a destructuring loop binds its control variables in the header's + // pattern rather than in its target, which holds only the synthetic binder the + // pattern takes apart. `for Point(x, y) in points` controls `x` and `y` exactly + // as `for x, y in points` does + if let Some(pattern) = &stmt_for.pattern { + let mut finder = PatternCaptureFinder::default(); + finder.visit_pattern(pattern); + names.extend(finder.names); + } + + names }; let used_names = { @@ -101,7 +118,7 @@ pub(crate) fn unused_loop_control_variable(checker: &Checker, stmt_for: &ast::St clippy::iter_over_hash_type, reason = "iteration order does not affect the diagnostics or fixes produced" )] - for (name, expr) in control_names { + for (name, range) in control_names { // Ignore names that are already underscore-prefixed. if checker.settings().ignores_unused_binding(name) { continue; @@ -137,7 +154,7 @@ pub(crate) fn unused_loop_control_variable(checker: &Checker, stmt_for: &ast::St rename: rename.clone(), certainty, }, - expr.range(), + range, ); if certainty == Certainty::Certain { @@ -152,19 +169,43 @@ pub(crate) fn unused_loop_control_variable(checker: &Checker, stmt_for: &ast::St if scope .get_all(name) .map(|binding_id| checker.semantic().binding(binding_id)) - .filter(|binding| binding.start() >= expr.start()) + .filter(|binding| binding.start() >= range.start()) .all(Binding::is_unused) { - diagnostic.set_fix(Fix::unsafe_edit(Edit::range_replacement( - rename, - expr.range(), - ))); + diagnostic.set_fix(Fix::unsafe_edit(Edit::range_replacement(rename, range))); } } } } } +/// A [`Visitor`] that collects the names a pattern captures, with the range of each. +/// +/// basedpython binds a destructuring loop's control variables here rather than in the +/// loop target, so the rule has to read them out of the pattern to see them at all. +#[derive(Default)] +struct PatternCaptureFinder<'a> { + names: Vec<(&'a str, TextRange)>, +} + +impl<'a> Visitor<'a> for PatternCaptureFinder<'a> { + fn visit_pattern(&mut self, pattern: &'a ast::Pattern) { + if let ast::Pattern::MatchAs(ast::PatternMatchAs { + name: Some(name), .. + }) + | ast::Pattern::MatchStar(ast::PatternMatchStar { + name: Some(name), .. + }) + | ast::Pattern::MatchMapping(ast::PatternMatchMapping { + rest: Some(name), .. + }) = pattern + { + self.names.push((name.as_str(), name.range())); + } + walk_pattern(self, pattern); + } +} + #[derive(Debug, Copy, Clone, PartialEq, Eq)] enum Certainty { Certain, diff --git a/crates/ruff_linter/src/rules/flake8_bugbear/snapshots/ruff_linter__rules__flake8_bugbear__tests__unused-loop-control-variable_B007_basedpython.by.snap b/crates/ruff_linter/src/rules/flake8_bugbear/snapshots/ruff_linter__rules__flake8_bugbear__tests__unused-loop-control-variable_B007_basedpython.by.snap new file mode 100644 index 0000000000..3cbc372032 --- /dev/null +++ b/crates/ruff_linter/src/rules/flake8_bugbear/snapshots/ruff_linter__rules__flake8_bugbear__tests__unused-loop-control-variable_B007_basedpython.by.snap @@ -0,0 +1,73 @@ +--- +source: crates/ruff_linter/src/rules/flake8_bugbear/mod.rs +--- +B007 [*] Loop control variable `y` not used within loop body + --> B007_basedpython.by:12:18 + | +11 | def unused_capture(points: list[Point]) -> None: +12 | for Point(x, y) in points: # B007 on `y` + | ^ +13 | print(x) + | +help: Rename unused `y` to `_y` + | +11 | def unused_capture(points: list[Point]) -> None: + - for Point(x, y) in points: # B007 on `y` +12 + for Point(x, _y) in points: # B007 on `y` +13 | print(x) + | +note: This is an unsafe fix and may change runtime behavior + +B007 [*] Loop control variable `b` not used within loop body + --> B007_basedpython.by:30:19 + | +28 | # nesting reaches the captures at every depth +29 | def nested(pairs: list[tuple[Point, Point]]) -> None: +30 | for (Point(a, b), Point(c, d)) in pairs: # B007 on `b`, `c` and `d` + | ^ +31 | print(a) + | +help: Rename unused `b` to `_b` + | +29 | def nested(pairs: list[tuple[Point, Point]]) -> None: + - for (Point(a, b), Point(c, d)) in pairs: # B007 on `b`, `c` and `d` +30 + for (Point(a, _b), Point(c, d)) in pairs: # B007 on `b`, `c` and `d` +31 | print(a) + | +note: This is an unsafe fix and may change runtime behavior + +B007 [*] Loop control variable `c` not used within loop body + --> B007_basedpython.by:30:29 + | +28 | # nesting reaches the captures at every depth +29 | def nested(pairs: list[tuple[Point, Point]]) -> None: +30 | for (Point(a, b), Point(c, d)) in pairs: # B007 on `b`, `c` and `d` + | ^ +31 | print(a) + | +help: Rename unused `c` to `_c` + | +29 | def nested(pairs: list[tuple[Point, Point]]) -> None: + - for (Point(a, b), Point(c, d)) in pairs: # B007 on `b`, `c` and `d` +30 + for (Point(a, b), Point(_c, d)) in pairs: # B007 on `b`, `c` and `d` +31 | print(a) + | +note: This is an unsafe fix and may change runtime behavior + +B007 [*] Loop control variable `d` not used within loop body + --> B007_basedpython.by:30:32 + | +28 | # nesting reaches the captures at every depth +29 | def nested(pairs: list[tuple[Point, Point]]) -> None: +30 | for (Point(a, b), Point(c, d)) in pairs: # B007 on `b`, `c` and `d` + | ^ +31 | print(a) + | +help: Rename unused `d` to `_d` + | +29 | def nested(pairs: list[tuple[Point, Point]]) -> None: + - for (Point(a, b), Point(c, d)) in pairs: # B007 on `b`, `c` and `d` +30 + for (Point(a, b), Point(c, _d)) in pairs: # B007 on `b`, `c` and `d` +31 | print(a) + | +note: This is an unsafe fix and may change runtime behavior diff --git a/crates/ruff_linter/src/rules/flake8_simplify/mod.rs b/crates/ruff_linter/src/rules/flake8_simplify/mod.rs index 209479f8be..6f94b4434e 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/mod.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/mod.rs @@ -49,7 +49,10 @@ mod tests { #[test_case(Rule::ExprOrNotExpr, Path::new("SIM221.py"))] #[test_case(Rule::ExprOrTrue, Path::new("SIM222.py"))] #[test_case(Rule::ExprAndFalse, Path::new("SIM223.py"))] + #[test_case(Rule::ExprOrTrue, Path::new("SIM222_basedpython.by"))] + #[test_case(Rule::ExprAndFalse, Path::new("SIM222_basedpython.by"))] #[test_case(Rule::YodaConditions, Path::new("SIM300.py"))] + #[test_case(Rule::YodaConditions, Path::new("SIM300_basedpython.by"))] #[test_case(Rule::IfElseBlockInsteadOfDictGet, Path::new("SIM401.py"))] #[test_case(Rule::SplitStaticString, Path::new("SIM905.py"))] #[test_case(Rule::DictGetWithNoneDefault, Path::new("SIM910.py"))] diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/yoda_conditions.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/yoda_conditions.rs index 2cf102c870..8cd4af2eec 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/rules/yoda_conditions.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/rules/yoda_conditions.rs @@ -6,6 +6,7 @@ use libcst_native::CompOp; use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast::{self as ast, CmpOp, Expr, UnaryOp}; use ruff_python_codegen::Stylist; +use ruff_python_semantic::SemanticModel; use ruff_python_stdlib::str::{self}; use ruff_text_size::Ranged; @@ -84,48 +85,58 @@ enum ConstantLikelihood { Definitely = 2, } -impl From<&Expr> for ConstantLikelihood { +impl ConstantLikelihood { /// Determine the [`ConstantLikelihood`] of an expression. - fn from(expr: &Expr) -> Self { + fn of(expr: &Expr, semantic: &SemanticModel) -> Self { + let of = |expr: &Expr| ConstantLikelihood::of(expr, semantic); match expr { _ if expr.is_literal_expr() => ConstantLikelihood::Definitely, Expr::Attribute(ast::ExprAttribute { attr, .. }) => { ConstantLikelihood::from_identifier(attr) } - Expr::Name(ast::ExprName { id, .. }) => ConstantLikelihood::from_identifier(id), + Expr::Name(name) => { + // A type parameter is not a constant, however it is spelled: `T` in + // `def f[T](...)` names a type the caller chooses. The one-letter + // convention makes almost every one of them read as `SCREAMING_CASE`, + // so without this a comparison against a type parameter is a Yoda + // condition whose "fix" reverses a perfectly ordinary comparison. + if semantic + .resolve_name(name) + .is_some_and(|id| semantic.binding(id).kind.is_type_param()) + { + ConstantLikelihood::Unlikely + } else { + ConstantLikelihood::from_identifier(&name.id) + } + } Expr::Tuple(tuple) => tuple .iter() - .map(ConstantLikelihood::from) + .map(of) .min() .unwrap_or(ConstantLikelihood::Definitely), Expr::List(list) => list .iter() - .map(ConstantLikelihood::from) + .map(of) .min() .unwrap_or(ConstantLikelihood::Definitely), Expr::Dict(dict) => dict .items .iter() .flat_map(|item| std::iter::once(&item.value).chain(item.key.as_ref())) - .map(ConstantLikelihood::from) + .map(of) .min() .unwrap_or(ConstantLikelihood::Definitely), - Expr::BinOp(ast::ExprBinOp { left, right, .. }) => cmp::min( - ConstantLikelihood::from(&**left), - ConstantLikelihood::from(&**right), - ), + Expr::BinOp(ast::ExprBinOp { left, right, .. }) => cmp::min(of(left), of(right)), Expr::UnaryOp(ast::ExprUnaryOp { op: UnaryOp::UAdd | UnaryOp::USub | UnaryOp::Invert, operand, range: _, node_index: _, - }) => ConstantLikelihood::from(&**operand), + }) => of(operand), _ => ConstantLikelihood::Unlikely, } } -} -impl ConstantLikelihood { /// Determine the [`ConstantLikelihood`] of an identifier. fn from_identifier(identifier: &str) -> Self { if str::is_cased_uppercase(identifier) { @@ -223,7 +234,8 @@ pub(crate) fn yoda_conditions( return; } - if ConstantLikelihood::from(left) <= ConstantLikelihood::from(right) { + let semantic = checker.semantic(); + if ConstantLikelihood::of(left, semantic) <= ConstantLikelihood::of(right, semantic) { return; } diff --git a/crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__expr-and-false_SIM222_basedpython.by.snap b/crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__expr-and-false_SIM222_basedpython.by.snap new file mode 100644 index 0000000000..1add375dbe --- /dev/null +++ b/crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__expr-and-false_SIM222_basedpython.by.snap @@ -0,0 +1,20 @@ +--- +source: crates/ruff_linter/src/rules/flake8_simplify/mod.rs +--- +SIM223 [*] Use `False` instead of `... and False` + --> SIM222_basedpython.by:19:8 + | +17 | if a or True: # SIM222 +18 | pass +19 | if b and False: # SIM223 + | ^^^^^^^^^^^ +20 | pass + | +help: Replace with `False` + | +18 | pass + - if b and False: # SIM223 +19 + if False: # SIM223 +20 | pass + | +note: This is an unsafe fix and may change runtime behavior diff --git a/crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__expr-or-true_SIM222_basedpython.by.snap b/crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__expr-or-true_SIM222_basedpython.by.snap new file mode 100644 index 0000000000..a09f0910fe --- /dev/null +++ b/crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__expr-or-true_SIM222_basedpython.by.snap @@ -0,0 +1,21 @@ +--- +source: crates/ruff_linter/src/rules/flake8_simplify/mod.rs +--- +SIM222 [*] Use `True` instead of `... or True` + --> SIM222_basedpython.by:17:8 + | +15 | # a boolean operator in a value expression is untouched +16 | def values(a: int, b: int) -> None: +17 | if a or True: # SIM222 + | ^^^^^^^^^ +18 | pass +19 | if b and False: # SIM223 + | +help: Replace with `True` + | +16 | def values(a: int, b: int) -> None: + - if a or True: # SIM222 +17 + if True: # SIM222 +18 | pass + | +note: This is an unsafe fix and may change runtime behavior diff --git a/crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__yoda-conditions_SIM300_basedpython.by.snap b/crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__yoda-conditions_SIM300_basedpython.by.snap new file mode 100644 index 0000000000..08d2702906 --- /dev/null +++ b/crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__yoda-conditions_SIM300_basedpython.by.snap @@ -0,0 +1,19 @@ +--- +source: crates/ruff_linter/src/rules/flake8_simplify/mod.rs +--- +SIM300 [*] Yoda condition detected + --> SIM300_basedpython.by:15:8 + | +13 | # a genuine yoda condition still fires +14 | def yoda(age: int) -> None: +15 | if 42 == age: # SIM300 + | ^^^^^^^^^ +16 | pass + | +help: Rewrite as `age == 42` + | +14 | def yoda(age: int) -> None: + - if 42 == age: # SIM300 +15 + if age == 42: # SIM300 +16 | pass + | diff --git a/crates/ruff_linter/src/rules/pycodestyle/mod.rs b/crates/ruff_linter/src/rules/pycodestyle/mod.rs index 937a024266..89f5afe288 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/mod.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/mod.rs @@ -67,6 +67,7 @@ mod tests { #[test_case(Rule::TrailingWhitespace, Path::new("W291.py"))] #[test_case(Rule::TrueFalseComparison, Path::new("E712.py"))] #[test_case(Rule::TypeComparison, Path::new("E721.py"))] + #[test_case(Rule::TypeComparison, Path::new("E721_basedpython.by"))] #[test_case(Rule::UselessSemicolon, Path::new("E70.py"))] #[test_case(Rule::UselessSemicolon, Path::new("E703.ipynb"))] #[test_case(Rule::WhitespaceAfterDecorator, Path::new("E204.py"))] diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/compound_statements.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/compound_statements.rs index dc9968d024..6c39e4e4b4 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/compound_statements.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/compound_statements.rs @@ -130,6 +130,14 @@ pub(crate) fn compound_statements( // This is used to allow `class C: ...`-style definitions in stubs. let mut allow_ellipsis = false; + // basedpython: `type Swap[*Ts] = match *Ts:` is a type whose `case` arms are type + // expressions rather than statements, so the `:` in `case (A, B): (B, A)` separates + // a pattern from a type and opens no suite. `type_match` holds the indent of the + // header line for as long as the arms it opened last, and `type_keyword` records the + // `type` that tells such a header from `x = match ...`, which really is a statement + let mut type_match: Option = None; + let mut type_keyword = false; + // Track indentation. let mut indent = 0u32; @@ -147,6 +155,9 @@ pub(crate) fn compound_statements( } TokenKind::Dedent => { indent = indent.saturating_sub(1); + if type_match.is_some_and(|header| indent <= header) { + type_match = None; + } } _ => {} } @@ -194,6 +205,7 @@ pub(crate) fn compound_statements( try_ = None; while_ = None; with = None; + type_keyword = false; } TokenKind::Colon => { if case.is_some() @@ -273,7 +285,12 @@ pub(crate) fn compound_statements( with = None; } TokenKind::Case => { - case = Some(token.range()); + if type_match.is_none() { + case = Some(token.range()); + } + } + TokenKind::Type => { + type_keyword = true; } TokenKind::If => { if_ = Some(token.range()); @@ -313,6 +330,9 @@ pub(crate) fn compound_statements( } TokenKind::Match => { match_ = Some(token.range()); + if source_type.is_basedpython() && type_keyword { + type_match = Some(indent); + } } _ => {} } diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/type_comparison.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/type_comparison.rs index 2835593119..e889c2dc7a 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/type_comparison.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/type_comparison.rs @@ -51,13 +51,21 @@ use crate::codes::Category; /// ``` #[derive(ViolationMetadata)] #[violation_metadata(stable_since = "v0.0.39", category = Category::Pedantic)] -pub(crate) struct TypeComparison; +pub(crate) struct TypeComparison { + basedpython: bool, +} impl Violation for TypeComparison { #[derive_message_formats] fn message(&self) -> String { - "Use `is` and `is not` for type comparisons, or `isinstance()` for isinstance checks" - .to_string() + // basedpython spells identity `===` and an isinstance check `x is C`, so the + // python wording names the wrong two operators there + if self.basedpython { + "Use `===` and `!==` for type comparisons, or `is` for isinstance checks".to_string() + } else { + "Use `is` and `is not` for type comparisons, or `isinstance()` for isinstance checks" + .to_string() + } } } @@ -78,7 +86,12 @@ pub(crate) fn type_comparison(checker: &Checker, compare: &ast::ExprCompare) { } // Disallow the comparison. - checker.report_diagnostic(TypeComparison, compare.range()); + checker.report_diagnostic( + TypeComparison { + basedpython: checker.source_type.is_basedpython(), + }, + compare.range(), + ); } } } diff --git a/crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__multiple-statements-on-one-line-colon_E70_basedpython.by.snap b/crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__multiple-statements-on-one-line-colon_E70_basedpython.by.snap index 11aae13b0f..4f8f16da18 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__multiple-statements-on-one-line-colon_E70_basedpython.by.snap +++ b/crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__multiple-statements-on-one-line-colon_E70_basedpython.by.snap @@ -7,3 +7,57 @@ E701 Multiple statements on one line (colon) 15 | # a class definition written on one line is still reported 16 | class OneLiner: x = 1 | ^ + +E701 Multiple statements on one line (colon) + --> E70_basedpython.by:39:19 + | +37 | # a `match` statement after the alias is a compound statement again +38 | match T: +39 | case int(): pass + | ^ +40 | case _: pass + | + +E701 Multiple statements on one line (colon) + --> E70_basedpython.by:40:15 + | +38 | match T: +39 | case int(): pass +40 | case _: pass + | ^ + +E701 Multiple statements on one line (colon) + --> E70_basedpython.by:46:19 + | +44 | def statement_expression(v: object) -> int: +45 | result = match v: +46 | case int(): 1 + | ^ +47 | case _: 0 +48 | return result + | + +E701 Multiple statements on one line (colon) + --> E70_basedpython.by:47:15 + | +45 | result = match v: +46 | case int(): 1 +47 | case _: 0 + | ^ +48 | return result + | + +E701 Multiple statements on one line (colon) + --> E70_basedpython.by:58:23 + | +56 | def method(self, v: object) -> None: +57 | match v: +58 | case int(): pass + | ^ + +E701 Multiple statements on one line (colon) + --> E70_basedpython.by:68:18 + | +67 | match flag: +68 | case True: pass + | ^ diff --git a/crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__type-comparison_E721_basedpython.by.snap b/crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__type-comparison_E721_basedpython.by.snap new file mode 100644 index 0000000000..51126e3190 --- /dev/null +++ b/crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__type-comparison_E721_basedpython.by.snap @@ -0,0 +1,30 @@ +--- +source: crates/ruff_linter/src/rules/pycodestyle/mod.rs +--- +E721 Use `===` and `!==` for type comparisons, or `is` for isinstance checks + --> E721_basedpython.by:4:8 + | +2 | # the two operators this rule names are not the ones python's message names +3 | def compare(a: object, b: object) -> None: +4 | if type(a) == type(b): + | ^^^^^^^^^^^^^^^^^^ +5 | pass +6 | if type(a) == int: + | + +E721 Use `===` and `!==` for type comparisons, or `is` for isinstance checks + --> E721_basedpython.by:6:8 + | +4 | if type(a) == type(b): +5 | pass +6 | if type(a) == int: + | ^^^^^^^^^^^^^^ +7 | pass + | + +E721 Use `===` and `!==` for type comparisons, or `is` for isinstance checks + --> E721_basedpython.by:11:12 + | +10 | def reified[reified T](x: T) -> None: +11 | assert T == int + | ^^^^^^^^ diff --git a/crates/ruff_linter/src/rules/pyflakes/mod.rs b/crates/ruff_linter/src/rules/pyflakes/mod.rs index 6cda6ca437..2ad8b998e6 100644 --- a/crates/ruff_linter/src/rules/pyflakes/mod.rs +++ b/crates/ruff_linter/src/rules/pyflakes/mod.rs @@ -91,6 +91,7 @@ mod tests { #[test_case(Rule::MultipleStarredExpressions, Path::new("F622.py"))] #[test_case(Rule::AssertTuple, Path::new("F631.py"))] #[test_case(Rule::IsLiteral, Path::new("F632.py"))] + #[test_case(Rule::IsLiteral, Path::new("F632_basedpython.by"))] #[test_case(Rule::InvalidPrintSyntax, Path::new("F633.py"))] #[test_case(Rule::IfTuple, Path::new("F634.py"))] // basedpython: a pattern clause's subject is a value, not a test diff --git a/crates/ruff_linter/src/rules/pyflakes/rules/invalid_literal_comparisons.rs b/crates/ruff_linter/src/rules/pyflakes/rules/invalid_literal_comparisons.rs index 6a8b9b164e..476a15ad7e 100644 --- a/crates/ruff_linter/src/rules/pyflakes/rules/invalid_literal_comparisons.rs +++ b/crates/ruff_linter/src/rules/pyflakes/rules/invalid_literal_comparisons.rs @@ -55,6 +55,8 @@ use crate::{AlwaysFixableViolation, Edit, Fix}; #[violation_metadata(stable_since = "v0.0.39", category = Category::Suspicious)] pub(crate) struct IsLiteral { cmp_op: IsCmpOp, + /// Whether the operator was written with basedpython's `===` / `!==` + spells_identity: bool, } impl AlwaysFixableViolation for IsLiteral { @@ -67,9 +69,12 @@ impl AlwaysFixableViolation for IsLiteral { } fn fix_title(&self) -> String { - let title = match self.cmp_op { - IsCmpOp::Is => "Replace `is` with `==`", - IsCmpOp::IsNot => "Replace `is not` with `!=`", + // basedpython's `===` / `!==` are the identity operators this rule replaces there + let title = match (self.cmp_op, self.spells_identity) { + (IsCmpOp::Is, false) => "Replace `is` with `==`", + (IsCmpOp::Is, true) => "Replace `===` with `==`", + (IsCmpOp::IsNot, false) => "Replace `is not` with `!=`", + (IsCmpOp::IsNot, true) => "Replace `!==` with `!=`", }; title.to_string() } @@ -83,25 +88,50 @@ pub(crate) fn invalid_literal_comparison( comparators: &[Expr], expr: &Expr, ) { - let mut lazy_located = None; + // basedpython keeps `is` as python identity only where its right-hand side is a + // literal; anywhere else `is` is a type test, and `===` is the spelling that always + // compares identity. telling `===` from `is` takes the tokens, which are worth + // locating up front once the file can contain either + let mut lazy_located = (checker.source_type.is_basedpython() + && ops.iter().any(|op| matches!(op, CmpOp::Is | CmpOp::IsNot))) + .then(|| locate_cmp_ops(expr, checker.tokens())); let mut left = left; for (index, (op, right)) in ops.iter().zip(comparators).enumerate() { + let spells_identity = lazy_located + .as_ref() + .and_then(|located| located.get(index)) + .is_some_and(|located_op| located_op.op == *op && located_op.spells_identity); + if matches!(op, CmpOp::Is | CmpOp::IsNot) && (helpers::is_constant_non_singleton(left) || helpers::is_constant_non_singleton(right) || helpers::is_mutable_iterable_initializer(left) || helpers::is_mutable_iterable_initializer(right)) + && (!checker.source_type.is_basedpython() || spells_identity || right.is_literal_expr()) { - let mut diagnostic = - checker.report_diagnostic(IsLiteral { cmp_op: op.into() }, expr.range()); + let mut diagnostic = checker.report_diagnostic( + IsLiteral { + cmp_op: op.into(), + spells_identity, + }, + expr.range(), + ); if lazy_located.is_none() { lazy_located = Some(locate_cmp_ops(expr, checker.tokens())); } diagnostic.try_set_optional_fix(|| { - if let Some(located_op) = - lazy_located.as_ref().and_then(|located| located.get(index)) - { - assert_eq!(located_op.op, *op); + let located_op = lazy_located.as_ref().and_then(|located| located.get(index)); + // the tokens and the operators come from one parse, so they line up. a + // basedpython file is the one place the two can legitimately disagree, + // because a spelling the token scan does not know reads as a different + // operator there — hence a dropped fix rather than a panic + debug_assert!( + checker.source_type.is_basedpython() + || located_op.is_none_or(|located_op| located_op.op == *op), + "located `{:?}` where the comparison has `{op:?}`", + located_op.map(|located_op| located_op.op) + ); + if let Some(located_op) = located_op.filter(|located_op| located_op.op == *op) { if let Ok(content) = match located_op.op { CmpOp::Is => Ok::("==".to_string()), CmpOp::IsNot => Ok("!=".to_string()), @@ -197,6 +227,12 @@ fn locate_cmp_ops(expr: &Expr, tokens: &Tokens) -> Vec { }; ops.push(op); } + TokenKind::EqEqEqual => { + ops.push(LocatedCmpOp::identity(token.range(), CmpOp::Is)); + } + TokenKind::BangEqEqual => { + ops.push(LocatedCmpOp::identity(token.range(), CmpOp::IsNot)); + } TokenKind::NotEqual => { ops.push(LocatedCmpOp::new(token.range(), CmpOp::NotEq)); } @@ -225,6 +261,11 @@ fn locate_cmp_ops(expr: &Expr, tokens: &Tokens) -> Vec { struct LocatedCmpOp { range: TextRange, op: CmpOp, + /// Whether the operator was written with basedpython's `===` / `!==`, the + /// spellings that compare identity there. A plain `is` in a basedpython file + /// is a [parametric type test](https://docs.basedpython.org/features/parametric-type-tests) + /// and parses to the same [`CmpOp`], so the two can only be told apart here. + spells_identity: bool, } impl LocatedCmpOp { @@ -232,6 +273,14 @@ impl LocatedCmpOp { Self { range: range.into(), op, + spells_identity: false, + } + } + + fn identity>(range: T, op: CmpOp) -> Self { + Self { + spells_identity: true, + ..Self::new(range, op) } } } diff --git a/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__is-literal_F632_basedpython.by.snap b/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__is-literal_F632_basedpython.by.snap new file mode 100644 index 0000000000..7473174a37 --- /dev/null +++ b/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__is-literal_F632_basedpython.by.snap @@ -0,0 +1,65 @@ +--- +source: crates/ruff_linter/src/rules/pyflakes/mod.rs +--- +F632 [*] Use `==` to compare constant literals + --> F632_basedpython.by:5:11 + | +3 | # spellings, and an `is` whose right-hand side is a literal, compare identity +4 | def identity(x: object) -> None: +5 | print(1 === 1) # F632 + | ^^^^^^^ +6 | print(x !== "a") # F632 + | +help: Replace `===` with `==` + | +4 | def identity(x: object) -> None: + - print(1 === 1) # F632 +5 + print(1 == 1) # F632 +6 | print(x !== "a") # F632 + | + +F632 [*] Use `!=` to compare constant literals + --> F632_basedpython.by:6:11 + | +4 | def identity(x: object) -> None: +5 | print(1 === 1) # F632 +6 | print(x !== "a") # F632 + | ^^^^^^^^^ +help: Replace `!==` with `!=` + | +5 | print(1 === 1) # F632 + - print(x !== "a") # F632 +6 + print(x != "a") # F632 +7 | + | + +F632 [*] Use `==` to compare constant literals + --> F632_basedpython.by:10:11 + | + 9 | def literal_right_hand_side(x: object) -> None: +10 | print(1 is 1) # F632 + | ^^^^^^ +11 | print(x is not "a") # F632 + | +help: Replace `is` with `==` + | +9 | def literal_right_hand_side(x: object) -> None: + - print(1 is 1) # F632 +10 + print(1 == 1) # F632 +11 | print(x is not "a") # F632 + | + +F632 [*] Use `!=` to compare constant literals + --> F632_basedpython.by:11:11 + | + 9 | def literal_right_hand_side(x: object) -> None: +10 | print(1 is 1) # F632 +11 | print(x is not "a") # F632 + | ^^^^^^^^^^^^ +help: Replace `is not` with `!=` + | +10 | print(1 is 1) # F632 + - print(x is not "a") # F632 +11 + print(x != "a") # F632 +12 | + | diff --git a/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__unused-variable_F841_basedpython.by.snap b/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__unused-variable_F841_basedpython.by.snap index da1004479d..7365f7df87 100644 --- a/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__unused-variable_F841_basedpython.by.snap +++ b/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__unused-variable_F841_basedpython.by.snap @@ -29,3 +29,27 @@ F841 Local variable `Unmatched` is assigned to but never used 46 | case _: | help: Remove assignment to unused variable `Unmatched` + +F841 Local variable `k` is assigned to but never used + --> F841_basedpython.by:78:21 + | +76 | # what python's own capture pattern does, so an unused capture still fires +77 | def matching_binders(p: Point) -> int: +78 | if let Point(j, k) := p: # F841 on `k` + | ^ +79 | return j +80 | match p: + | +help: Remove assignment to unused variable `k` + +F841 Local variable `m` is assigned to but never used + --> F841_basedpython.by:81:23 + | +79 | return j +80 | match p: +81 | case Point(l, m): # F841 on `m` + | ^ +82 | return l +83 | return 0 + | +help: Remove assignment to unused variable `m` diff --git a/crates/ruff_linter/src/rules/pyupgrade/mod.rs b/crates/ruff_linter/src/rules/pyupgrade/mod.rs index dc0abeeb9c..6f952c11e2 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/mod.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/mod.rs @@ -65,6 +65,7 @@ mod tests { #[test_case(Rule::QuotedAnnotation, Path::new("UP037_0.py"))] #[test_case(Rule::QuotedAnnotation, Path::new("UP037_1.py"))] #[test_case(Rule::QuotedAnnotation, Path::new("UP037_2.pyi"))] + #[test_case(Rule::QuotedAnnotation, Path::new("UP037_basedpython.by"))] #[test_case(Rule::QuotedAnnotation, Path::new("UP037_3.py"))] #[test_case(Rule::RedundantOpenModes, Path::new("UP015.py"))] #[test_case(Rule::RedundantOpenModes, Path::new("UP015_1.py"))] diff --git a/crates/ruff_linter/src/rules/pyupgrade/snapshots/ruff_linter__rules__pyupgrade__tests__UP037_basedpython.by.snap b/crates/ruff_linter/src/rules/pyupgrade/snapshots/ruff_linter__rules__pyupgrade__tests__UP037_basedpython.by.snap new file mode 100644 index 0000000000..2bacb5d540 --- /dev/null +++ b/crates/ruff_linter/src/rules/pyupgrade/snapshots/ruff_linter__rules__pyupgrade__tests__UP037_basedpython.by.snap @@ -0,0 +1,4 @@ +--- +source: crates/ruff_linter/src/rules/pyupgrade/mod.rs +--- + diff --git a/crates/ruff_linter/src/rules/ruff/mod.rs b/crates/ruff_linter/src/rules/ruff/mod.rs index 6635c112dd..5770e3ddd9 100644 --- a/crates/ruff_linter/src/rules/ruff/mod.rs +++ b/crates/ruff_linter/src/rules/ruff/mod.rs @@ -62,6 +62,7 @@ mod tests { #[test_case(Rule::UnnecessaryKeyCheck, Path::new("RUF019.py"))] #[test_case(Rule::NeverUnion, Path::new("RUF020.py"))] #[test_case(Rule::ParenthesizeChainedOperators, Path::new("RUF021.py"))] + #[test_case(Rule::ParenthesizeChainedOperators, Path::new("RUF021_basedpython.by"))] #[test_case(Rule::UnsortedDunderAll, Path::new("RUF022.py"))] #[test_case(Rule::UnsortedDunderSlots, Path::new("RUF023.py"))] #[test_case(Rule::MutableFromkeysValue, Path::new("RUF024.py"))] diff --git a/crates/ruff_linter/src/rules/ruff/snapshots/ruff_linter__rules__ruff__tests__parenthesize-chained-operators_RUF021_basedpython.by.snap b/crates/ruff_linter/src/rules/ruff/snapshots/ruff_linter__rules__ruff__tests__parenthesize-chained-operators_RUF021_basedpython.by.snap new file mode 100644 index 0000000000..5c673b2ec4 --- /dev/null +++ b/crates/ruff_linter/src/rules/ruff/snapshots/ruff_linter__rules__ruff__tests__parenthesize-chained-operators_RUF021_basedpython.by.snap @@ -0,0 +1,16 @@ +--- +source: crates/ruff_linter/src/rules/ruff/mod.rs +--- +RUF021 [*] Parenthesize `a and b` expressions when chaining `and` and `or` together, to make the precedence clear + --> RUF021_basedpython.by:21:12 + | +19 | # a boolean expression in a value position still wants the parentheses +20 | def values(a: bool, b: bool, c: bool) -> bool: +21 | return a and b or c + | ^^^^^^^ +help: Parenthesize the `and` subexpression + | +20 | def values(a: bool, b: bool, c: bool) -> bool: + - return a and b or c +21 + return (a and b) or c + | diff --git a/crates/ruff_python_semantic/src/model.rs b/crates/ruff_python_semantic/src/model.rs index 369830232e..0aa5ffb3ab 100644 --- a/crates/ruff_python_semantic/src/model.rs +++ b/crates/ruff_python_semantic/src/model.rs @@ -892,6 +892,18 @@ impl<'a> SemanticModel<'a> { self.in_basedpython_file() && self.in_trailing_lambda_block() } + /// True when the model is inside a type expression in a basedpython file, + /// where `or` and `and` are the + /// [keyword spellings](https://docs.basedpython.org/features/or-and-types) + /// of union and intersection. + /// + /// `A or B` there is the type `A | B`, not a boolean expression whose value + /// is `A` whenever `A` is truthy, so a rule that reasons about python's + /// boolean operators has nothing to say about it. + pub fn in_basedpython_type_expression(&self) -> bool { + self.in_basedpython_file() && self.in_type_definition() + } + /// True when the innermost function scope is a /// [trailing-lambda](https://docs.basedpython.org/features/trailing-lambdas) /// block, whose receiver the parser binds outside the source's reach diff --git a/crates/ty_project/src/lib.rs b/crates/ty_project/src/lib.rs index 065013861c..b9f0907709 100644 --- a/crates/ty_project/src/lib.rs +++ b/crates/ty_project/src/lib.rs @@ -1281,6 +1281,43 @@ mod tests { Ok(()) } + /// A diagnostic names its types in the syntax of the file it is reported in, and + /// that has to hold however the inference behind it was reached. Checking the `.py` + /// importer first forces the `.by` module's inference, and the message salsa caches + /// there is the one every later reader of that file sees. + #[test] + fn basedpython_diagnostic_keeps_its_spelling_when_a_python_importer_is_checked_first() { + let root = SystemPathBuf::from("/project"); + let project = ProjectMetadata::new("test", root.clone()); + let mut db = TestDb::new(project); + + db.write_files([ + ( + root.join("lib.by"), + "def g():\n c: \"c\" = \"b\"\n return c\n", + ), + (root.join("main.py"), "from lib import g\n\ng()\n"), + ]) + .unwrap(); + + let main = system_path_to_file(&db, root.join("main.py")).unwrap(); + let lib = system_path_to_file(&db, root.join("lib.by")).unwrap(); + + // inferring `g`'s return type infers its body, diagnostics and all + check_file_impl(&db, db.program_file(main)).unwrap(); + + let messages: Vec = check_file_impl(&db, db.program_file(lib)) + .unwrap() + .iter() + .map(|diagnostic| diagnostic.concise_message().to_string()) + .collect(); + + assert_eq!( + messages, + vec![r#"Object of type `"b"` is not assignable to `"c"`"#.to_string()] + ); + } + #[test] fn explicit_nested_included_file_is_a_literal_match() { let root = SystemPathBuf::from("/project"); diff --git a/crates/ty_python_semantic/src/lib.rs b/crates/ty_python_semantic/src/lib.rs index a7ad4c4a4b..6a5f6c2f2f 100644 --- a/crates/ty_python_semantic/src/lib.rs +++ b/crates/ty_python_semantic/src/lib.rs @@ -481,11 +481,7 @@ pub fn check_file_with( /// hint — should go through this, or it will spell types in a syntax the file /// cannot be written in. pub fn with_display_for_file(db: &dyn Db, file: File, body: impl FnOnce() -> R) -> R { - if file.source_type(db).is_basedpython() { - crate::types::display::with_basedpython_display(body) - } else { - body() - } + crate::types::display::with_basedpython_display(file.source_type(db).is_basedpython(), body) } fn check_file_inner( diff --git a/crates/ty_python_semantic/src/types/display.rs b/crates/ty_python_semantic/src/types/display.rs index 6e74b900e9..3566d45248 100644 --- a/crates/ty_python_semantic/src/types/display.rs +++ b/crates/ty_python_semantic/src/types/display.rs @@ -920,15 +920,26 @@ fn basedpython_display_enabled() -> bool { BASEDPYTHON_DISPLAY.with(std::cell::Cell::get) } -/// Run `f` with basedpython-style type display enabled. Used by -/// diagnostic emission for `.by` files -pub(crate) fn with_basedpython_display(f: impl FnOnce() -> R) -> R { - BASEDPYTHON_DISPLAY.with(|cell| { - let prev = cell.replace(true); - let result = f(); - cell.set(prev); - result - }) +/// Run `f` with basedpython-style type display set to `enabled`. +/// +/// Both directions matter: inferring a `.by` file reaches definitions in `.py` files +/// and vice versa, and each file's own diagnostics have to be spelled in that file's +/// syntax rather than in whichever one the outermost caller happened to be reading. +pub(crate) fn with_basedpython_display(enabled: bool, f: impl FnOnce() -> R) -> R { + /// Restores the previous setting however `f` ends. Type inference is where this is + /// set, and salsa cancels an inference by unwinding through it — a `f()` that never + /// returns would otherwise leave the thread spelling every later type in the syntax + /// of the file whose inference was cancelled. + struct Restore(bool); + + impl Drop for Restore { + fn drop(&mut self) { + BASEDPYTHON_DISPLAY.with(|cell| cell.set(self.0)); + } + } + + let _restore = BASEDPYTHON_DISPLAY.with(|cell| Restore(cell.replace(enabled))); + f() } /// Format a file location suffix for disambiguation (e.g., " @ path:line:column") diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index efc4a1c48d..d9d8c9a900 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -1278,6 +1278,15 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { /// Infers types in the given [`InferenceRegion`]. fn infer_region(&mut self) { + // a diagnostic raised here is written for the file being inferred, so the types + // it names have to be spelled in that file's own syntax. the spelling is taken + // from the file rather than from whoever asked for the inference because this + // work is cached: were it taken from the caller, the first reader of a `.by` + // file would decide how every later reader sees the messages salsa kept + crate::with_display_for_file(self.db(), self.file(), || self.infer_region_inner()); + } + + fn infer_region_inner(&mut self) { match self.region { InferenceRegion::Statement(statement) => self.infer_region_statement(statement), InferenceRegion::Scope(scope, tcx) => self.infer_region_scope(scope, tcx), diff --git a/docs/basedpython/features/linter.md b/docs/basedpython/features/linter.md index 276e6ae5f5..3e8550f015 100644 --- a/docs/basedpython/features/linter.md +++ b/docs/basedpython/features/linter.md @@ -78,12 +78,34 @@ depending on a line that basedpython does not need a rule that suggests a replacement suggests the python one. `SIM108` above is the case where that composes; where it does not, the suggestion is still valid -`.by`, just not the shortest way to write it - -nothing in ruff's rule set is known to report a construct that is correct -basedpython. `F821` used to: an unqualified builder inside a +`.by`, just not the shortest way to write it. `E721` is the exception, and names +basedpython's operators instead, because python's advice there — reach for `is` +— spells [a type test](parametric-type-tests.md) in `.by` rather than the +identity comparison it means in python + +a rule that reads a construct as the python it resembles is a false positive, +and each one is answered where the misreading is, not by silencing the rule: + +- `or` and `and` inside a type expression are + [union and intersection](or-and-types.md), so the boolean rules — `SIM222`, + `SIM223`, `RUF021` — say nothing there +- a bare string in a type position is + [the literal type](literal-types.md), not a forward reference, so `UP037` + does not offer to take its quotes off +- `is` is a type test unless its right-hand side is a literal, so `F632` reports + only what really compares identity: the `===` and `!==` spellings, and an + `is` against a literal +- a [match type](match-types.md)'s `case` arms are types, so `E701` does not + read the `:` in one as opening a suite +- a [destructuring](destructuring.md) binder always binds, so its captures are + reported the way the equivalent python unpacking's are — `B007` for a loop, + `F841` never +- a type parameter is not a constant, however the one-letter convention spells + it, so `assert T == int` is not a `SIM300` yoda condition + +`F821` is the older instance of the same thing: an unqualified builder inside a [trailing-lambda](trailing-lambdas.md) block resolves against the block's [implicit receiver](implicit-receivers.md), and the linter cannot see receiver -types, so it now defers every unresolved name inside a block to `by check`. the +types, so it defers every unresolved name inside a block to `by check`. the same deferral covers `self` and an [enum variant](context-sensitive-resolution.md) written bare From 74ac27a3ae7e7df46ef4b5d0fe2a3599cc482953 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 7 Sep 2026 19:30:10 +1000 Subject: [PATCH 4/7] lower type expressions written in a call `TypeVar("T", bound=A & B)` type checked clean and transpiled to itself, so the emitted python evaluated `A.__and__(B)` at import. `NewType`, `TypeAliasType`, `ParamSpec`, `TypeVarTuple` and the functional `NamedTuple` and `TypedDict` were all silent the same way, for `?`, `not`, `or`/`and` and the rest of the type syntax as much as for `&`: the transpiler's type-expression walker recognised twelve positions, and none of them was an argument. Which arguments those are is now the type checker's answer rather than a list the transpiler keeps. `CallTypeForm` names the constructs and their type-expression arguments in one place, inference dispatches on it, and the walker asks it, so a form ty learns to check is one the transpiler lowers without further change. The mdtest that covers this is one the divergence harness runs: with the walker's hook removed it fails with `TypeError: unsupported operand type(s) for &`, which is what the miscompilation did to anyone who wrote it. --- .../src/transforms/intersection.rs | 95 +++++++ .../src/transforms/type_expr_walker.rs | 35 +++ crates/by_transforms/src/type_info.rs | 21 ++ .../mdtest/basedpython_intersection.md | 46 ++++ crates/ty_python_semantic/src/types.rs | 5 +- .../src/types/call_type_forms.rs | 239 ++++++++++++++++++ .../src/types/infer/builder.rs | 38 +-- 7 files changed, 461 insertions(+), 18 deletions(-) create mode 100644 crates/ty_python_semantic/src/types/call_type_forms.rs diff --git a/crates/by_transforms/src/transforms/intersection.rs b/crates/by_transforms/src/transforms/intersection.rs index 488e163b81..b593ff1479 100644 --- a/crates/by_transforms/src/transforms/intersection.rs +++ b/crates/by_transforms/src/transforms/intersection.rs @@ -558,4 +558,99 @@ mod tests { "}, ); } + + // a handful of typing constructs spell a type through a call rather than through an + // annotation. those arguments are type expressions, so the surface syntax written in + // one lowers exactly as it does after a `:` — an unlowered `A & B` reaching the output + // is a runtime `A.__and__(B)` + + #[test] + fn typevar_bound_lowers() { + check( + "import typing\nT = typing.TypeVar(\"T\", bound=A & B)\n", + indoc! {r#" + from ty_extensions import Intersection + import typing + T = typing.TypeVar("T", bound=Intersection[A, B]) + "#}, + ); + } + + #[test] + fn typevar_constraints_lower() { + check( + "import typing\nT = typing.TypeVar(\"T\", A & B, C or D)\n", + indoc! {r#" + from ty_extensions import Intersection + import typing + T = typing.TypeVar("T", Intersection[A, B], C | D) + "#}, + ); + } + + #[test] + fn paramspec_default_elements_lower() { + check( + "import typing\nP = typing.ParamSpec(\"P\", default=[A & B, C])\n", + indoc! {r#" + from ty_extensions import Intersection + import typing + P = typing.ParamSpec("P", default=[Intersection[A, B], C]) + "#}, + ); + } + + #[test] + fn newtype_base_lowers() { + check( + "import typing\nD = typing.NewType(\"D\", A or B)\n", + indoc! {r#" + import typing + D = typing.NewType("D", A | B) + "#}, + ); + } + + #[test] + fn functional_named_tuple_field_lowers() { + check( + "import typing\nNT = typing.NamedTuple(\"NT\", [(\"f\", A & B)])\n", + indoc! {r#" + from ty_extensions import Intersection + import typing + NT = typing.NamedTuple("NT", [("f", Intersection[A, B])]) + "#}, + ); + } + + #[test] + fn functional_typed_dict_field_lowers() { + check( + "import typing\nTD = typing.TypedDict(\"TD\", {\"f\": A & B})\n", + indoc! {r#" + from ty_extensions import Intersection + import typing + TD = typing.TypedDict("TD", {"f": Intersection[A, B]}) + "#}, + ); + } + + #[test] + fn type_alias_type_value_lowers() { + check_py312( + "import typing\nAL = typing.TypeAliasType(\"AL\", A & B)\n", + indoc! {r#" + from ty_extensions import Intersection + import typing + AL = typing.TypeAliasType("AL", Intersection[A, B]) + "#}, + ); + } + + // the keyword is only a type operator where the call really is one of those + // constructs; an ordinary call's arguments stay boolean expressions + #[test] + fn ordinary_call_argument_untouched() { + unchanged("f(A and B)\n"); + } } diff --git a/crates/by_transforms/src/transforms/type_expr_walker.rs b/crates/by_transforms/src/transforms/type_expr_walker.rs index 101903c6a0..de6c993228 100644 --- a/crates/by_transforms/src/transforms/type_expr_walker.rs +++ b/crates/by_transforms/src/transforms/type_expr_walker.rs @@ -32,6 +32,11 @@ //! 9. `Annotated[T, meta…]` only the first arg //! 10. `Callable[[P1, P2], R]` parameter list elements + return type //! 12. class base list +//! 13. the type-expression arguments of a typing construct that spells a type through a +//! call — `NewType("D", int)`, `TypeVar("T", bound=int)`, the functional +//! `NamedTuple` and `TypedDict`. which arguments those are is the type checker's +//! answer rather than a list kept here, so a form ty learns to check as a type is +//! one this lowers without further change use ruff_python_ast::helpers::declaration_annotation_type; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; @@ -374,6 +379,36 @@ impl<'ast> Visitor<'ast> for TypePosWalker<'_> { } return; } + + // a typing construct that spells a type through a call + if let Some(types) = self.types { + let type_arguments = types.call_type_expression_arguments(c); + if !type_arguments.is_empty() { + // a form can hold its types inside a literal — a `NamedTuple`'s + // field list, a `TypedDict`'s field dict — so an argument counts as + // handled once it *contains* one, and is not walked again as a value + let holds_type_argument = |expr: &Expr| { + type_arguments + .iter() + .any(|argument| expr.range().contains_range(argument.range())) + }; + for argument in &type_arguments { + self.visit_type_expr(argument, TypePos::Root); + } + // everything else the call holds is an ordinary value + for arg in &c.arguments.args { + if !holds_type_argument(arg) { + self.visit_expr(arg); + } + } + for kw in &c.arguments.keywords { + if !holds_type_argument(&kw.value) { + self.visit_expr(&kw.value); + } + } + return; + } + } } // lambda parameter annotations (basedpython supports typed lambdas) if let Expr::Lambda(l) = expr { diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index cf9df35f79..0ee655a86a 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -10,6 +10,7 @@ use ruff_python_stdlib::basedpython::IMPLICIT_TYPING_NAMES; use ruff_text_size::TextRange; use ty_python_core::scope::ScopeKind; use ty_python_core::{global_scope, place_table, semantic_index}; +use ty_python_semantic::types::call_type_forms::CallTypeForm; use ty_python_semantic::types::{ DisplaySettings, DynamicType, KnownClass, KnownInstanceType, Type, UnpackedKwargs, character, }; @@ -91,6 +92,18 @@ pub(crate) trait TypeInfo { /// unresolved name should be treated as a runtime subscript, not a type) fn subscript_is_known_type_context(&self, value: &Expr) -> bool; + /// the arguments of `call` that are type expressions, because the call is one of the + /// typing constructs that spells a type through a call — `NewType("D", int)`, + /// `TypeVar("T", bound=int)`, the functional `NamedTuple` and `TypedDict` + /// + /// the type checker decides this, from the same + /// [`CallTypeForm`](ty_python_semantic::types::call_type_forms::CallTypeForm) it uses + /// to check those arguments as types. a form ty checks as a type and the transpiler + /// leaves alone is a silent miscompilation: the surface syntax survives into the + /// output as ordinary python, where `A & B` is a runtime `__and__` call and + /// `A and B or C` evaluates to `C` + fn call_type_expression_arguments<'ast>(&self, call: &'ast ExprCall) -> Vec<&'ast Expr>; + fn is_function(&self, name: &ExprName) -> bool; /// basedpython: how many entries the class pattern's class lists in @@ -626,6 +639,14 @@ impl TypeInfo for SemanticModel<'_> { } } + fn call_type_expression_arguments<'ast>(&self, call: &'ast ExprCall) -> Vec<&'ast Expr> { + call.func + .inferred_type(self) + .and_then(|callee| CallTypeForm::of(self.db(), callee)) + .map(|form| form.type_expressions(&call.arguments)) + .unwrap_or_default() + } + fn is_function(&self, name: &ExprName) -> bool { name.inferred_type(self) .is_some_and(|ty| ty.as_function_literal().is_some()) diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_intersection.md b/crates/ty_python_semantic/resources/mdtest/basedpython_intersection.md index 0123f058cf..f49c972148 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_intersection.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_intersection.md @@ -55,3 +55,49 @@ class C: ... def f(x: A & B & C) -> None: reveal_type(x) # revealed: A & B & C ``` + +## intersection in a type spelled through a call + +A few typing constructs name a type in a call argument rather than after a `:` — `NewType` names its +base, `TypeVar` names its bound and its constraints, and the functional `NamedTuple` and `TypedDict` +name their field types inside a literal. Those arguments are type expressions, so `&` means the same +thing in one as it does in an annotation, and the transpiler has to lower it there too: an `A & B` +that survived into the emitted python would be a runtime `A.__and__(B)`. + +```by +import typing + +class A: ... + +class B: ... + +T = typing.TypeVar("T", bound=A & B) +U = typing.TypeVar("U", A & B, int) +Pair = typing.NamedTuple("Pair", [("left", A & B), ("right", int)]) +Row = typing.TypedDict("Row", {"cell": A & B}) + +def f(x: Pair, y: Row) -> None: + reveal_type(x.left) # revealed: A & B + reveal_type(y["cell"]) # revealed: A & B +``` + +## an ordinary call is not one of those constructs + +Only a call that resolves to one of those constructs reads its arguments as types. Anywhere else +`and` is python's boolean operator, and the value it produces is the one the call receives. + +```by +import typing + +class A: ... + +class B(A): ... + +Both = typing.NewType("Both", B) + +def pick(value: object) -> object: + return value + +chosen = pick(A and B) +reveal_type(chosen) # revealed: object +``` diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 40c4f3bf42..61865adb38 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -173,6 +173,7 @@ mod bool; mod bound_super; mod build_stamps; mod call; +pub mod call_type_forms; mod callable; pub mod character; mod class; @@ -699,7 +700,7 @@ pub enum TypingModule { impl TypingModule { /// Return the module for a `TypedDict` special form, including a union of the special forms /// exported by `typing` and `typing_extensions`. - fn from_typed_dict_type<'db>(db: &'db dyn Db, ty: Type<'db>) -> Option { + pub(crate) fn from_typed_dict_type<'db>(db: &'db dyn Db, ty: Type<'db>) -> Option { match ty { Type::SpecialForm(SpecialFormType::TypedDict(module)) => Some(module), Type::Union(union) => { @@ -726,7 +727,7 @@ impl TypingModule { } } - const fn from_type_alias_class(class: KnownClass) -> Option { + pub(crate) const fn from_type_alias_class(class: KnownClass) -> Option { match class { KnownClass::TypeAliasType => Some(Self::Typing), KnownClass::ExtensionsTypeAliasType => Some(Self::TypingExtensions), diff --git a/crates/ty_python_semantic/src/types/call_type_forms.rs b/crates/ty_python_semantic/src/types/call_type_forms.rs new file mode 100644 index 0000000000..1ca77a7640 --- /dev/null +++ b/crates/ty_python_semantic/src/types/call_type_forms.rs @@ -0,0 +1,239 @@ +//! The typing constructs that spell a type through a *call* rather than through an +//! annotation. +//! +//! Most type expressions sit somewhere a reader can point at syntactically — after a +//! `:`, after a `->`, on the right of a `type` alias. These do not: `NewType("D", int)` +//! names its base in an argument, `TypeVar("T", bound=int)` names its bound in a +//! keyword, and the functional `NamedTuple` and `TypedDict` name their field types +//! inside a list or dict literal. Nothing about the call syntax says so — the answer +//! comes from what the callee resolves to. +//! +//! Two consumers need that answer. Type inference needs it to check the arguments as +//! type expressions, and the basedpython transpiler needs it to lower the surface syntax +//! written in them: `TypeVar("T", bound=A & B)` has to reach the emitted python as +//! `bound=Intersection[A, B]`, exactly as `x: A & B` does. Stating it once, here, is +//! what keeps a form the type checker accepts from being one the transpiler leaves +//! behind — which is a silent miscompilation, because the unlowered expression is +//! perfectly good python that means something else at runtime. + +use ruff_python_ast as ast; + +use crate::Db; +use crate::types::{KnownClass, SpecialFormType, Type, TypingModule}; + +/// A typing construct whose call arguments hold type expressions. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CallTypeForm { + /// `NewType("D", int)` — the second argument is the base type + NewType, + /// `TypeAliasType("X", int | str)` — the second argument is the alias value + TypeAliasType, + /// `NamedTuple("NT", [("f", int)])` — the second argument holds `(name, type)` pairs + NamedTuple, + /// `TypedDict("TD", {"f": int})` — the second argument holds `key: type` items + TypedDict, + /// `TypeVar("T", int, str, bound=…, default=…)` — the constraints, the bound and the + /// default are all type expressions + TypeVar, + /// `ParamSpec("P", default=[int, str])` — the default is a parameter list + ParamSpec, + /// `TypeVarTuple("Ts", default=Unpack[tuple[int, ...]])` — the default is a type + TypeVarTuple, +} + +impl CallTypeForm { + /// Which construct, if any, `callee` is. `callee` is the inferred type of the + /// expression in call position, so an unresolved or shadowed name answers `None` + /// rather than being matched by spelling. + pub fn of<'db>(db: &'db dyn Db, callee: Type<'db>) -> Option { + if callee == Type::SpecialForm(SpecialFormType::NamedTuple) { + return Some(Self::NamedTuple); + } + if TypingModule::from_typed_dict_type(db, callee).is_some() { + return Some(Self::TypedDict); + } + match callee.as_class_literal()?.known(db)? { + KnownClass::NewType => Some(Self::NewType), + KnownClass::TypeVar | KnownClass::ExtensionsTypeVar => Some(Self::TypeVar), + KnownClass::ParamSpec | KnownClass::ExtensionsParamSpec => Some(Self::ParamSpec), + KnownClass::TypeVarTuple | KnownClass::ExtensionsTypeVarTuple => { + Some(Self::TypeVarTuple) + } + known_class if TypingModule::from_type_alias_class(known_class).is_some() => { + Some(Self::TypeAliasType) + } + _ => None, + } + } + + /// The sub-expressions of `arguments` that this form reads as type expressions. + /// + /// Each one is a complete type expression, so a caller that walks type expressions + /// can hand them to the same traversal it uses for an annotation. Arguments a form + /// reads as ordinary values — every form's leading name, a `TypedDict`'s field keys, + /// `TypeAliasType`'s `type_params` — are not returned. + pub fn type_expressions(self, arguments: &ast::Arguments) -> Vec<&ast::Expr> { + let mut found = Vec::new(); + match self { + Self::NewType | Self::TypeAliasType => found.extend(arguments.args.get(1)), + Self::NamedTuple => { + // `[("f", int), ("g", str)]`, or the same as a tuple, each field written + // as either a list or a tuple of its name and its type + if let Some(fields) = arguments.args.get(1) { + for field in sequence_elements(fields).into_iter().flatten() { + found.extend(sequence_elements(field).and_then(|pair| pair.get(1))); + } + } + } + Self::TypedDict => { + if let Some(ast::Expr::Dict(fields)) = arguments.args.get(1) { + found.extend( + fields + .items + .iter() + .filter(|item| item.key.is_some()) + .map(|item| &item.value), + ); + } + found.extend( + arguments + .find_keyword("extra_items") + .map(|keyword| &keyword.value), + ); + } + Self::TypeVar => { + // every positional argument after the name is a constraint + found.extend(arguments.args.iter().skip(1)); + found.extend(arguments.find_keyword("bound").map(|kw| &kw.value)); + found.extend(arguments.find_keyword("default").map(|kw| &kw.value)); + } + Self::TypeVarTuple => { + found.extend(arguments.find_keyword("default").map(|kw| &kw.value)); + } + Self::ParamSpec => { + // a `ParamSpec` default is a parameter list, `[int, str]`, whose elements + // are the type expressions — or `...`, which holds none + if let Some(default) = arguments.find_keyword("default") { + match &default.value { + ast::Expr::List(list) => found.extend(&list.elts), + value => found.push(value), + } + } + } + } + found + } +} + +/// The elements of a list or tuple literal, or `None` for anything else. +fn sequence_elements(expr: &ast::Expr) -> Option<&[ast::Expr]> { + match expr { + ast::Expr::List(list) => Some(&list.elts), + ast::Expr::Tuple(tuple) => Some(&tuple.elts), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ruff_python_parser::parse_expression; + use ruff_text_size::Ranged; + + /// The source text of each argument `form` reads as a type expression in `call`. + fn type_expressions(form: CallTypeForm, call: &str) -> Vec { + let parsed = parse_expression(call).expect("a call expression"); + let ast::Expr::Call(call_expr) = parsed.expr() else { + panic!("expected a call expression"); + }; + form.type_expressions(&call_expr.arguments) + .into_iter() + .map(|expr| call[expr.range()].to_string()) + .collect() + } + + #[test] + fn new_type_reads_its_base() { + assert_eq!( + type_expressions(CallTypeForm::NewType, r#"NewType("D", int | str)"#), + ["int | str"] + ); + } + + #[test] + fn type_alias_type_reads_its_value_but_not_its_type_params() { + assert_eq!( + type_expressions( + CallTypeForm::TypeAliasType, + r#"TypeAliasType("X", list[T], type_params=(T,))"# + ), + ["list[T]"] + ); + } + + #[test] + fn type_var_reads_its_constraints_bound_and_default() { + assert_eq!( + type_expressions( + CallTypeForm::TypeVar, + r#"TypeVar("T", int, str, bound=object, default=int, covariant=True)"# + ), + ["int", "str", "object", "int"] + ); + } + + #[test] + fn param_spec_reads_the_elements_of_its_default() { + assert_eq!( + type_expressions( + CallTypeForm::ParamSpec, + r#"ParamSpec("P", default=[int, str])"# + ), + ["int", "str"] + ); + assert_eq!( + type_expressions(CallTypeForm::ParamSpec, r#"ParamSpec("P", default=...)"#), + ["..."] + ); + } + + #[test] + fn type_var_tuple_reads_its_default() { + assert_eq!( + type_expressions( + CallTypeForm::TypeVarTuple, + r#"TypeVarTuple("Ts", default=Unpack[tuple[int, ...]])"# + ), + ["Unpack[tuple[int, ...]]"] + ); + } + + #[test] + fn named_tuple_reads_each_field_type_but_not_its_name() { + assert_eq!( + type_expressions( + CallTypeForm::NamedTuple, + r#"NamedTuple("NT", [("f", int), ["g", str]])"# + ), + ["int", "str"] + ); + } + + #[test] + fn named_tuple_reads_nothing_from_a_field_list_it_cannot_see_into() { + assert!( + type_expressions(CallTypeForm::NamedTuple, r#"NamedTuple("NT", fields)"#).is_empty() + ); + } + + #[test] + fn typed_dict_reads_each_field_type_and_extra_items() { + assert_eq!( + type_expressions( + CallTypeForm::TypedDict, + r#"TypedDict("TD", {"f": int, **rest}, total=False, extra_items=str)"# + ), + ["int", "str"] + ); + } +} diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index d9d8c9a900..fd1ba583b3 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -61,6 +61,7 @@ use crate::types::call::bind::{ requires_overload_evaluation, }; use crate::types::call::{Argument, Binding, Bindings, CallArguments, CallError, CallErrorKind}; +use crate::types::call_type_forms::CallTypeForm; use crate::types::callable::CallableTypeKind; use crate::types::class::{ ClassLiteral, CodeGeneratorKind, DynamicClassScopeOffset, DynamicNamedTupleAnchor, @@ -4782,35 +4783,40 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let func_ty = self .try_expression_type(func) .unwrap_or_else(|| self.infer_expression(func, TypeContext::default())); - if func_ty == Type::SpecialForm(SpecialFormType::NamedTuple) { - // Only the `fields` argument is deferred for `NamedTuple`; - // other arguments are inferred eagerly. - self.infer_typing_namedtuple_fields(&arguments.args[1]); - return; - } let known_class = func_ty .as_class_literal() .and_then(|cls| cls.known(self.db())); - match (known_class, self.region) { - (Some(KnownClass::NewType), _) => { + // which construct this is, and so which of its arguments are type expressions, is + // decided in one place — the basedpython transpiler asks the same question of the + // same call, because a form whose arguments are checked as types is one whose + // arguments have to be lowered as types too + match (CallTypeForm::of(self.db(), func_ty), self.region) { + (Some(CallTypeForm::NamedTuple), _) => { + // Only the `fields` argument is deferred for `NamedTuple`; + // other arguments are inferred eagerly. + self.infer_typing_namedtuple_fields(&arguments.args[1]); + return; + } + (Some(CallTypeForm::NewType), _) => { self.infer_newtype_assignment_deferred(arguments); return; } - ( - Some(KnownClass::TypeAliasType | KnownClass::ExtensionsTypeAliasType), - InferenceRegion::Deferred(definition), - ) => { + (Some(CallTypeForm::TypeAliasType), InferenceRegion::Deferred(definition)) => { self.infer_typealiastype_assignment_deferred(definition, target, arguments); return; } - (Some(KnownClass::Type), InferenceRegion::Deferred(definition)) => { - self.infer_builtins_type_deferred(definition, value); + (Some(CallTypeForm::TypedDict), _) => { + self.infer_functional_typeddict_deferred(arguments); return; } _ => {} } - if TypingModule::from_typed_dict_type(self.db(), func_ty).is_some() { - self.infer_functional_typeddict_deferred(arguments); + // `type("C", bases, ns)` and `new_class` build a class rather than name a type, so + // neither is a form whose arguments are type expressions + if let (Some(KnownClass::Type), InferenceRegion::Deferred(definition)) = + (known_class, self.region) + { + self.infer_builtins_type_deferred(definition, value); return; } if let InferenceRegion::Deferred(definition) = self.region From ea5e772d8138cf6363212e98358133bc722b7157 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 7 Sep 2026 19:32:28 +1000 Subject: [PATCH 5/7] read an `is` target as a type expression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the right-hand side of `is` in a `.by` file is a type position, but it was inferred as a value expression — so none of the type-expression validation ran on it. an annotation and a type test rejected different things: of 23 targets, annotation position and `cast` position rejected the same 7 and `is` rejected none, lowering each to a program that raises. the target is now inferred as a type expression, and the type it names is classified for a runtime form. what the runtime can check exactly is lowered to that check; what it can only partly check is rejected rather than approximated, since a test narrows and has to earn its `True`: - a class, `type[C]`, a union, `None`, a literal, an enum member, a template literal type and a conformance interface all lower to the check that decides membership of the type. a literal pins the class too, since `1 == True` would otherwise let a `bool` satisfy `Literal[1]` - `Any`, a callable type, a `TypedDict`, an intersection and a protocol with an unnameable member are rejected by `erased-type-check`, each with its own reason. `float`, `complex`, `LiteralString` and `type[Any]` each report the true reason rather than "has no runtime form", and a bare `Callable` lowers to `callable()` - a tuple target now means the tuple *type*. `v is (int, str)` answered `True` for an `int`, because it lowered to `isinstance`'s classinfo tuple the emitted check needs a *value*, and the type the source names parts company with one more often than it looks like it does: `type AL = int` evaluates to a `TypeAliasType`, `Literal[Color.RED]` and `Annotated[int, "x"]` to special forms, `list[Any]` to a subscripted generic — none of which `isinstance` will take. so the spelling is rebuilt from the type rather than passed through, and the source is used only where rebuilding cannot reach: a class the emitting module cannot name as a global, and only when the source wrote a plain dotted name. a sealed hierarchy now declares each variant in the enum body — `Circle: ClassVar[type[_Shape_Circle]]`, `Point: ClassVar[_Shape_Point]` — which is what lets the checker see that one is a class and the other a value, and the lowering asks it rather than matching the written name. matching the name both over-fired (a local binding shadowing the enum was rewritten anyway) and under-fired (`S = Shape; s is S.Point` was missed, emitting an `isinstance` against a singleton that raises). the declarations also clear the `unresolved-attribute` a lowered enum used to report on its own variants. `basedpython_is_keeps_identity` goes away with the value position it existed to serve: `None`, an enum member and a literal all fall out of their own types. the spelling now rides on the ast. `===` / `!==` and the `is` keyword parse to one `CmpOp`, and four places told them apart by reading the source text between the operands — so a parenthesised operand, a line continuation or a comment silently turned a type test back into python identity, and `is` and `is not` disagreed about the same spelling. `ExprCompare::identity_ops` records which operators were written `===` / `!==`; the checker, the narrower, the formatter and the transpiler all read it, and the parser builds the side table lazily so an ordinary comparison allocates nothing. a chained type test is now a syntax error, but only where a type test is something that gets handed on: python chains `a is int is str` into `a is int and int is str`, whose second half asks whether the class `int` has the type `str`, while `a < b is None` hands nothing on. narrowing skips a chain the parser rejected instead of making its body unreachable. `generate_comparison` printed `is` for source that wrote `===`, so E713/E714 rewrote an identity comparison into a type test through a *safe* fix, and E711/E712 destroyed every `===` in a chain; it takes the spelling now. `assertIs` writes `===`, and BY003 rewrites an `isinstance` only when its argument has the shape of a type expression — the predicate the reverse direction already used. F632 reads the recorded spelling as well, rather than scanning the tokens for it a second time; the scan is left to place the fix, which is what lets `1 === 1 !== 2` offer `==` for one operator and `!=` for the other. `identity_swap` no longer decides on ast shape, so an f-string, a call or a unary-minus target takes the same path as a name; its replacement range accounts for parentheses around an operand, which produced invalid python for `(a) is not str`, and an unspaced `a===b` brings its own spaces. a settled test types as its answer rather than `bool`, which is what the editor's data-flow view reads. the reverse direction writes an `isinstance` back out only when its argument is something a type expression can say, mapping a classinfo tuple to a union. also: a recursive alias overflowed the stack on `v is A` — `type A = int | B` with `type B = str | A` — so the alias recursion carries what it has already opened; the template-literal pattern regex and the static matcher disagreed on `-0`, where `str(-0)` is `"0"` so the sign belongs to the non-zero alternative; and control characters are escaped rather than passed through, since CPython refuses a source containing a NUL. the runtime contract each target kind lowers to is executed end to end in `parametric_is_runtime`, rather than only type-checked. --- .../src/reverse_transforms/identity_swap.rs | 118 ++- .../src/transforms/ast_driver.rs | 69 +- .../src/transforms/checked_cast.rs | 62 +- .../src/transforms/coalesce_chain.rs | 3 + .../src/transforms/context_sensitive.rs | 10 +- crates/by_transforms/src/transforms/enums.rs | 70 +- .../src/transforms/identity_swap.rs | 193 ++--- .../src/transforms/init_method.rs | 13 +- .../src/transforms/mutable_defaults.rs | 55 +- .../src/transforms/parametric_is.rs | 762 +++++++++++------- .../src/transforms/reified_class.rs | 2 +- .../src/transforms/reified_generic.rs | 2 +- .../by_transforms/src/transforms/type_is.rs | 9 +- crates/by_transforms/src/type_info.rs | 53 +- .../tests/parametric_is_runtime.rs | 118 +++ .../test/fixtures/basedpython/BY001.by | 10 +- .../fixtures/pyflakes/F632_basedpython.by | 17 + .../src/checkers/ast/analyze/expression.rs | 3 +- .../basedpython/rules/manual_isinstance.rs | 30 +- .../flake8_pytest_style/rules/assertion.rs | 2 +- .../rules/unittest_assert.rs | 33 +- .../flake8_simplify/rules/ast_bool_op.rs | 2 + .../flake8_simplify/rules/ast_unary_op.rs | 4 + .../if_else_block_instead_of_dict_get.rs | 2 + .../if_else_block_instead_of_dict_lookup.rs | 2 + .../flake8_simplify/rules/needless_bool.rs | 5 + .../rules/reimplemented_builtin.rs | 4 + .../pycodestyle/rules/literal_comparisons.rs | 2 + .../src/rules/pycodestyle/rules/not_tests.rs | 3 + .../rules/invalid_literal_comparisons.rs | 70 +- ...tests__is-literal_F632_basedpython.by.snap | 117 +++ .../rules/repeated_equality_comparison.rs | 1 + .../pyupgrade/rules/outdated_version_block.rs | 1 + .../ruff_linter/src/rules/refurb/helpers.rs | 4 + .../rules/single_item_membership_test.rs | 2 + .../rules/unnecessary_regular_expression.rs | 2 + crates/ruff_python_ast/ast.toml | 14 + crates/ruff_python_ast/src/comparable.rs | 9 + crates/ruff_python_ast/src/generated.rs | 15 + crates/ruff_python_ast/src/helpers.rs | 12 +- crates/ruff_python_ast/src/node.rs | 1 + crates/ruff_python_ast/src/nodes.rs | 82 +- crates/ruff_python_ast/src/visitor.rs | 1 + .../src/visitor/transformer.rs | 1 + crates/ruff_python_codegen/src/generator.rs | 27 +- .../test/fixtures/ruff/identity_operators.by | 16 +- .../src/expression/binary_like.rs | 74 +- .../src/expression/mod.rs | 1 + .../format@identity_operators.by.snap | 31 +- .../valid/expressions/identity_compare.py | 9 +- crates/ruff_python_parser/src/error.rs | 6 + .../src/parser/expression.rs | 52 +- .../ruff_python_parser/src/parser/helpers.rs | 11 +- ...arser__tests__ipython_escape_commands.snap | 1 + crates/ruff_python_parser/src/parser/tests.rs | 67 ++ ...__string__tests__parse_fstring_equals.snap | 1 + ...ring__tests__parse_fstring_not_equals.snap | 1 + ...__string__tests__parse_tstring_equals.snap | 1 + ...ring__tests__parse_tstring_not_equals.snap | 1 + ...xpressions__compare__invalid_order.py.snap | 2 + ...s__compare__invalid_rhs_expression.py.snap | 2 + ...xpressions__compare__missing_rhs_0.py.snap | 1 + ...xpressions__compare__missing_rhs_2.py.snap | 1 + ...essions__compare__named_expression.py.snap | 2 + ...sions__compare__starred_expression.py.snap | 4 + ...tax@expressions__dict__double_star.py.snap | 3 + ...__list__star_expression_precedence.py.snap | 1 + ...ressions__parenthesized__generator.py.snap | 1 + ..._parenthesized__tuple_starred_expr.py.snap | 4 + ...s__set__star_expression_precedence.py.snap | 1 + ...lid_syntax@for_stmt_invalid_target.py.snap | 1 + ...or_stmt_invalid_target_binary_expr.py.snap | 2 + ...for_stmt_invalid_target_in_keyword.py.snap | 6 + ...ements__invalid_assignment_targets.py.snap | 1 + ...nvalid_augmented_assignment_target.py.snap | 1 + ...id_syntax@while_stmt_missing_colon.py.snap | 1 + ...iguous_lpar_with_items_binary_expr.py.snap | 1 + .../valid_syntax@expressions__await.py.snap | 1 + .../valid_syntax@expressions__compare.py.snap | 17 + ...ressions__dictionary_comprehension.py.snap | 3 + ...valid_syntax@expressions__f_string.py.snap | 1 + ...alid_syntax@expressions__generator.py.snap | 3 + ...ntax@expressions__identity_compare.py.snap | 65 +- .../valid_syntax@expressions__if.py.snap | 2 + .../valid_syntax@expressions__list.py.snap | 1 + ...ax@expressions__list_comprehension.py.snap | 5 + ...tax@expressions__set_comprehension.py.snap | 3 + ...valid_syntax@expressions__t_string.py.snap | 1 + .../valid_syntax@expressions__yield.py.snap | 1 + ...lid_syntax@expressions__yield_from.py.snap | 1 + ...id_syntax@for_in_target_valid_expr.py.snap | 3 + ...tax@match_classify_as_identifier_1.py.snap | 1 + ...tax@match_classify_as_identifier_2.py.snap | 2 + ...atement__ambiguous_lpar_with_items.py.snap | 1 + .../valid_syntax@statement__assert.py.snap | 1 + .../valid_syntax@statement__for.py.snap | 1 + .../valid_syntax@statement__if.py.snap | 1 + .../valid_syntax@statement__match.py.snap | 3 + .../valid_syntax@statement__raise.py.snap | 2 + .../valid_syntax@statement__return.py.snap | 1 + .../valid_syntax@statement__type.py.snap | 2 + .../valid_syntax@statement__while.py.snap | 1 + crates/ty_ide/src/inlay_hints.rs | 10 +- .../resources/mdtest/basedpython_enums.md | 33 +- .../mdtest/basedpython_escaping_local.md | 2 +- .../mdtest/basedpython_identity_narrow.md | 89 +- .../mdtest/basedpython_inferred_narrowing.md | 6 +- .../mdtest/basedpython_parametric_is.md | 204 ++++- crates/ty_python_semantic/src/lib.rs | 5 +- crates/ty_python_semantic/src/reified.rs | 49 +- .../ty_python_semantic/src/semantic_model.rs | 128 ++- crates/ty_python_semantic/src/types.rs | 6 +- .../src/types/class/static_literal.rs | 9 +- .../src/types/conformance.rs | 10 - crates/ty_python_semantic/src/types/enums.rs | 28 +- .../ty_python_semantic/src/types/function.rs | 4 +- crates/ty_python_semantic/src/types/infer.rs | 9 + .../src/types/infer/builder.rs | 414 +++++----- .../src/types/infer/builder/class.rs | 4 +- .../src/types/infer/builder/function.rs | 8 +- .../types/infer/builder/type_expression.rs | 15 +- crates/ty_python_semantic/src/types/narrow.rs | 75 +- .../src/types/reified_infer.rs | 699 +++++++++++++++- .../ty_python_semantic/src/types/template.rs | 18 +- .../features/differences-from-python.md | 2 +- docs/basedpython/features/enums.md | 7 +- docs/basedpython/features/identity-swap.md | 109 ++- docs/basedpython/features/index.md | 2 +- zensical.toml | 2 +- 129 files changed, 3177 insertions(+), 1217 deletions(-) diff --git a/crates/by_transforms/src/reverse_transforms/identity_swap.rs b/crates/by_transforms/src/reverse_transforms/identity_swap.rs index d3616fec8c..23b9177827 100644 --- a/crates/by_transforms/src/reverse_transforms/identity_swap.rs +++ b/crates/by_transforms/src/reverse_transforms/identity_swap.rs @@ -7,12 +7,23 @@ //! basedpython's `is` is the instance check, so a python identity comparison //! round-trips to `===` / `!==` and an `isinstance` call round-trips to `is` //! -//! a literal right-hand side is left alone, mirroring the forward transform's -//! own literal guard: `x is None` is identity in both languages, so rewriting -//! it to `x === None` would churn idiomatic source for no change in meaning. -//! this is why the operator must be rewritten rather than skipped — leaving a -//! python `is not` in place re-reads it as `not isinstance(...)` on the way -//! back out +//! `x is None` is left alone: basedpython reads it as a type test against the +//! type `None`, which is the same runtime check, so rewriting it to +//! `x === None` would churn idiomatic source for no change in meaning. Every +//! *other* literal must still be rewritten — `x is 3` is identity in python but +//! a test for the type `Literal[3]` in basedpython, and those differ for an int +//! outside the interned range. +//! +//! This is also why the operator has to be rewritten rather than skipped: +//! leaving a python `is not` in place re-reads it as a type test on the way +//! back out. +//! +//! An `isinstance` call is only rewritten when its second argument is +//! something a type expression can say. A tuple of classes is `isinstance`'s +//! own spelling for a union and reads as a *tuple type* in a type expression, +//! so it is written back out as `A | B`; anything else — `type(y)`, a variable +//! holding classinfo — keeps the call it was, which basedpython runs just as +//! python does use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; @@ -45,9 +56,9 @@ impl<'src> IdentitySwapReverse<'src> { for (op, rhs) in c.ops.iter().zip(c.comparators.iter()) { let rhs_start = rhs.range().start(); let between = &self.source[usize::from(lhs_end)..usize::from(rhs_start)]; - // a literal rhs means identity in basedpython too, so the forward - // transform leaves it as `is` — mirror that and don't churn it - if !rhs.is_literal_expr() { + // `is None` reads the same in both languages; every other literal + // rhs changes meaning, so it takes the identity spelling + if !rhs.is_none_literal_expr() { let words: &[&str] = match op { CmpOp::Is => &["is"], CmpOp::IsNot => &["is", "not"], @@ -123,13 +134,13 @@ impl<'src> IdentitySwapReverse<'src> { let Some((x, y)) = isinstance_operands(call) else { return; }; - let (x_src, y_src) = ( - self.src(x.range()).to_owned(), - self.src(y.range()).to_owned(), - ); + let Some(target) = self.type_expression_target(y) else { + return; + }; + let x_src = self.src(x.range()).to_owned(); self.folded_into_not.push(call.range()); self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("{x_src} is not {y_src}"), + format!("{x_src} is not {target}"), unary.range(), ))); } @@ -141,15 +152,45 @@ impl<'src> IdentitySwapReverse<'src> { let Some((x, y)) = isinstance_operands(call) else { return; }; - let (x_src, y_src) = ( - self.src(x.range()).to_owned(), - self.src(y.range()).to_owned(), - ); + let Some(target) = self.type_expression_target(y) else { + return; + }; + let x_src = self.src(x.range()).to_owned(); self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("{x_src} is {y_src}"), + format!("{x_src} is {target}"), call.range(), ))); } + + /// how an `isinstance` classinfo argument is written as the type expression + /// a type test takes, or `None` when its shape is not one a type expression + /// has and the call has to stay a call. + /// + /// A tuple is `isinstance`'s spelling for "any of these", which a type + /// expression spells `A | B` — writing the tuple back out would name the + /// *tuple type* instead, a different test entirely. + fn type_expression_target(&self, classinfo: &Expr) -> Option { + match classinfo { + Expr::Name(_) | Expr::Attribute(_) | Expr::Subscript(_) => { + Some(self.src(classinfo.range()).to_owned()) + } + Expr::BinOp(binop) if binop.op == ruff_python_ast::Operator::BitOr => { + Some(self.src(classinfo.range()).to_owned()) + } + Expr::Tuple(tuple) => { + let arms: Option> = tuple + .elts + .iter() + .map(|element| self.type_expression_target(element)) + .collect(); + let arms = arms?; + // an empty `isinstance(x, ())` is always `False`; there is no + // union to write for it + (!arms.is_empty()).then(|| arms.join(" | ")) + } + _ => None, + } + } } /// the two operands of an `isinstance(x, y)` call. anything else — a keyword @@ -328,14 +369,43 @@ mod tests { ); } - /// a literal rhs is identity in both languages, so the forward transform - /// leaves it as `is` and the reverse must not churn it #[test] - fn literal_comparisons_left_alone() { + fn none_comparisons_left_alone() { + // basedpython reads `is None` as a test for the type `None`, which is + // the same runtime check python's identity performs check("y = a is None\n", "y = a is None\n"); check("y = a is not None\n", "y = a is not None\n"); - check("y = a is True\n", "y = a is True\n"); - check("y = a is not 1\n", "y = a is not 1\n"); + } + + #[test] + fn a_classinfo_tuple_becomes_a_union() { + // `isinstance`'s tuple means "any of these"; a type expression spells + // that `A | B`, and a tuple there is the tuple *type* + check("y = isinstance(a, (int, str))\n", "y = a is int | str\n"); + check( + "y = not isinstance(a, (int, (str, bytes)))\n", + "y = a is not int | str | bytes\n", + ); + } + + #[test] + fn a_classinfo_no_type_expression_can_say_keeps_the_call() { + // a call is not a type expression, and neither is the empty tuple — + // whose test is always `False` and has no union to write + check( + "y = isinstance(a, type(a))\n", + "y = isinstance(a, type(a))\n", + ); + check("y = isinstance(a, ())\n", "y = isinstance(a, ())\n"); + } + + #[test] + fn other_literal_comparisons_take_the_identity_operator() { + // `a is 1` is identity in python but a test for the type `Literal[1]` + // in basedpython, and those part company for an int outside the + // interned range — so the operator has to be written back out + check("y = a is True\n", "y = a === True\n"); + check("y = a is not 1\n", "y = a !== 1\n"); } /// the comment case cannot round-trip byte for byte — the operator's layout diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index 9766d46ea2..add7ce5cc3 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -111,6 +111,17 @@ pub(crate) struct PassContext { /// statement materialized inside an expression is a syntax error, not a /// composition pub(crate) statement_inserts: Vec<(TextSize, Vec)>, + /// Sub-statement edits standing in for a construct the same pass re-emits + /// somewhere else: the `_MISSING` a mutable default leaves in the + /// signature, whose written value the body guard evaluates instead. + /// + /// Identical to a [`template_edits`](Self::template_edits) entry except + /// that it leads every other edit at its span. The others are rewrites of + /// the construct, and the construct is no longer here — they materialize + /// where the pass re-emits it. Without this they would be ordered against + /// the substitution by shape alone, which cannot tell two substitutions of + /// one span apart + pub(crate) relocating_edits: Vec<(TextRange, Vec)>, /// Hard transpile errors a pass surfaced — abort the pipeline rather /// than emit partial / invalid output. Each entry is a human-readable /// message suitable for showing the user @@ -211,6 +222,11 @@ enum SubPatch { /// [`PassContext::statement_inserts`]. Materializes exactly like /// [`SubPatch::Template`]; the difference is only in what may claim it Statement(Vec), + /// a template standing in for a construct the same pass re-emits somewhere + /// else — see [`PassContext::relocating_edits`]. Materializes exactly like + /// [`SubPatch::Template`]; the difference is only that it leads every other + /// edit at its span + Relocating(Vec), } /// The sub-edits a template materializes, in position order: those nested in @@ -325,7 +341,9 @@ fn apply_within( out.push_source(source, cursor, s); match &all[idx].2 { SubPatch::Text(t) => out.push_generated(t, s), - SubPatch::Template(frags) | SubPatch::Statement(frags) => { + SubPatch::Template(frags) + | SubPatch::Statement(frags) + | SubPatch::Relocating(frags) => { let inner: Vec = contained[k + 1..] .iter() .copied() @@ -637,9 +655,9 @@ pub(crate) fn run_against_source<'a>( // keep their source bytes and the lowerings inside them compose &destructure_pass, // text-edit-emitting passes first (read source ranges). - // type_is must run before identity_swap so type-position `a is T` - // wins the first-wins overlap dedup over identity_swap's - // value-context `isinstance(a, T)` rewrite + // type_is rewrites a `-> a is T` return guard into `TypeIs[T]`, which + // claims the whole guard; `parametric_is` would otherwise lower the + // same `is` pair inside it &type_is_pass, // `from x export y` → `from x import y as y`: two source edits inside // an import statement, independent of every other pass @@ -1078,6 +1096,13 @@ pub(crate) fn run_against_source<'a>( let at = usize::from(at); (at, at, SubPatch::Statement(frags)) })) + .chain(ctx.relocating_edits.into_iter().map(|(r, frags)| { + ( + usize::from(r.start()), + usize::from(r.end()), + SubPatch::Relocating(frags), + ) + })) .collect(); // start asc. tie-break by edit shape: // 1. zero-width insertions first — they don't consume bytes, so any @@ -1087,30 +1112,34 @@ pub(crate) fn run_against_source<'a>( // 2. then wider replacements before narrower ones — so a wider edit // wins over (or, for templates, absorbs) a narrow one nested inside // it - // 3. at one identical span, a *substitution* — plain text, or a template - // with no `Src` passthrough — ahead of a *rewrite*, a template that - // re-emits part of the span. a substitution says the construct does not - // appear here at all, which a rewrite of it cannot outrank: the pass - // that substitutes may be relocating the construct (default - // re-evaluation moves a parameter default into the body), and the - // rewrite still materializes wherever the passthrough re-emits it + // 3. at one identical span, a *relocating* edit leads: it says the + // construct has moved, and the pass that moved it re-emits the span + // itself, so every other edit there materializes at the new home + // 4. then a *substitution* — plain text, or a template with no `Src` + // passthrough — ahead of a *rewrite*, a template that re-emits part of + // the span. a substitution says the construct does not appear here at + // all, which a rewrite of it cannot outrank sub_edits.sort_by(|a, b| { let priority = |e: &(usize, usize, SubPatch)| { let rewrites = i64::from(match &e.2 { SubPatch::Text(_) => false, - SubPatch::Template(frags) | SubPatch::Statement(frags) => { + SubPatch::Template(frags) + | SubPatch::Statement(frags) + | SubPatch::Relocating(frags) => { frags.iter().any(|frag| matches!(frag, Fragment::Src(_))) } }); let statement = i64::from(!matches!(e.2, SubPatch::Statement(_))); + let relocating = i64::from(!matches!(e.2, SubPatch::Relocating(_))); // (start, is_replacement_not_insertion, statement-insert-first, - // neg_end-for-wider-first, substitution-before-rewrite) + // neg_end-for-wider-first, relocating-first, + // substitution-before-rewrite) if e.1 == e.0 { - (e.0, 0i64, statement, 0i64, rewrites) // insertion + (e.0, 0i64, statement, 0i64, relocating, rewrites) // insertion } else { #[allow(clippy::cast_possible_wrap)] let neg_end = -(e.1 as i64); - (e.0, 1i64, statement, neg_end, rewrites) + (e.0, 1i64, statement, neg_end, relocating, rewrites) } }; priority(a).cmp(&priority(b)) @@ -1134,7 +1163,7 @@ pub(crate) fn run_against_source<'a>( } let is_template = matches!( sub_edits[i].2, - SubPatch::Template(_) | SubPatch::Statement(_) + SubPatch::Template(_) | SubPatch::Statement(_) | SubPatch::Relocating(_) ); for (m, edit) in sub_edits.iter().enumerate() { if m == i || claimed[m] { @@ -1182,7 +1211,9 @@ pub(crate) fn run_against_source<'a>( if !claimed[j] { match &sub_edits[j].2 { SubPatch::Text(t) => combined.push_generated(t, start), - SubPatch::Template(frags) | SubPatch::Statement(frags) => { + SubPatch::Template(frags) + | SubPatch::Statement(frags) + | SubPatch::Relocating(frags) => { let contained = template_claimees(frags, &sub_edits, &claimed, j, None); materialize_fragments( &mut combined, @@ -1204,7 +1235,9 @@ pub(crate) fn run_against_source<'a>( let repl = match &sub_edits[i].2 { // a plain-text replacement wins over anything inside it SubPatch::Text(t) => Replacement::generated(t, start), - SubPatch::Template(frags) | SubPatch::Statement(frags) => { + SubPatch::Template(frags) + | SubPatch::Statement(frags) + | SubPatch::Relocating(frags) => { // the claimees nested in this span materialize inside the // template's `Src` passthrough fragments let contained = diff --git a/crates/by_transforms/src/transforms/checked_cast.rs b/crates/by_transforms/src/transforms/checked_cast.rs index 24bb5c43d5..f26717d3af 100644 --- a/crates/by_transforms/src/transforms/checked_cast.rs +++ b/crates/by_transforms/src/transforms/checked_cast.rs @@ -20,12 +20,10 @@ //! still compose and it is evaluated exactly once. //! //! How a *checked* form validates is decided by the **same engine that decides -//! `x is T`** — [`build_predicate`], via [`TypeInfo::parametric_cast_plan`]. The -//! two forms ask one question (does this value satisfy this specialization at -//! runtime) and differ only in two parameters: +//! `x is T`** — [`build_predicate`], via [`TypeInfo::parametric_is_plan`]. The +//! two forms ask one question (does this value satisfy this type at runtime) +//! over the same kind of expression, and differ in one parameter: //! -//! - [`TargetPosition::Type`], because a cast's target is a *type* expression -//! while an `is`-rhs is a value expression, so ty infers it differently; //! - [`ProbeStrictness::Lenient`], because a cast is an assertion: arguments the //! runtime cannot see are not held against the value, keeping //! `[1, 2] cast! list[int]` legal. An `is`-test is strict — a `True` narrows, @@ -52,9 +50,7 @@ use ruff_python_ast::{CastKind, Expr, Stmt}; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; -use super::parametric_is::{ - PARAMETRIC_IS_RUNTIME, PROTOCOL_IS_RUNTIME, ProbeStrictness, TargetPosition, build_predicate, -}; +use super::parametric_is::{PredicateRuntime, ProbeStrictness, build_predicate}; use crate::type_info::{CastCheck, SoundnessCheck, TypeInfo}; /// the lambda parameter a predicate-form cast binds its value to, so the value @@ -151,9 +147,8 @@ struct CastLower<'a> { types: &'a dyn TypeInfo, edits: Vec<(TextRange, Vec)>, used: BTreeSet, - needs_parametric: bool, - needs_protocol: bool, - needs_conformance: bool, + /// the runtime helpers the emitted predicates call + runtimes: BTreeSet, } impl<'a> CastLower<'a> { @@ -162,9 +157,7 @@ impl<'a> CastLower<'a> { types, edits: Vec::new(), used: BTreeSet::new(), - needs_parametric: false, - needs_protocol: false, - needs_conformance: false, + runtimes: BTreeSet::new(), } } @@ -189,13 +182,10 @@ impl<'a> CastLower<'a> { &value_ref, value_arg, type_arg, - TargetPosition::Type, ProbeStrictness::Lenient, ); if !needs.all_plain && !needs.erased { - self.needs_parametric |= needs.parametric_runtime; - self.needs_protocol |= needs.protocol_runtime; - self.needs_conformance |= needs.conformance_runtime; + self.runtimes.extend(PredicateRuntime::used(&needs)); let mut fragments = vec![Fragment::Lit(format!("lambda {CAST_VALUE_PARAM}: "))]; fragments.extend(predicate); return (helper.as_predicate(), fragments); @@ -292,15 +282,8 @@ impl TypeAwarePass for CheckedCastPass { } // `_parametric_is` / `_by_protocol_is` must precede the predicates that // call them - if inner.needs_parametric { - ctx.required_imports.push(PARAMETRIC_IS_RUNTIME.to_owned()); - } - if inner.needs_protocol { - ctx.required_imports.push(PROTOCOL_IS_RUNTIME.to_owned()); - } - if inner.needs_conformance { - ctx.required_imports - .push(super::conformance::WITNESS_RUNTIME.to_owned()); + for runtime in inner.runtimes { + ctx.required_imports.push(runtime.source().to_owned()); } for helper in &inner.used { ctx.required_imports.push(helper.runtime().to_owned()); @@ -371,9 +354,13 @@ mod tests { let out = check( "from typing import Literal\n\ndef f(a: object):\n b = a cast! Literal[\"x\", \"y\"]\n", ); + // each arm pins the class as well as the value: python's `1 == True` + // would otherwise let a `bool` satisfy `Literal[1]` assert!( out.contains( - "b = _checked_cast_pred(a, lambda _by_cast_value: _by_cast_value in (\"x\", \"y\"))" + "b = _checked_cast_pred(a, lambda _by_cast_value: \ + ((type(_by_cast_value) is str and _by_cast_value == \"x\") or \ + (type(_by_cast_value) is str and _by_cast_value == \"y\")))" ), "got:\n{out}" ); @@ -387,7 +374,9 @@ mod tests { ); assert!( out.contains( - "b = _try_cast_pred(a, lambda _by_cast_value: _by_cast_value in (\"x\", \"y\"))" + "b = _try_cast_pred(a, lambda _by_cast_value: \ + ((type(_by_cast_value) is str and _by_cast_value == \"x\") or \ + (type(_by_cast_value) is str and _by_cast_value == \"y\")))" ), "got:\n{out}" ); @@ -400,7 +389,9 @@ mod tests { let out = check("from typing import Literal\n\ndef f(a: object):\n b = a cast? Literal[7]\n"); assert!( - out.contains("lambda _by_cast_value: _by_cast_value in (7,)"), + out.contains( + "lambda _by_cast_value: (type(_by_cast_value) is int and _by_cast_value == 7)" + ), "got:\n{out}" ); } @@ -411,7 +402,10 @@ mod tests { "from typing import Literal\n\ndef f(a: object):\n b = a cast? Literal[True] | None\n", ); assert!( - out.contains("lambda _by_cast_value: _by_cast_value in (True, None)"), + out.contains( + "lambda _by_cast_value: ((type(_by_cast_value) is bool and \ + _by_cast_value == True) or _by_cast_value is None)" + ), "got:\n{out}" ); } @@ -488,7 +482,7 @@ mod tests { fn union_arms_are_decomposed() { let out = check("def f(a: object):\n b = a cast? list[int] | None\n"); assert!( - out.contains("b = _try_cast_pred(a, lambda _by_cast_value: _parametric_is_lenient(_by_cast_value, list[int], (0,)) or _by_cast_value is None)"), + out.contains("b = _try_cast_pred(a, lambda _by_cast_value: (_parametric_is_lenient(_by_cast_value, list[int], (0,)) or _by_cast_value is None))"), "got:\n{out}" ); } @@ -684,8 +678,8 @@ mod tests { assert!( out.contains( "b = _checked_cast_pred(a, lambda _by_cast_value: \ - _parametric_is_lenient(_by_cast_value, A[int], (0,)) \ - or isinstance(_by_cast_value, str))" + (_parametric_is_lenient(_by_cast_value, A[int], (0,)) \ + or isinstance(_by_cast_value, str)))" ), "each arm lowered by its own kind: {out}" ); diff --git a/crates/by_transforms/src/transforms/coalesce_chain.rs b/crates/by_transforms/src/transforms/coalesce_chain.rs index 25c1b5ea4d..d7c6898b02 100644 --- a/crates/by_transforms/src/transforms/coalesce_chain.rs +++ b/crates/by_transforms/src/transforms/coalesce_chain.rs @@ -384,6 +384,9 @@ fn compare_is_not_none(left: Expr) -> Expr { range: TextRange::default(), left: Box::new(left), ops: Box::new([CmpOp::IsNot]), + // python identity against `None`, written `is not` — the lowering emits + // python, so it never wants basedpython's `!==` spelling + identity_ops: None, comparators: Box::new([Expr::NoneLiteral(ExprNoneLiteral { node_index: AtomicNodeIndex::NONE, range: TextRange::default(), diff --git a/crates/by_transforms/src/transforms/context_sensitive.rs b/crates/by_transforms/src/transforms/context_sensitive.rs index dcfc4b656e..d08fc82175 100644 --- a/crates/by_transforms/src/transforms/context_sensitive.rs +++ b/crates/by_transforms/src/transforms/context_sensitive.rs @@ -209,9 +209,10 @@ mod tests { indoc! {" from __future__ import annotations from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class Shape: - pass + Circle: ClassVar[type[_Shape_Circle]] + Empty: ClassVar[_Shape_Empty] @final @dataclass(frozen=True, slots=True) @@ -308,9 +309,10 @@ mod tests { indoc! {" from __future__ import annotations from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class Shape: - pass + Circle: ClassVar[type[_Shape_Circle]] + Empty: ClassVar[_Shape_Empty] @final @dataclass(frozen=True, slots=True) diff --git a/crates/by_transforms/src/transforms/enums.rs b/crates/by_transforms/src/transforms/enums.rs index 08baf667da..635a54f835 100644 --- a/crates/by_transforms/src/transforms/enums.rs +++ b/crates/by_transforms/src/transforms/enums.rs @@ -414,6 +414,26 @@ fn emit_sealed_hierarchy( .unwrap_or_default(); out.push_gen(&format!("{vis}class {name}{params}:\n")); + // each variant is *declared* in the body and *assigned* below it. the + // assignment is what the runtime needs; the declaration is what lets a + // reader — and the checker — see that `Shape.Point` is a value and + // `Shape.Circle` a class, which an assignment from outside the body does + // not say. the annotations are strings (the lowering emits `from __future__ + // import annotations`), so naming a class defined further down costs + // nothing at run time + if !variants.is_empty() { + imports.add("typing", "ClassVar"); + } + for variant in variants { + let variant_class = format!("_{name}_{}", variant.name); + let declared = match variant.kind { + // a payload variant is the class its call constructs + VariantKind::Tuple => format!("type[{variant_class}]"), + // a unit variant is the one instance of its class + VariantKind::Unit => variant_class, + }; + out.push_gen(&format!(" {}: ClassVar[{declared}]\n", variant.name)); + } // ordinary members (methods, classmethods, constants) — copied verbatim, // already indented under the enum in the source. they may refer to variants // (`A.Foo`) freely: the references resolve lazily at call time, by which @@ -421,7 +441,7 @@ fn emit_sealed_hierarchy( for member in members { emit_member(out, source, member); } - if members.is_empty() { + if members.is_empty() && variants.is_empty() { out.push_gen(" pass\n"); } // variant subclasses, emitted at module level and attached to the enum @@ -722,9 +742,9 @@ mod tests { indoc! {" from __future__ import annotations from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class _Shape: - pass + Circle: ClassVar[type[_Shape_Circle]] @final @dataclass(frozen=True, slots=True) @@ -751,7 +771,9 @@ mod tests { "}, indoc! {" from __future__ import annotations + from typing import ClassVar class E: + A: ClassVar[_E_A] MAX: int = 10 class _E_A(E): @@ -776,9 +798,10 @@ mod tests { indoc! {" from __future__ import annotations from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class Shape: - pass + Circle: ClassVar[type[_Shape_Circle]] + Point: ClassVar[_Shape_Point] @final @dataclass(frozen=True, slots=True) @@ -812,9 +835,10 @@ mod tests { indoc! {" from __future__ import annotations from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class Shape: - pass + Rectangle: ClassVar[type[_Shape_Rectangle]] + Polygon: ClassVar[type[_Shape_Polygon]] @final @dataclass(frozen=True, slots=True) @@ -848,9 +872,10 @@ mod tests { indoc! {" from __future__ import annotations from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class Value: - pass + Pair: ClassVar[type[_Value_Pair]] + Nothing: ClassVar[_Value_Nothing] @final @dataclass(frozen=True, slots=True) @@ -888,8 +913,9 @@ mod tests { from __future__ import annotations from ty_extensions import JustFloat from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class Shape: + Circle: ClassVar[type[_Shape_Circle]] def area(self) -> JustFloat: return 0.0 @@ -940,9 +966,9 @@ mod tests { indoc! {" from __future__ import annotations from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class Shape: - pass + Circle: ClassVar[type[_Shape_Circle]] @final @dataclass(frozen=True) @@ -969,9 +995,10 @@ mod tests { indoc! {" from __future__ import annotations from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class E: - pass + A: ClassVar[type[_E_A]] + B: ClassVar[_E_B] @final @dataclass(frozen=True, slots=True) @@ -1008,11 +1035,12 @@ mod tests { from __future__ import annotations from typing import TypeVar, Generic from dataclasses import dataclass - from typing import final + from typing import final, ClassVar _T = TypeVar(\"_T\") _E = TypeVar(\"_E\") class Result(Generic[_T, _E]): - pass + Ok: ClassVar[type[_Result_Ok]] + Err: ClassVar[type[_Result_Err]] @final @dataclass(frozen=True, slots=True) @@ -1045,9 +1073,10 @@ mod tests { indoc! {" from __future__ import annotations from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class Result[T, E]: - pass + Ok: ClassVar[type[_Result_Ok]] + Err: ClassVar[type[_Result_Err]] @final @dataclass(frozen=True, slots=True) @@ -1083,9 +1112,10 @@ mod tests { indoc! {" from __future__ import annotations from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class Tree[T]: - pass + Leaf: ClassVar[_Tree_Leaf] + Node: ClassVar[type[_Tree_Node]] class _Tree_Leaf(Tree): __slots__ = () diff --git a/crates/by_transforms/src/transforms/identity_swap.rs b/crates/by_transforms/src/transforms/identity_swap.rs index e815ae55bb..4501643eb7 100644 --- a/crates/by_transforms/src/transforms/identity_swap.rs +++ b/crates/by_transforms/src/transforms/identity_swap.rs @@ -1,24 +1,18 @@ -//! `===` / `!==` are real python identity; `is` / `is not` mean -//! `isinstance` / `not isinstance`. parser flattens both spellings to -//! `CmpOp::Is`/`IsNot`, so disambiguation reads the operator text from -//! source. +//! `===` / `!==` are real python identity, which python spells `is` / `is not`. +//! basedpython gives the `is` keyword to the type test instead, and the parser +//! folds both spellings onto the same [`CmpOp`], recording which one it saw. +//! +//! This pass lowers only the identity spelling, by replacing the operator text. +//! Every type test belongs to [`parametric_is`](super::parametric_is), which +//! decides its lowering from the target's *type* rather than from the shape the +//! target was written in. use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{ - Arguments, AtomicNodeIndex, CmpOp, Expr, ExprCall, ExprContext, ExprName, ExprUnaryOp, - ModModule, Operator, Stmt, UnaryOp, name::Name, -}; +use ruff_python_ast::{CmpOp, Expr, ModModule, Stmt}; +use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; use ruff_text_size::{Ranged, TextRange, TextSize}; -use super::ast_driver::{AstPass, PassContext, render_expr}; - -/// whether the type-aware `parametric_is` pass owns lowering this `is`-rhs, so -/// `identity_swap` must leave it alone. it owns a name / attribute / subscript -/// target (may name a specialization or alias) and a `|` union of them -fn owned_by_parametric_is(rhs: &Expr) -> bool { - matches!(rhs, Expr::Name(_) | Expr::Attribute(_) | Expr::Subscript(_)) - || matches!(rhs, Expr::BinOp(binop) if binop.op == Operator::BitOr) -} +use super::ast_driver::{AstPass, PassContext}; pub(crate) struct IdentitySwap<'src> { source: &'src str, @@ -51,93 +45,64 @@ struct State<'src> { impl State<'_> { fn process_compare(&mut self, c: &ruff_python_ast::ExprCompare) { let mut lhs_end = c.left.range().end(); - let mut lhs: &Expr = c.left.as_ref(); - for (op, rhs) in c.ops.iter().zip(c.comparators.iter()) { - let rhs_start = rhs.range().start(); - let between = &self.source[usize::from(lhs_end)..usize::from(rhs_start)]; - let trimmed = between.trim(); - match op { - CmpOp::Is => { - if trimmed.starts_with("===") { - if let Some(pos) = between.find("===") { - let op_start = lhs_end + TextSize::try_from(pos).unwrap(); - let op_range = - TextRange::new(op_start, op_start + TextSize::from(3u32)); - self.edits.push((op_range, "is".to_owned())); - } - } else if trimmed == "is" - && !rhs.is_literal_expr() - && !owned_by_parametric_is(rhs) - { - let call = isinstance_call(lhs.clone(), rhs.clone(), false); - let pair_range = TextRange::new(lhs.range().start(), rhs.range().end()); - self.edits.push((pair_range, render_expr(&call))); - } + for (index, rhs) in c.comparators.iter().enumerate() { + let gap = TextRange::new(lhs_end, rhs.range().start()); + if c.is_identity_operator(index) { + let replacement = match c.ops.get(index) { + Some(CmpOp::Is) => "is", + Some(CmpOp::IsNot) => "is not", + _ => unreachable!("only `is` / `is not` carry the identity spelling"), + }; + if let Some(range) = identity_operator_range(self.source, gap) { + // `===` needs no space around it and `is` does, so an + // operator the source wrote tight against its operands + // (`a===b`) has to bring its own + let before = self.source[..usize::from(range.start())] + .ends_with(|c: char| c.is_whitespace()); + let after = self.source[usize::from(range.end())..] + .starts_with(|c: char| c.is_whitespace()); + let padded = format!( + "{}{replacement}{}", + if before { "" } else { " " }, + if after { "" } else { " " }, + ); + self.edits.push((range, padded)); } - CmpOp::IsNot => { - if trimmed.starts_with("!==") { - if let Some(pos) = between.find("!==") { - let op_start = lhs_end + TextSize::try_from(pos).unwrap(); - let op_range = - TextRange::new(op_start, op_start + TextSize::from(3u32)); - self.edits.push((op_range, "is not".to_owned())); - } - } else if !rhs.is_literal_expr() && !owned_by_parametric_is(rhs) { - let call = isinstance_call(lhs.clone(), rhs.clone(), true); - let pair_range = TextRange::new(lhs.range().start(), rhs.range().end()); - self.edits.push((pair_range, render_expr(&call))); - } - } - _ => {} } lhs_end = rhs.range().end(); - lhs = rhs; } } } -fn isinstance_call(lhs: Expr, rhs: Expr, negate: bool) -> Expr { - let call = Expr::Call(ExprCall { - node_index: AtomicNodeIndex::NONE, - range_start: ruff_text_size::TextSize::default(), - func: Box::new(Expr::Name(ExprName { - node_index: AtomicNodeIndex::NONE, - range: TextRange::default(), - id: Name::from("isinstance"), - ctx: ExprContext::Load, - })), - arguments: Arguments { - node_index: AtomicNodeIndex::NONE, - range: TextRange::default(), - args: Box::new([lhs, rhs]), - keywords: thin_vec::ThinVec::new(), - }, - cast_kind: None, - is_string_tag: false, - }); - if negate { - Expr::UnaryOp(ExprUnaryOp { - node_index: AtomicNodeIndex::NONE, - range: TextRange::default(), - op: UnaryOp::Not, - operand: Box::new(call), - }) - } else { - call - } +/// the source range of the `===` / `!==` written in `gap` — the span between +/// the two operands it joins. +/// +/// The operand ranges stop inside any parentheses wrapping them, and a comment +/// may sit in the gap of a bracketed expression, so the operator is found by +/// tokenizing rather than by searching for its text. +fn identity_operator_range(source: &str, gap: TextRange) -> Option { + let start = SimpleTokenizer::new(source, gap) + .skip_trivia() + .find(|token| token.kind() != SimpleTokenKind::RParen)? + .start(); + let rest = &source[usize::from(start)..]; + ["===", "!=="] + .into_iter() + .find(|symbol| rest.starts_with(symbol)) + .map(|_| TextRange::at(start, TextSize::from(3u32))) } impl<'ast> Visitor<'ast> for State<'_> { + fn visit_stmt(&mut self, stmt: &'ast Stmt) { + walk_stmt(self, stmt); + } + fn visit_expr(&mut self, expr: &'ast Expr) { - if let Expr::Compare(c) = expr { - self.process_compare(c); + if let Expr::Compare(compare) = expr { + self.process_compare(compare); } walk_expr(self, expr); } - - fn visit_stmt(&mut self, stmt: &'ast Stmt) { - walk_stmt(self, stmt); - } } #[cfg(test)] @@ -167,6 +132,14 @@ mod tests { check("x is int\n", "isinstance(x, int)\n"); } + #[test] + fn parenthesized_operands_still_lower() { + // an operand's own range stops inside its parentheses, so the rewrite + // has to close what it swallowed and open what the source closes after + check("(x) is int\n", "(isinstance(x, int))\n"); + check("x is ( int )\n", "(isinstance(x, int) )\n"); + } + #[test] fn is_not_to_not_isinstance() { check("x is not int\n", "not isinstance(x, int)\n"); @@ -188,24 +161,21 @@ mod tests { } #[test] - fn is_bool_kept() { - check("a is True\n", "a is True\n"); - check("a is False\n", "a is False\n"); - } - - #[test] - fn is_number_kept() { - check("a is 0\n", "a is 0\n"); - } - - #[test] - fn is_string_kept() { - check("a is \"x\"\n", "a is \"x\"\n"); - } - - #[test] - fn is_ellipsis_kept() { - check("a is ...\n", "a is ...\n"); + fn is_literal_tests_the_literal_type() { + // a literal names a type holding exactly the values equal to it, and the + // class guard is what keeps python's `1 == True` from widening that + check( + "def f(a: object):\n return a is True\n", + "def f(a: object):\n return (type(a) is bool and a == True)\n", + ); + check( + "def f(a: object):\n return a is 0\n", + "def f(a: object):\n return (type(a) is int and a == 0)\n", + ); + check( + "def f(a: object):\n return a is \"x\"\n", + "def f(a: object):\n return (type(a) is str and a == \"x\")\n", + ); } #[test] @@ -218,6 +188,13 @@ mod tests { check("a !== None\n", "a is not None\n"); } + #[test] + fn an_unspaced_identity_operator_brings_its_own_spaces() { + // `===` needs no space around it; `is` does + check("c = a===b\n", "c = a is b\n"); + check("d = a!==b\n", "d = a is not b\n"); + } + #[test] fn python_eq_unchanged() { unchanged("a == b\n"); diff --git a/crates/by_transforms/src/transforms/init_method.rs b/crates/by_transforms/src/transforms/init_method.rs index 9f74d082f3..a22b68bf84 100644 --- a/crates/by_transforms/src/transforms/init_method.rs +++ b/crates/by_transforms/src/transforms/init_method.rs @@ -56,6 +56,7 @@ impl TypeAwarePass for InitMethod<'_> { float_literals: self.float_literals, edits: RefCell::new(Vec::new()), templates: RefCell::new(Vec::new()), + relocating: RefCell::new(Vec::new()), errors: RefCell::new(Vec::new()), needs_missing: RefCell::new(false), }; @@ -67,6 +68,7 @@ impl TypeAwarePass for InitMethod<'_> { } ctx.text_edits.extend(state.edits.into_inner()); ctx.template_edits.extend(state.templates.into_inner()); + ctx.relocating_edits.extend(state.relocating.into_inner()); ctx.errors.extend(state.errors.into_inner()); } } @@ -101,6 +103,8 @@ struct State<'src> { float_literals: FloatLiteralLowering, edits: RefCell>, templates: RefCell)>>, + /// the `_MISSING` substitutions, whose defaults the guards re-evaluate + relocating: RefCell)>>, errors: RefCell>, /// whether the `_MISSING` sentinel reached the output needs_missing: RefCell, @@ -302,7 +306,11 @@ impl State<'_> { // the whole body is written here, so the guards a defaulted or // relaxed-order parameter needs are written here too — `mutable_ // defaults` has no source statement to anchor them before - let (sentinels, guards) = parameter_guards(func, self.types); + let super::mutable_defaults::ParameterGuards { + sentinels, + written, + guards, + } = parameter_guards(func, self.types); let header_indent = self.line_indent(func.range.start()).to_owned(); let body_indent = format!("{header_indent} "); let mut frags = vec![Fragment::Lit(":".to_owned())]; @@ -319,7 +327,8 @@ impl State<'_> { } // an inherited default is a signature edit that needs no guard, so the sentinels // are written whether or not the body gains anything - self.templates.borrow_mut().extend(sentinels); + self.relocating.borrow_mut().extend(sentinels); + self.templates.borrow_mut().extend(written); let pos = func.range.end(); if guards.is_empty() { // no passthrough to carry, so emit plain text: a template diff --git a/crates/by_transforms/src/transforms/mutable_defaults.rs b/crates/by_transforms/src/transforms/mutable_defaults.rs index 9f1d129c21..ccd776af42 100644 --- a/crates/by_transforms/src/transforms/mutable_defaults.rs +++ b/crates/by_transforms/src/transforms/mutable_defaults.rs @@ -88,6 +88,8 @@ struct MutableDefaults<'src> { source: &'src str, types: &'src dyn TypeInfo, edits: Vec<(TextRange, Vec)>, + /// the `_MISSING` substitutions, whose defaults the guards re-evaluate + relocating: Vec<(TextRange, Vec)>, /// the guard suites, anchored at the body statement they precede guards: Vec<(TextSize, Vec)>, used: bool, @@ -122,11 +124,21 @@ fn written_default(pw: &ParameterWithDefault, value: &str) -> (TextRange, Vec (Vec<(TextRange, Vec)>, Vec) { +pub(crate) struct ParameterGuards { + /// the `_MISSING` left where a written default stood. *relocating*: the + /// guard evaluates the default's own source instead, so at this span this + /// edit leads every lowering written inside the default + pub(crate) sentinels: Vec<(TextRange, Vec)>, + /// a default written into the signature that the source did not write — an + /// inherited one, or the `_MISSING` a relaxed-order parameter needs. these + /// relocate nothing; they add text at the end of a parameter + pub(crate) written: Vec<(TextRange, Vec)>, + pub(crate) guards: Vec, +} + +pub(crate) fn parameter_guards(f: &StmtFunctionDef, types: &dyn TypeInfo) -> ParameterGuards { let mut sentinels = Vec::new(); + let mut written = Vec::new(); let mut guards = Vec::new(); let params = f.parameters.as_ref(); // positional parameters: swap non-scalar defaults for the sentinel, and give @@ -150,10 +162,10 @@ pub(crate) fn parameter_guards( // the parameters after it "after a default", exactly as a written one would None if let Some(value) = types.inherited_parameter_default(pw) => { seen_default = true; - sentinels.push(written_default(pw, &value)); + written.push(written_default(pw, &value)); } None if seen_default => { - sentinels.push(written_default(pw, "_MISSING")); + written.push(written_default(pw, "_MISSING")); guards.push(Guard::Required { name: pw.parameter.name.id.to_string(), function: f.name.id.to_string(), @@ -174,12 +186,16 @@ pub(crate) fn parameter_guards( Some(_) => {} None => { if let Some(value) = types.inherited_parameter_default(pw) { - sentinels.push(written_default(pw, &value)); + written.push(written_default(pw, &value)); } } } } - (sentinels, guards) + ParameterGuards { + sentinels, + written, + guards, + } } /// Whether the *callee's* body could evaluate `default` at all. @@ -237,8 +253,13 @@ impl MutableDefaults<'_> { if is_bodyless_init_shorthand(f) { return; } - let (sentinels, guards) = parameter_guards(f, self.types); - self.edits.extend(sentinels); + let ParameterGuards { + sentinels, + written, + guards, + } = parameter_guards(f, self.types); + self.relocating.extend(sentinels); + self.edits.extend(written); if guards.is_empty() { return; } @@ -281,6 +302,7 @@ impl TypeAwarePass for MutableDefaultsPass<'_> { source: self.source, types, edits: Vec::new(), + relocating: Vec::new(), guards: Vec::new(), used: false, unanchored: Vec::new(), @@ -303,6 +325,7 @@ impl TypeAwarePass for MutableDefaultsPass<'_> { ctx.required_imports.push("_MISSING = object()".to_owned()); } ctx.template_edits.extend(inner.edits); + ctx.relocating_edits.extend(inner.relocating); ctx.statement_inserts.extend(inner.guards); } } @@ -880,7 +903,8 @@ mod tests { fn default_lowerings_survive() { // the default is re-emitted in the body through a `Src` passthrough, so // the lowerings written inside it land in the guard rather than being - // dropped with the signature they came from + // dropped with the signature they came from. `1 is int` is a type test + // the checker settles, so what lands is the constant it settled on check( indoc! {" def f(x = [1 is int]): @@ -890,7 +914,7 @@ mod tests { _MISSING = object() def f(x = _MISSING): if x is _MISSING: - x = [isinstance(1, int)] + x = [True] return x "}, ); @@ -899,8 +923,9 @@ mod tests { #[test] fn default_lowering_spanning_the_whole_default_survives() { // the sentinel and the `is` lowering claim the *same* span. the sentinel - // substitutes it and the lowering rewrites it, so the substitution - // decides the signature and the rewrite materializes in the guard + // relocates the default, so it decides the signature and the lowering + // materializes in the guard — which holds even here, where the lowering + // settles to a constant and so replaces the span just as flatly check( indoc! {" def f(x = 1 is int): @@ -910,7 +935,7 @@ mod tests { _MISSING = object() def f(x = _MISSING): if x is _MISSING: - x = isinstance(1, int) + x = True return x "}, ); diff --git a/crates/by_transforms/src/transforms/parametric_is.rs b/crates/by_transforms/src/transforms/parametric_is.rs index d5667430ee..77be606b25 100644 --- a/crates/by_transforms/src/transforms/parametric_is.rs +++ b/crates/by_transforms/src/transforms/parametric_is.rs @@ -42,22 +42,20 @@ //! //! [`build_predicate`] is the core of this pass *and* of //! [`checked_cast`](super::checked_cast): both ask one question — does this -//! value satisfy this specialization at runtime — over the same -//! [`ParametricIsPlan`]. they differ only in two parameters: -//! -//! - [`TargetPosition`], because a cast's target is a type expression while an -//! `is`-rhs is a value expression, so ty infers the two differently -//! - [`ProbeStrictness`], because an `is`-test must *earn* a `True` (it narrows) -//! while a cast is an assertion that only holds the value to arguments the -//! runtime can actually see +//! value satisfy this type at runtime — over the same [`ParametricIsPlan`], +//! built from the same type expression. they differ in one parameter, +//! [`ProbeStrictness`]: an `is`-test must *earn* a `True` (it narrows) while a +//! cast is an assertion that only holds the value to arguments the runtime can +//! actually see +use std::collections::BTreeSet; use std::fmt::Write as _; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{self as ast, CmpOp, Expr, Stmt}; +use ruff_python_ast::{self as ast, CmpOp, Expr, PySourceType, Stmt}; +use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; use ruff_text_size::{Ranged, TextRange}; -use ty_python_semantic::reified::is_keyword_comparison; -use ty_python_semantic::{ArgVariance, ParametricIsPlan, ProtocolMemberCheck}; +use ty_python_semantic::{ArgVariance, ParametricIsPlan, ProtocolMemberCheck, TargetSpelling}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use crate::type_info::TypeInfo; @@ -257,7 +255,7 @@ def _parametric_is_lenient(value, alias, variances): /// covariant → subtype, 2 contravariant → supertype, 3 bivariant → any). /// annotations are read with `typing.get_type_hints` (resolving string /// annotations and inherited members), falling back to a raw `__mro__` walk -pub(crate) const PROTOCOL_IS_RUNTIME: &str = "\ +const PROTOCOL_IS_RUNTIME: &str = "\ _by_proto_missing = object() def _by_member_annotation(klass, name): @@ -392,6 +390,20 @@ def _by_protocol_is(value, members): return True "; +/// matches a value against a template literal type — a pattern such as +/// `f"a{int}b"`, whose type is the set of strings it can produce. +/// +/// the regular expression comes from the checker, which builds it from the same +/// reading of the pattern's holes that decides the static answer, so the test +/// accepts exactly the strings the type contains. a non-`str` value is not one +/// of them +const PATTERN_IS_RUNTIME: &str = "\ +import re as _by_re + +def _by_pattern_is(value, pattern): + return isinstance(value, str) and _by_re.fullmatch(pattern, value) is not None +"; + /// the runtime variance code `_by_variance_ok` expects fn variance_code(variance: ArgVariance) -> u8 { match variance { @@ -437,17 +449,6 @@ fn protocol_members_literal(checks: &[ProtocolMemberCheck]) -> String { format!("[{entries}]") } -/// which inference position a target expression lives in. this is the *only* -/// difference between an `is`-test and a checked cast at the front end: an -/// `is`-rhs is a value expression (its type is the class object), a `cast` -/// target is a type expression (its type is the instance). both then classify -/// through the same engine -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum TargetPosition { - Value, - Type, -} - /// how a runtime probe treats a value that carries no reification #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum ProbeStrictness { @@ -474,6 +475,8 @@ pub(crate) struct PredicateNeeds { pub(crate) protocol_runtime: bool, /// the predicate calls `_by_conforms` pub(crate) conformance_runtime: bool, + /// the predicate calls `_by_pattern_is` + pub(crate) pattern_runtime: bool, /// no arm carried a parametric claim — a plain `isinstance` covers the whole /// target, so a caller may use its compact shallow form instead pub(crate) all_plain: bool, @@ -497,72 +500,21 @@ impl PredicateNeeds { } } -/// the plan for one `(value, target)` pair, resolved through the position's -/// inference rules -fn plan_for( - types: &dyn TypeInfo, - value_expr: &Expr, - target: &Expr, - position: TargetPosition, -) -> Option { - match position { - TargetPosition::Value => types.parametric_is_plan(value_expr, target), - TargetPosition::Type => types.parametric_cast_plan(value_expr, target), - } -} - /// the bare (non-negated) runtime predicate for one target arm, referencing the /// value through `value_ref` so the caller controls how it is bound. /// /// this is the shared core of `x is T` and `x cast T`: both ask the same -/// question — does this value satisfy this specialization at runtime — and -/// differ only in `position` (how the target is inferred) and `probe` (what an -/// unreified value means). +/// question — does this value satisfy this type at runtime — and differ only in +/// `probe`, which decides what an unreified value means. fn arm_predicate( types: &dyn TypeInfo, value_ref: &dyn Fn() -> Fragment, value_expr: &Expr, arm: &Expr, - position: TargetPosition, probe: ProbeStrictness, needs: &mut PredicateNeeds, ) -> Vec { - // a `None` arm (an `X | None` optional) is an identity check, not - // `isinstance(_, None)` — `None` is a value, not a class - if matches!(arm, Expr::NoneLiteral(_)) { - needs.all_true = false; - needs.references_value = true; - return vec![value_ref(), Fragment::Lit(" is None".to_owned())]; - } - // an interface something visibly conforms to cannot be answered by - // `isinstance`: a conforming type is not a subclass, and a protocol is not - // even a legal `isinstance` target. the registry answers first, and a value - // nothing registered falls back to carrying the requirements - if let Some(test) = types.conformance_test(arm) { - needs.all_plain = false; - needs.all_true = false; - needs.references_value = true; - needs.conformance_runtime = true; - let members = match &test.members { - Some(names) => { - let mut spelled = String::from(", ("); - for name in names { - let _ = write!(spelled, "\"{name}\", "); - } - spelled.push(')'); - spelled - } - None => String::new(), - }; - return vec![ - Fragment::Lit("_by_conforms(".to_owned()), - value_ref(), - Fragment::Lit(", ".to_owned()), - Fragment::Src(arm.range()), - Fragment::Lit(format!("{members})")), - ]; - } - let Some(plan) = plan_for(types, value_expr, arm, position) else { + let Some(plan) = types.parametric_is_plan(value_expr, arm) else { needs.all_true = false; needs.references_value = true; return vec![ @@ -573,9 +525,33 @@ fn arm_predicate( Fragment::Lit(")".to_owned()), ]; }; - needs.all_plain = false; - if !matches!(plan, ParametricIsPlan::Fold(true)) { - needs.all_true = false; + fragments_for_plan(types, plan, value_ref, arm, probe, needs) +} + +/// the fragments one already-classified plan lowers to. split out from +/// [`arm_predicate`] so a union plan can build its arms, which the source need +/// not spell separately — a PEP 695 alias names a whole union with one word +fn fragments_for_plan( + types: &dyn TypeInfo, + plan: ParametricIsPlan, + value_ref: &dyn Fn() -> Fragment, + arm: &Expr, + probe: ProbeStrictness, + needs: &mut PredicateNeeds, +) -> Vec { + // a union reports through its arms instead: a disjunction of plain + // `isinstance` calls is still a plain check, and it holds as soon as one arm + // does + if !matches!(plan, ParametricIsPlan::Union(_)) { + if !matches!( + plan, + ParametricIsPlan::Isinstance(_) | ParametricIsPlan::Unresolved + ) { + needs.all_plain = false; + } + if !matches!(plan, ParametricIsPlan::Fold(true)) { + needs.all_true = false; + } } match plan { // an erased arm can't be checked at runtime; ty reports the error (a @@ -597,7 +573,7 @@ fn arm_predicate( frags.push(Fragment::Lit(")".to_owned())); frags } - ParametricIsPlan::Probe(variances) => { + ParametricIsPlan::Probe { target, variances } => { needs.parametric_runtime = true; needs.references_value = true; let codes: Vec = variances.iter().copied().map(variance_code).collect(); @@ -605,13 +581,14 @@ fn arm_predicate( ProbeStrictness::Strict => "_parametric_is(", ProbeStrictness::Lenient => "_parametric_is_lenient(", }; - vec![ + let mut frags = vec![ Fragment::Lit(call.to_owned()), value_ref(), Fragment::Lit(", ".to_owned()), - Fragment::Src(arm.range()), - Fragment::Lit(format!(", {})", variance_tuple(&codes))), - ] + target_fragment(&target, arm), + ]; + frags.push(Fragment::Lit(format!(", {})", variance_tuple(&codes)))); + frags } ParametricIsPlan::ProtocolStructural(checks) => { needs.protocol_runtime = true; @@ -623,35 +600,192 @@ fn arm_predicate( Fragment::Lit(format!(", {members})")), ] } + // nothing is known about the target, so the source's own spelling is + // the only thing to test against — and the error that left it unknown + // is already reported + // an interface something visibly conforms to cannot be answered by + // `isinstance`: a conforming type is not a subclass. the registry + // answers first, and a value nothing registered falls back to carrying + // the requirements + ParametricIsPlan::Conformance { target, members } => { + needs.references_value = true; + needs.conformance_runtime = true; + let mut spelled = String::from(", ("); + for name in &members { + let _ = write!(spelled, "\"{name}\", "); + } + spelled.push(')'); + vec![ + Fragment::Lit("_by_conforms(".to_owned()), + value_ref(), + Fragment::Lit(", ".to_owned()), + target_fragment(&target, arm), + Fragment::Lit(format!("{spelled})")), + ] + } + // the checker could not read the target as a type. that is how a unit + // enum variant arrives: the enum lowering has already rewritten it into + // a singleton instance, so what the source named as a type names a + // value here — and identity is the test for a value. anything else + // keeps the plain instance check, and whatever left the target unknown + // is already reported where it is written + ParametricIsPlan::Unresolved => { + needs.references_value = true; + if types.is_plain_value(arm) { + return vec![ + value_ref(), + Fragment::Lit(" is ".to_owned()), + Fragment::Src(arm.range()), + ]; + } + vec![ + Fragment::Lit("isinstance(".to_owned()), + value_ref(), + Fragment::Lit(", ".to_owned()), + Fragment::Src(arm.range()), + Fragment::Lit(")".to_owned()), + ] + } + ParametricIsPlan::Isinstance(target) => { + needs.references_value = true; + let mut frags = vec![ + Fragment::Lit("isinstance(".to_owned()), + value_ref(), + Fragment::Lit(", ".to_owned()), + ]; + frags.push(target_fragment(&target, arm)); + frags.push(Fragment::Lit(")".to_owned())); + frags + } + // `None` is a value, not a class, so `isinstance` cannot take it — and + // there is only one `None`, which makes identity the whole test + // a bare `Callable` asks exactly what `callable()` answers + ParametricIsPlan::IsCallable => { + needs.references_value = true; + vec![ + Fragment::Lit("callable(".to_owned()), + value_ref(), + Fragment::Lit(")".to_owned()), + ] + } + ParametricIsPlan::IsNone => { + needs.references_value = true; + vec![value_ref(), Fragment::Lit(" is None".to_owned())] + } + // the class guard is not redundant: python's `1 == True` would let a + // `bool` satisfy `Literal[1]` without it + ParametricIsPlan::Equality { class, value } => { + needs.references_value = true; + vec![ + Fragment::Lit("(type(".to_owned()), + value_ref(), + Fragment::Lit(format!(") is {class} and ")), + value_ref(), + Fragment::Lit(" == ".to_owned()), + target_fragment(&value, arm), + Fragment::Lit(")".to_owned()), + ] + } + ParametricIsPlan::Identity(target) => { + needs.references_value = true; + vec![ + value_ref(), + Fragment::Lit(" is ".to_owned()), + target_fragment(&target, arm), + ] + } + ParametricIsPlan::Subclass(target) => { + needs.references_value = true; + let mut frags = vec![Fragment::Lit("(isinstance(".to_owned()), value_ref()]; + frags.push(Fragment::Lit(", type) and issubclass(".to_owned())); + frags.push(value_ref()); + frags.push(Fragment::Lit(", ".to_owned())); + frags.push(target_fragment(&target, arm)); + frags.push(Fragment::Lit("))".to_owned())); + frags + } + // a template literal type is a set of strings; the regular expression + // ty built spells exactly the strings it produces + ParametricIsPlan::Pattern(pattern) => { + needs.pattern_runtime = true; + needs.references_value = true; + vec![ + Fragment::Lit("_by_pattern_is(".to_owned()), + value_ref(), + Fragment::Lit(format!(", {})", python_string_literal(&pattern))), + ] + } + // a value satisfies a union as soon as one arm holds + ParametricIsPlan::Union(arms) => { + let mut frags = vec![Fragment::Lit("(".to_owned())]; + let mut any_true = false; + for (index, arm_plan) in arms.iter().enumerate() { + if index > 0 { + frags.push(Fragment::Lit(" or ".to_owned())); + } + any_true |= matches!(arm_plan, ParametricIsPlan::Fold(true)); + frags.extend(fragments_for_plan( + types, + arm_plan.clone(), + value_ref, + arm, + probe, + needs, + )); + } + frags.push(Fragment::Lit(")".to_owned())); + if any_true { + needs.all_true = true; + } + frags + } + } +} + +/// how a plan's target is written: its own spelling, or the source the target +/// was written as when the plan carries none +fn target_fragment(target: &TargetSpelling, arm: &Expr) -> Fragment { + match target { + TargetSpelling::Written => Fragment::Src(arm.range()), + TargetSpelling::Rebuilt(text) => Fragment::Lit(text.clone()), } } -/// the full runtime predicate for a target expression, splitting a union into -/// the disjunction of its arms. shared by `is` and `cast`. +/// a python string literal for `text`, escaped so the emitted source reads it +/// back byte for byte. only the characters that end or reinterpret a +/// single-quoted literal need escaping +fn python_string_literal(text: &str) -> String { + let mut out = String::with_capacity(text.len() + 2); + out.push('"'); + for ch in text.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + _ => out.push(ch), + } + } + out.push('"'); + out +} + +/// the full runtime predicate for a target expression. shared by `is` and +/// `cast`. +/// +/// A union target needs no splitting here: the target is a type expression, so +/// its type is the union and the plan carries one arm per member — including +/// the arms of a union the source never spelled as one. pub(crate) fn build_predicate( types: &dyn TypeInfo, value_ref: &dyn Fn() -> Fragment, value_expr: &Expr, target: &Expr, - position: TargetPosition, probe: ProbeStrictness, ) -> (Vec, PredicateNeeds) { let mut needs = PredicateNeeds::new(); - let Some(arms) = union_arms(target) else { - let frags = arm_predicate( - types, value_ref, value_expr, target, position, probe, &mut needs, - ); - return (frags, needs); - }; - let mut frags = Vec::new(); - for (index, arm) in arms.iter().enumerate() { - if index > 0 { - frags.push(Fragment::Lit(" or ".to_owned())); - } - frags.extend(arm_predicate( - types, value_ref, value_expr, arm, position, probe, &mut needs, - )); - } + let frags = arm_predicate(types, value_ref, value_expr, target, probe, &mut needs); (frags, needs) } @@ -664,9 +798,8 @@ struct ParametricIs<'src, 'ti> { source: &'src str, types: &'ti dyn TypeInfo, edits: Vec<(TextRange, Vec)>, - needs_probe: bool, - needs_protocol: bool, - needs_conformance: bool, + /// the runtime helpers the predicates emitted so far call + runtimes: BTreeSet, } impl ParametricIs<'_, '_> { @@ -687,18 +820,26 @@ impl ParametricIs<'_, '_> { /// predicate builder and then wrapped for this form: negation, and keeping /// an effectful lhs alive when the predicate doesn't mention it fn lower_pair(&mut self, lhs: &Expr, rhs: &Expr, negate: bool) -> Vec { - let value = || Fragment::Src(lhs.range()); - let (frags, needs) = build_predicate( - self.types, - &value, - lhs, - rhs, - TargetPosition::Value, - ProbeStrictness::Strict, - ); - self.needs_probe |= needs.parametric_runtime; - self.needs_protocol |= needs.protocol_runtime; - self.needs_conformance |= needs.conformance_runtime; + // a predicate may mention the value more than once — a union tests each + // arm, an equality also checks the class — so an effectful left operand + // is bound to a lambda parameter and evaluated once. counting the + // references is what tells the two cases apart: a single mention needs + // no binding, and the lambda would only obscure the output + let mentions = std::cell::Cell::new(0usize); + let counting = || { + mentions.set(mentions.get() + 1); + Fragment::Src(lhs.range()) + }; + let (frags, needs) = + build_predicate(self.types, &counting, lhs, rhs, ProbeStrictness::Strict); + let via_lambda = mentions.get() > 1 && !effect_free(lhs); + let (frags, needs) = if via_lambda { + let param = || Fragment::Lit(UNION_VALUE_PARAM.to_owned()); + build_predicate(self.types, ¶m, lhs, rhs, ProbeStrictness::Strict) + } else { + (frags, needs) + }; + self.runtimes.extend(PredicateRuntime::used(&needs)); // a predicate that folded to a constant inverts in place rather than // growing a `not` @@ -713,10 +854,34 @@ impl ParametricIs<'_, '_> { return Self::with_lhs_effects(lhs, vec![Fragment::Lit(value.to_owned())]); } + // `x is not None` rather than `not x is None`: the same test, and the + // one a reader (and every linter) expects. an identity predicate is the + // only shape python can negate in place, and it is spelled with the + // operator in a literal fragment of its own + if negate + && let [value, Fragment::Lit(operator), rest @ ..] = frags.as_slice() + && operator.starts_with(" is ") + { + let mut negated = vec![ + value.clone(), + Fragment::Lit(operator.replacen(" is ", " is not ", 1)), + ]; + negated.extend_from_slice(rest); + return negated; + } + let mut result = Vec::new(); if negate { result.push(Fragment::Lit("not ".to_owned())); } + if via_lambda { + result.push(Fragment::Lit(format!("(lambda {UNION_VALUE_PARAM}: "))); + result.extend(frags); + result.push(Fragment::Lit(")(".to_owned())); + result.push(Fragment::Src(lhs.range())); + result.push(Fragment::Lit(")".to_owned())); + return result; + } result.extend(frags); if needs.references_value { result @@ -725,122 +890,94 @@ impl ParametricIs<'_, '_> { } } - /// one arm of a union `is`-target, delegated to the shared predicate - /// builder so `is` and `cast` stay in lockstep - fn lower_arm(&mut self, value: &dyn Fn() -> Fragment, lhs: &Expr, arm: &Expr) -> Vec { - let mut needs = PredicateNeeds::new(); - let frags = arm_predicate( - self.types, - value, - lhs, - arm, - TargetPosition::Value, - ProbeStrictness::Strict, - &mut needs, - ); - self.needs_probe |= needs.parametric_runtime; - self.needs_protocol |= needs.protocol_runtime; - self.needs_conformance |= needs.conformance_runtime; - frags - } - - /// `lhs is (T1 | T2 | …)` — a test against a union type — is the disjunction - /// of the per-arm tests (`type(lhs) <: Ti` for any arm). the lhs is bound - /// once: referenced directly when it has no effects, else through a lambda - /// parameter so the arms share a single evaluation - fn lower_union(&mut self, lhs: &Expr, arms: &[&Expr], negate: bool) -> Vec { - let via_lambda = !effect_free(lhs); - let value = || { - if via_lambda { - Fragment::Lit(UNION_VALUE_PARAM.to_owned()) - } else { - Fragment::Src(lhs.range()) - } - }; - let mut inner: Vec = Vec::new(); - for (index, arm) in arms.iter().enumerate() { - if index > 0 { - inner.push(Fragment::Lit(" or ".to_owned())); - } - inner.extend(self.lower_arm(&value, lhs, arm)); - } - let mut frags = Vec::new(); - if via_lambda { - frags.push(Fragment::Lit(format!( - "{}(lambda {UNION_VALUE_PARAM}: ", - if negate { "not " } else { "" } - ))); - frags.extend(inner); - frags.push(Fragment::Lit(")(".to_owned())); - frags.push(Fragment::Src(lhs.range())); - frags.push(Fragment::Lit(")".to_owned())); - } else { - frags.push(Fragment::Lit(if negate { "not (" } else { "(" }.to_owned())); - frags.extend(inner); - frags.push(Fragment::Lit(")".to_owned())); - } - frags - } - fn process_compare(&mut self, compare: &ast::ExprCompare) { let mut lhs: &Expr = &compare.left; - for (op, rhs) in compare.ops.iter().zip(&compare.comparators) { - // identity_swap defers every non-literal name/attribute/subscript - // and union `is`-rhs to this pass, which owns the - // isinstance-vs-parametric decision (a bare class → isinstance, a - // specialization or an alias to one → a parametric test, a union → - // the disjunction of the arms) - if matches!(op, CmpOp::Is | CmpOp::IsNot) - && is_keyword_comparison(self.source, *op, lhs, rhs) - // a subscript that resolves to a plain value (`candidates[0]` - // holding an enum member) keeps python identity semantics, - // same as identity_swap's rule for unsubscripted rhs - && !self.types.is_keeps_identity(rhs) - { - let negate = matches!(op, CmpOp::IsNot); - let replacement = - if matches!(rhs, Expr::Name(_) | Expr::Attribute(_) | Expr::Subscript(_)) { - Some(self.lower_pair(lhs, rhs, negate)) - } else { - union_arms(rhs).map(|arms| self.lower_union(lhs, &arms, negate)) - }; - if let Some(replacement) = replacement { - let pair_range = TextRange::new(lhs.range().start(), rhs.range().end()); - self.edits.push((pair_range, replacement)); - } + for (index, rhs) in compare.comparators.iter().enumerate() { + // every type test is lowered here, whatever shape its target was + // written in. the plan comes from the target's *type*, so a target + // the source spells as one word (`x is Alias`) and the union that + // word stands for reach the same lowering + if compare.is_type_test(index, PySourceType::BasedPython) { + let negate = compare.ops.get(index) == Some(&CmpOp::IsNot); + let (opening, closing) = paren_padding(self.source, lhs, rhs); + let mut replacement = Vec::new(); + replacement.push(Fragment::Lit("(".repeat(opening))); + replacement.extend(self.lower_pair(lhs, rhs, negate)); + replacement.push(Fragment::Lit(")".repeat(closing))); + let pair_range = TextRange::new(lhs.range().start(), rhs.range().end()); + self.edits.push((pair_range, replacement)); } lhs = rhs; } } } -/// the lambda parameter that binds an effectful union-test lhs for its arms. -/// unlikely to collide: an `is`-target arm is a type expression, and this name -/// would have to appear free inside one -const UNION_VALUE_PARAM: &str = "_by_is_value"; +/// a runtime helper one of the emitted predicates calls. the preamble emits one +/// definition per helper a pass actually used, in this order — `_parametric_is` +/// and `_by_protocol_is` must precede the predicates that call them +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum PredicateRuntime { + Parametric, + Protocol, + Conformance, + Pattern, +} -/// the flat arms of a union type expression (`A | B | C` → `[A, B, C]`), or -/// `None` when `expr` is not a `|` union -fn union_arms(expr: &Expr) -> Option> { - if !matches!(expr, Expr::BinOp(binop) if binop.op == ast::Operator::BitOr) { - return None; +impl PredicateRuntime { + /// the definitions this helper needs in the emitted module + pub(crate) fn source(self) -> &'static str { + match self { + Self::Parametric => PARAMETRIC_IS_RUNTIME, + Self::Protocol => PROTOCOL_IS_RUNTIME, + Self::Conformance => super::conformance::WITNESS_RUNTIME, + Self::Pattern => PATTERN_IS_RUNTIME, + } } - let mut arms = Vec::new(); - collect_union_arms(expr, &mut arms); - Some(arms) -} -fn collect_union_arms<'a>(expr: &'a Expr, arms: &mut Vec<&'a Expr>) { - if let Expr::BinOp(binop) = expr - && binop.op == ast::Operator::BitOr - { - collect_union_arms(&binop.left, arms); - collect_union_arms(&binop.right, arms); - } else { - arms.push(expr); + /// the helpers `needs` recorded, in preamble order + pub(crate) fn used(needs: &PredicateNeeds) -> impl Iterator { + [ + needs.parametric_runtime.then_some(Self::Parametric), + needs.protocol_runtime.then_some(Self::Protocol), + needs.conformance_runtime.then_some(Self::Conformance), + needs.pattern_runtime.then_some(Self::Pattern), + ] + .into_iter() + .flatten() } } +/// the lambda parameter that binds an effectful test value for a predicate that +/// mentions it more than once. unlikely to collide: an `is`-target is a type +/// expression, and this name would have to appear free inside one +const UNION_VALUE_PARAM: &str = "_by_is_value"; + +/// how many parentheses a replacement for one comparison pair must supply +/// itself, because the source wrote them around an operand rather than around +/// the pair. +/// +/// An operand's range stops inside its own parentheses, so the pair +/// `lhs.start() .. rhs.end()` of `(a) is str` swallows the `)` while leaving the +/// `(` outside it. Rather than guess which outer `(` that was — the `(` of an +/// enclosing call looks exactly the same — the replacement closes what it +/// swallowed and opens what the source will close after it. +fn paren_padding(source: &str, lhs: &Expr, rhs: &Expr) -> (usize, usize) { + let gap = TextRange::new(lhs.range().end(), rhs.range().start()); + let tokens: Vec<_> = SimpleTokenizer::new(source, gap).skip_trivia().collect(); + // parens closing around the left operand come first in the gap, parens + // opening around the right operand last; the operator sits between them + let closing = tokens + .iter() + .take_while(|token| token.kind() == SimpleTokenKind::RParen) + .count(); + let opening = tokens + .iter() + .rev() + .take_while(|token| token.kind() == SimpleTokenKind::LParen) + .count(); + (opening, closing) +} + impl<'ast> Visitor<'ast> for ParametricIs<'_, '_> { fn visit_stmt(&mut self, stmt: &'ast Stmt) { walk_stmt(self, stmt); @@ -870,9 +1007,7 @@ impl TypeAwarePass for ParametricIsPass<'_> { source: self.source, types, edits: Vec::new(), - needs_probe: false, - needs_protocol: false, - needs_conformance: false, + runtimes: BTreeSet::new(), }; for stmt in stmts { inner.visit_stmt(stmt); @@ -882,15 +1017,8 @@ impl TypeAwarePass for ParametricIsPass<'_> { // (`T == list[int]`), which is already restricted to 3.12+ by the // reified-generic requirement; a user-generic probe (`A[int]`) works // on any target - if inner.needs_probe { - ctx.required_imports.push(PARAMETRIC_IS_RUNTIME.to_owned()); - } - if inner.needs_protocol { - ctx.required_imports.push(PROTOCOL_IS_RUNTIME.to_owned()); - } - if inner.needs_conformance { - ctx.required_imports - .push(super::conformance::WITNESS_RUNTIME.to_owned()); + for runtime in inner.runtimes { + ctx.required_imports.push(runtime.source().to_owned()); } ctx.template_edits.extend(inner.edits); } @@ -1344,7 +1472,8 @@ mod tests { #[test] fn implicit_alias_target_probes_like_the_specialization() { // `X = A[int]` binds `X` to the specialization itself, so `y is X` - // resolves exactly as `y is A[int]` would — a probe here + // resolves exactly as `y is A[int]` would — and is written back out + // that way, which is what the runtime probe needs to unwind let out = out(indoc! {" class A[T]: def __init__(self, v: T): @@ -1354,30 +1483,30 @@ mod tests { return y is X "}); assert!( - out.contains("return _parametric_is(y, X, (0,))"), - "alias name probes through `X`: {out}" + out.contains("return _parametric_is(y, A[int], (0,))"), + "alias name probes against the specialization it binds: {out}" ); } #[test] fn implicit_alias_builtin_target_probes() { // an alias to a builtin specialization probes just like the direct form - // — the alias name `X` is passed through so the runtime unwinds it let out = out(indoc! {" X = list[int] def f(y: object) -> bool: return y is X "}); assert!( - out.contains("return _parametric_is(y, X, (0,))"), - "alias to a builtin probes through `X`: {out}" + out.contains("return _parametric_is(y, list[int], (0,))"), + "alias to a builtin probes against the specialization: {out}" ); } #[test] fn pep695_type_alias_target_probes_through_value() { - // `type W = A[int]` is a `TypeAliasType`; the probe unwraps `.__value__` - // at runtime, so `y is W` still resolves against `A[int]` + // `type W = A[int]` evaluates to a `TypeAliasType`, which `isinstance` + // would refuse — so the specialization it stands for is written out + // instead of the name let out = out(indoc! {" class A[T]: def __init__(self, v: T): @@ -1387,8 +1516,8 @@ mod tests { return y is W "}); assert!( - out.contains("return _parametric_is(y, W, (0,))"), - "type alias probes through `W`: {out}" + out.contains("return _parametric_is(y, A[int], (0,))"), + "type alias probes against its value: {out}" ); } @@ -1428,10 +1557,10 @@ mod tests { def __init__(self, v: T): self.v: list[T] = [v] def f(a: object) -> bool: - return a is A[int] | object + return a is A[int] | str "}); assert!( - out.contains("return (_parametric_is(a, A[int], (0,)) or isinstance(a, object))"), + out.contains("return (_parametric_is(a, A[int], (0,)) or isinstance(a, str))"), "each arm lowered by its own kind: {out}" ); } @@ -1488,8 +1617,8 @@ mod tests { "}); assert!( out.contains( - "return (lambda _by_is_value: isinstance(_by_is_value, int) or \ - isinstance(_by_is_value, str))(g())" + "return (lambda _by_is_value: (isinstance(_by_is_value, int) or \ + isinstance(_by_is_value, str)))(g())" ), "effectful lhs bound once: {out}" ); @@ -1620,25 +1749,24 @@ mod tests { } #[test] - fn a_target_outside_the_union_compares_the_cell() { - // no arm matches `list[bytes]`, so this can never be true. it used to - // fold to `False` statically; now the parameter carries a reified type - // parameter constrained to `(int, str)`, so it lowers to a cell - // comparison that is always false at runtime instead. same answer, one - // comparison rather than a constant — the fold is not recoverable from - // the token path, which carries source ranges rather than types + fn a_target_outside_the_union_can_never_hold() { + // no arm matches `list[bytes]`, so this can never be true — and the + // constraint `(int, str)` on the reified parameter says so statically, + // before any runtime residue is reached let out = out(indoc! {" def f(x: list[int] | list[str]) -> bool: return x is list[bytes] "}); assert!( - out.contains("return (__by_erased_0 == bytes)"), - "target outside the union compares the cell: {out}" + out.contains("return False"), + "a target outside the union can never hold: {out}" ); } #[test] - fn value_subscript_rhs_falls_back_to_isinstance() { + fn a_subscript_of_a_value_keeps_the_plain_instance_check() { + // `pair[0]` is not something a type expression can say, so the checker + // reports it and the lowering keeps the `isinstance` the source wrote let out = out(indoc! {" class A: ... pair = (A, A) @@ -1680,9 +1808,9 @@ mod tests { #[test] fn stdlib_enum_member_rhs_keeps_identity() { - // an enum member is a singleton instance, not a class, so - // `isinstance(x, Color.RED)` would be a runtime TypeError; the pair - // must keep `is` / `is not` + // `Color.RED` names the type `Literal[Color.RED]`, which holds exactly + // one object — so the test is identity, and `isinstance(x, Color.RED)` + // (a runtime `TypeError`) is never emitted let out = out(indoc! {" from enum import Enum @@ -1690,17 +1818,106 @@ mod tests { RED = 1 GREEN = 2 - print(Color.RED is Color.RED) - print(Color.RED is not Color.GREEN) + def f(c: Color) -> None: + print(c is Color.RED) + print(c is not Color.GREEN) "}); assert!( - out.contains("print(Color.RED is Color.RED)"), - "enum member rhs keeps identity: {out}" + out.contains("print(c is Color.RED)"), + "enum member rhs is an identity check: {out}" ); assert!( - out.contains("print(Color.RED is not Color.GREEN)"), - "enum member rhs keeps identity under is not: {out}" + out.contains("print(c is not Color.GREEN)"), + "enum member rhs is an identity check under `is not`: {out}" + ); + } + + #[test] + fn an_alias_to_the_enum_still_reaches_the_variant() { + // the target is decided from the *type* it names, so a binding standing + // in for the enum reaches the same variant. a match on the written name + // would miss this one and emit `isinstance` against a singleton + let out = out(indoc! {" + enum class Shape: + case Circle(radius: float) + case Point + + S = Shape + + def f(s: Shape) -> bool: + return s is S.Point + "}); + assert!( + out.contains("return s is S.Point"), + "an aliased enum's unit variant is still an identity check: {out}" + ); + } + + #[test] + fn a_binding_shadowing_the_enum_names_what_it_holds() { + // the same rule the other way: `Shape` here is the local class, whose + // `Point` is an ordinary class and therefore an instance check + let out = out(indoc! {" + enum class Shape: + case Circle(radius: float) + case Point + + class Other: + class Point: pass + + def f(x: object) -> bool: + Shape = Other + return x is Shape.Point + "}); + assert!( + out.contains("return isinstance(x, Shape.Point)"), + "a shadowed name names what the binding holds: {out}" + ); + } + + #[test] + fn a_literal_target_keeps_the_source_spelling() { + // the source already wrote a literal that is valid where it sits — + // rebuilding one would pick its own quote character, which a python + // before 3.12 forbids reusing inside an f-string + let out = out(indoc! {" + def f(x: object) -> str: + return f\"{x is 'q'}\" + "}); + assert!( + out.contains("x == 'q'"), + "the literal is re-emitted as written: {out}" + ); + } + + #[test] + fn an_effectful_value_is_evaluated_once_for_a_literal_target() { + // the equality check mentions the value twice — once for the class + // guard — so an effectful operand is bound rather than repeated + let out = out(indoc! {" + def g() -> object: + return 1 + + def f() -> bool: + return g() is 1 + "}); + assert_eq!(out.matches("g()").count(), 2, "single evaluation:\n{out}"); + } + + #[test] + fn a_pattern_target_escapes_what_the_source_cannot_carry() { + // a control character has no raw spelling in python source — CPython + // refuses a file containing a NUL outright — so the regex carries its + // escape instead + let out = out(indoc! {" + def f(s: str) -> bool: + return s is f\"a\\x00b{int}\" + "}); + assert!( + out.contains("\\\\x00") || out.contains("a\\\\x00b"), + "the pattern escapes the control character: {out}" ); + assert!(!out.contains('\0'), "no raw NUL reaches the output: {out}"); } #[test] @@ -1710,11 +1927,12 @@ mod tests { enum class Genre: case A, B - print(Genre.A is not Genre.B) + def f(g: Genre) -> None: + print(g is not Genre.B) "}); assert!( - out.contains("print(Genre.A is not Genre.B)"), - "caseless variant rhs keeps identity: {out}" + out.contains("print(g is not Genre.B)"), + "caseless variant rhs is an identity check: {out}" ); } @@ -1727,12 +1945,12 @@ mod tests { case Circle(radius: float) case Point - c = Shape.Circle(1.0) - print(c is Shape.Circle) - print(c is not Shape.Point) + def f(c: Shape) -> None: + print(c is Shape.Circle) + print(c is not Shape.Point) "}); assert!( - out.contains("print(isinstance(c, Shape.Circle))"), + out.contains("print(isinstance(c, _Shape_Circle))"), "payload variant rhs is a class and lowers to isinstance: {out}" ); assert!( diff --git a/crates/by_transforms/src/transforms/reified_class.rs b/crates/by_transforms/src/transforms/reified_class.rs index 51984d3f55..fe2ae49431 100644 --- a/crates/by_transforms/src/transforms/reified_class.rs +++ b/crates/by_transforms/src/transforms/reified_class.rs @@ -227,7 +227,7 @@ impl<'src> ReifiedClass<'src> { } fn specialize(&mut self, class: &StmtClassDef) { - let reads = reified_class_reads(self.source, PySourceType::BasedPython, class); + let reads = reified_class_reads(PySourceType::BasedPython, class); if reads.names.is_empty() { return; } diff --git a/crates/by_transforms/src/transforms/reified_generic.rs b/crates/by_transforms/src/transforms/reified_generic.rs index d845701762..e965deadd6 100644 --- a/crates/by_transforms/src/transforms/reified_generic.rs +++ b/crates/by_transforms/src/transforms/reified_generic.rs @@ -230,7 +230,7 @@ impl<'src> ReifiedGeneric<'src> { } fn wrap(&mut self, function: &StmtFunctionDef) { - if reified_type_param_names(self.source, PySourceType::BasedPython, function).is_empty() { + if reified_type_param_names(PySourceType::BasedPython, function).is_empty() { return; } if !self.supports_native_generics { diff --git a/crates/by_transforms/src/transforms/type_is.rs b/crates/by_transforms/src/transforms/type_is.rs index 88eea4e5ea..49ddc402e0 100644 --- a/crates/by_transforms/src/transforms/type_is.rs +++ b/crates/by_transforms/src/transforms/type_is.rs @@ -8,11 +8,10 @@ //! identical to PEP 742 `TypeIs[T]`; the parameter name is lost in //! lowering since `TypeIs` doesn't carry it. //! -//! traversal is delegated to [`type_expr_walker`] (with `types = None` — -//! value-position `a is T` is *not* a type expression here; it's the -//! basedpython surface form for `isinstance(a, T)`, owned by -//! `identity_swap`). running before `identity_swap` in the `AstPass` list so -//! type-position rewrites win the first-wins overlap dedup +//! traversal is delegated to [`type_expr_walker`] (with `types = None`): the +//! `a is T` written in a *return guard* is what this rewrites, and the same +//! pair written in a body is a type test that [`parametric_is`] lowers. this +//! pass claims the guard first, so the two never rewrite the same span use ruff_python_ast::helpers::{ReturnGuardForm, return_guards}; use ruff_python_ast::visitor::{Visitor, walk_stmt}; diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index 0ee655a86a..35da2c5acc 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -167,22 +167,16 @@ pub(crate) trait TypeInfo { rhs: &Expr, ) -> Option; - /// [`Self::parametric_is_plan`] for a checked cast's `(value, target)` pair. - /// the same classification engine backs both; only the target's inference - /// position differs (a cast target is a type expression) - fn parametric_cast_plan( - &self, - value: &Expr, - target: &Expr, - ) -> Option; - - /// whether an `is`/`is not` comparison whose rhs is `expr` keeps python - /// identity semantics instead of lowering to `isinstance`: true when - /// `expr` resolves to a plain *value* — an enum member (`Color.RED`, a - /// based-enum unit variant like `Shape.Point`), another literal, or an - /// instance of a concrete non-type class — which `isinstance` would - /// reject as its classinfo argument at runtime - fn is_keeps_identity(&self, expr: &Expr) -> bool; + /// whether the *value* `expr` evaluates to is a plain value rather than a + /// class — an enum member, a literal, an instance of a concrete non-type + /// class. `isinstance` would reject such a value as its classinfo argument, + /// so the runtime test for it is identity. + /// + /// Only consulted for a target the checker could not read as a type at all. + /// The enum lowering rewrites a unit variant into a singleton instance + /// before this pass runs, so `s is Shape.Point` reaches here naming + /// something that is a type in the source and a value in what is emitted + fn is_plain_value(&self, expr: &Expr) -> bool; /// whether `expr` is a PEP 604 union standing where the runtime will /// evaluate it — `isinstance(x, int | str)`, a `cast` target, an alias @@ -235,11 +229,6 @@ pub(crate) trait TypeInfo { attribute: &ruff_python_ast::ExprAttribute, ) -> Option; - /// how `x is ` is answered when `target` is an interface something - /// visibly conforms to. `None` when the ordinary `isinstance` lowering is - /// still right - fn conformance_test(&self, target: &Expr) -> Option; - /// the conversions a statement's value needs: an annotated assignment, an /// attribute assignment, or a `return`. one wrap for a value that converts /// whole, or one per element for a collection literal @@ -704,22 +693,8 @@ impl TypeInfo for SemanticModel<'_> { SemanticModel::parametric_is_plan(self, lhs, rhs) } - fn parametric_cast_plan( - &self, - value: &Expr, - target: &Expr, - ) -> Option { - SemanticModel::parametric_cast_plan(self, value, target) - } - - fn is_keeps_identity(&self, expr: &Expr) -> bool { - expr.inferred_type(self).is_some_and(|ty| { - ty_python_semantic::types::basedpython_is_keeps_identity( - self.db(), - &self.program_environment(), - ty, - ) - }) + fn is_plain_value(&self, expr: &Expr) -> bool { + SemanticModel::denotes_plain_value(self, expr) } fn is_runtime_union(&self, expr: &Expr) -> bool { @@ -773,10 +748,6 @@ impl TypeInfo for SemanticModel<'_> { SemanticModel::witness_dispatch(self, attribute) } - fn conformance_test(&self, target: &Expr) -> Option { - SemanticModel::conformance_test(self, target) - } - fn statement_conversions( &self, stmt: &Stmt, diff --git a/crates/by_transforms/tests/parametric_is_runtime.rs b/crates/by_transforms/tests/parametric_is_runtime.rs index 311b81f375..669147ee48 100644 --- a/crates/by_transforms/tests/parametric_is_runtime.rs +++ b/crates/by_transforms/tests/parametric_is_runtime.rs @@ -339,6 +339,111 @@ print("ok") /// Returns `None` (test skips) when none is found — PEP 695 class syntax is a /// hard requirement here. /// pep 695 type parameters — what [`PROGRAM`] needs (3.12+) +/// basedpython whose module-level `assert`s exercise the runtime check each +/// *non-parametric* target lowers to. The checker's own tests pin the type it +/// reads and the transpiler's pin the text it emits; only running the result +/// says whether the check accepts the values the type contains and no others. +const TARGETS_PROGRAM: &str = r#" +from typing import Callable, Literal, Protocol, runtime_checkable + +class A: ... +class B(A): ... + +# a class is the `isinstance` it looks like +assert (B() is A) is True, "a subclass instance is an A" +assert (A() is B) is False, "a base instance is not a B" + +# `type[C]` asks two things, and the runtime can answer both +assert (B is type[A]) is True, "a subclass is a type[A]" +assert (B() is type[A]) is False, "an instance is not a type[A]" +assert (int is type[A]) is False, "an unrelated class is not a type[A]" + +# a literal names the values equal to it, with the class pinned: `1 == True` in +# python, and `Literal[1]` does not contain `True` +def literal_one(v: object) -> bool: + return v is Literal[1] + +assert literal_one(1) is True, "1 is a Literal[1]" +assert literal_one(True) is False, "True is not a Literal[1]" +assert literal_one(2) is False, "2 is not a Literal[1]" + +def literal_true(v: object) -> bool: + return v is Literal[True] + +assert literal_true(True) is True, "True is a Literal[True]" +assert literal_true(1) is False, "1 is not a Literal[True]" + +# a union holds when any arm does, including one the source spells with a +# single word +type Key = int | str + +def key(v: object) -> bool: + return v is Key + +assert key(1) is True, "an int is a Key" +assert key("a") is True, "a str is a Key" +assert key(1.5) is False, "a float is not a Key" + +# a template literal type is the set of strings its pattern produces +def item(s: str) -> bool: + return s is f"item-{int}" + +assert item("item-12") is True, "item-12 matches" +assert item("item--3") is True, "a negative renders with its sign" +assert item("item-0") is True, "zero renders as one digit" +assert item("item--0") is False, "str(-0) is 0, so -0 renders no such string" +assert item("item-01") is False, "a leading zero is not an int rendering" +assert item("item-ab") is False, "a non-numeric tail does not match" +assert item("item-") is False, "an empty hole does not match" +assert (1 is f"item-{int}") is False, "a non-str is never one of the strings" + +# a pattern's literal text matches itself, metacharacters and all +def dotted(s: str) -> bool: + return s is f"a.b{int}" + +assert dotted("a.b1") is True, "the dot is literal text" +assert dotted("axb1") is False, "and does not stand for any character" + +# `None` is a value, so its test is the identity python already performs +def optional(v: object) -> bool: + return v is None + +assert optional(None) is True, "None is None" +assert optional(0) is False, "0 is not None" + +# a bare `Callable` asks exactly what `callable()` answers +def call(v: object) -> bool: + return v is Callable + +assert call(len) is True, "a builtin is callable" +assert call(A) is True, "a class is callable" +assert call(1) is False, "an int is not" + +# a `@runtime_checkable` protocol is the check python itself performs +@runtime_checkable +class HasName(Protocol): + name: str + +class Named: + def __init__(self) -> None: + self.name = "x" + +assert (Named() is HasName) is True, "a value with the member satisfies it" +assert (1 is HasName) is False, "one without does not" + +# `is not` is the negation, evaluated once +calls = [] + +def once() -> object: + calls.append(1) + return 1 + +assert (once() is not Literal[2]) is True, "the negation holds" +assert len(calls) == 1, "an effectful value is evaluated once" + +print("ok") +"#; + const PEP695_PROBE: &str = "type X[T] = T"; /// pep 696 defaults in native syntax plus the `has_default()` accessor the probe @@ -406,6 +511,19 @@ fn parametric_protocol_checks_run_correctly() { run_program(&python, PROGRAM); } +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn every_target_kind_checks_correctly_at_runtime() { + let Some(python) = python_supporting(PEP695_PROBE) else { + eprintln!("skipping type-test target runtime test: no PEP 695-capable interpreter found"); + return; + }; + run_program(&python, TARGETS_PROGRAM); +} + #[test] #[expect( clippy::print_stderr, diff --git a/crates/ruff_linter/resources/test/fixtures/basedpython/BY001.by b/crates/ruff_linter/resources/test/fixtures/basedpython/BY001.by index 623b7c8278..1f808f6ebb 100644 --- a/crates/ruff_linter/resources/test/fixtures/basedpython/BY001.by +++ b/crates/ruff_linter/resources/test/fixtures/basedpython/BY001.by @@ -26,12 +26,14 @@ def not_coalescing(a, b, c): # a falsiness test is not the same test _ = a if a else b - # not a `None` comparison - _ = a if a is not b else b + # not a `None` comparison. written with the identity operator because `is` + # takes a type, and `b` is a value + _ = a if a !== b else b _ = a if a == None else b - # a chained comparison guards more than one thing - _ = a if a is not None is not b else b + # a chained comparison guards more than one thing. it is written with the + # identity operator because a type test may not join a chain at all + _ = a if a !== None !== b else b # the left operand has to be evaluated twice to write it this way, so it # cannot be anything that runs code — a subscription included diff --git a/crates/ruff_linter/resources/test/fixtures/pyflakes/F632_basedpython.by b/crates/ruff_linter/resources/test/fixtures/pyflakes/F632_basedpython.by index cb69fb5138..d57c6a2936 100644 --- a/crates/ruff_linter/resources/test/fixtures/pyflakes/F632_basedpython.by +++ b/crates/ruff_linter/resources/test/fixtures/pyflakes/F632_basedpython.by @@ -17,3 +17,20 @@ def type_tests[T](x: T, xs: list[int]) -> None: # this one lowers to `isinstance([], [])`, which raises at runtime — but the # mistake in it is the target, not the operator, so it is ty's to report print([] is []) + + +# which spelling was written is recorded on the comparison, not recovered from the +# source between the operands, so it survives everything that can sit there +def spelling_survives_what_sits_between(x: object) -> None: + a = (1) === (1) # F632 + b = 1 \ + === 1 # F632 + c = (1 === # a comment between the operands + 1) # F632 + d = 1===1 # F632 + + +# each operator of a chain answers for its own spelling +def chained(x: object) -> None: + e = x < 1 === 1 # F632 + f = 1 === 1 !== 2 # F632 diff --git a/crates/ruff_linter/src/checkers/ast/analyze/expression.rs b/crates/ruff_linter/src/checkers/ast/analyze/expression.rs index e0a31180b1..5af4e1dc93 100644 --- a/crates/ruff_linter/src/checkers/ast/analyze/expression.rs +++ b/crates/ruff_linter/src/checkers/ast/analyze/expression.rs @@ -1689,13 +1689,14 @@ pub(crate) fn expression(expr: &Expr, checker: &Checker) { comparators, range: _, node_index: _, + identity_ops: _, }, ) => { if checker.any_rule_enabled(&[Rule::NoneComparison, Rule::TrueFalseComparison]) { pycodestyle::rules::literal_comparisons(checker, compare); } if checker.is_rule_enabled(Rule::IsLiteral) { - pyflakes::rules::invalid_literal_comparison(checker, left, ops, comparators, expr); + pyflakes::rules::invalid_literal_comparison(checker, compare); } if checker.is_rule_enabled(Rule::TypeComparison) { pycodestyle::rules::type_comparison(checker, compare); diff --git a/crates/ruff_linter/src/rules/basedpython/rules/manual_isinstance.rs b/crates/ruff_linter/src/rules/basedpython/rules/manual_isinstance.rs index 52ce7d36de..1384ab5f6c 100644 --- a/crates/ruff_linter/src/rules/basedpython/rules/manual_isinstance.rs +++ b/crates/ruff_linter/src/rules/basedpython/rules/manual_isinstance.rs @@ -1,6 +1,6 @@ use ruff_diagnostics::Applicability; use ruff_macros::{ViolationMetadata, derive_message_formats}; -use ruff_python_ast::{self as ast, Expr, UnaryOp}; +use ruff_python_ast::{self as ast, Expr, Operator, UnaryOp}; use ruff_text_size::{Ranged, TextRange}; use crate::checkers::ast::Checker; @@ -43,7 +43,7 @@ use crate::{AlwaysFixableViolation, Edit, Fix}; /// than as several classes. /// /// ## References -/// - [basedpython documentation: identity and isinstance](https://docs.basedpython.org/features/identity-swap) +/// - [basedpython documentation: type tests and identity](https://docs.basedpython.org/features/identity-swap) #[derive(ViolationMetadata)] #[violation_metadata(stable_since = "0.0.1-a10", category = Category::Style)] pub(crate) struct ManualIsinstance; @@ -59,6 +59,24 @@ impl AlwaysFixableViolation for ManualIsinstance { } } +/// whether `classinfo` is something a type expression can name — a name, a +/// dotted name, a subscript of one, or a `|` union of those. +/// +/// `isinstance` takes a runtime value, and most of what can stand there is not +/// a type expression: a tuple means the tuple *type* rather than a choice of +/// classes, and `type(y)` or `registry["a"]` mean nothing there at all. +fn names_a_type(classinfo: &Expr) -> bool { + match classinfo { + Expr::Name(_) => true, + Expr::Attribute(attribute) => names_a_type(&attribute.value), + Expr::Subscript(subscript) => names_a_type(&subscript.value), + Expr::BinOp(binop) if binop.op == Operator::BitOr => { + names_a_type(&binop.left) && names_a_type(&binop.right) + } + _ => false, + } +} + /// BY003 pub(crate) fn manual_isinstance(checker: &Checker, call: &ast::ExprCall) { if !checker.source_type.is_basedpython() { @@ -76,9 +94,11 @@ pub(crate) fn manual_isinstance(checker: &Checker, call: &ast::ExprCall) { if !call.arguments.keywords.is_empty() { return; } - // a tuple in the class position reads as a tuple type, not as a choice of - // classes - if class.is_tuple_expr() { + // `is` takes a *type expression*, so the rewrite is only available when the + // classinfo argument is something one can say. a tuple reads as the tuple + // type rather than as a choice of classes; a call or a subscript of a value + // is not a type expression at all + if !names_a_type(class) { return; } diff --git a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/assertion.rs b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/assertion.rs index 52261b2409..a0968cbf41 100644 --- a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/assertion.rs +++ b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/assertion.rs @@ -322,7 +322,7 @@ pub(crate) fn unittest_assertion( && checker.semantic().current_expression_parent().is_none() && !checker.comment_ranges().intersects(expr.range()) { - if let Ok(stmt) = unittest_assert.generate_assert(args, keywords) { + if let Ok(stmt) = unittest_assert.generate_assert(args, keywords, checker.source_type) { diagnostic.set_fix(Fix::unsafe_edit(Edit::range_replacement( checker.generator().stmt(&stmt), parenthesized_range( diff --git a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/unittest_assert.rs b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/unittest_assert.rs index 987184bcfa..01f0b69041 100644 --- a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/unittest_assert.rs +++ b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/unittest_assert.rs @@ -1,7 +1,8 @@ use anyhow::{Result, anyhow, bail}; use ruff_python_ast::name::Name; use ruff_python_ast::{ - self as ast, Arguments, CmpOp, Expr, ExprContext, Identifier, Keyword, Stmt, UnaryOp, + self as ast, Arguments, CmpOp, Expr, ExprContext, Identifier, Keyword, PySourceType, Stmt, + UnaryOp, }; use ruff_text_size::TextRange; use rustc_hash::{FxBuildHasher, FxHashMap}; @@ -170,11 +171,26 @@ fn assert(expr: &Expr, msg: Option<&Expr>) -> Stmt { }) } -fn compare(left: &Expr, cmp_op: CmpOp, right: &Expr) -> Expr { +fn compare(left: &Expr, cmp_op: CmpOp, right: &Expr, source_type: PySourceType) -> Expr { Expr::Compare(ast::ExprCompare { left: Box::new(left.clone()), ops: Box::from([cmp_op]), comparators: Box::from([right.clone()]), + // `assertIs` asserts python *identity*, which basedpython writes `===` + // — its `is` keyword is a type test, a different assertion entirely. + // recording no spelling prints `is`, so this has to say so. + // + // `assertIsNone` is the exception: `is None` is a test for the type + // `None`, which holds one object, so it is the same test either way — + // and it is the spelling a reader expects + identity_ops: (source_type.is_basedpython() + && matches!(cmp_op, CmpOp::Is | CmpOp::IsNot) + && !right.is_none_literal_expr()) + .then(|| { + Box::new(ast::IdentityOperators { + ops: Box::from([true]), + }) + }), range: TextRange::default(), node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) @@ -278,7 +294,12 @@ impl UnittestAssert { Ok(args_map) } - pub(crate) fn generate_assert(self, args: &[Expr], keywords: &[Keyword]) -> Result { + pub(crate) fn generate_assert( + self, + args: &[Expr], + keywords: &[Keyword], + source_type: PySourceType, + ) -> Result { let args = self.args_map(args, keywords)?; match self { UnittestAssert::True @@ -339,7 +360,7 @@ impl UnittestAssert { UnittestAssert::IsNot => CmpOp::IsNot, _ => unreachable!(), }; - let expr = compare(first, cmp_op, second); + let expr = compare(first, cmp_op, second, source_type); Ok(assert(&expr, msg)) } UnittestAssert::In | UnittestAssert::NotIn => { @@ -355,7 +376,7 @@ impl UnittestAssert { } else { CmpOp::NotIn }; - let expr = compare(member, cmp_op, container); + let expr = compare(member, cmp_op, container, source_type); Ok(assert(&expr, msg)) } UnittestAssert::IsNone | UnittestAssert::IsNotNone => { @@ -372,7 +393,7 @@ impl UnittestAssert { range: TextRange::default(), node_index: ruff_python_ast::AtomicNodeIndex::NONE, }); - let expr = compare(expr, cmp_op, &node); + let expr = compare(expr, cmp_op, &node, source_type); Ok(assert(&expr, msg)) } UnittestAssert::IsInstance | UnittestAssert::NotIsInstance => { diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_bool_op.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_bool_op.rs index 9e5259bbb0..f0987d1871 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_bool_op.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_bool_op.rs @@ -466,6 +466,7 @@ fn match_eq_target(expr: &Expr) -> Option<(&Name, &Expr)> { comparators, range: _, node_index: _, + identity_ops: _, }) = expr else { return None; @@ -554,6 +555,7 @@ pub(crate) fn compare_with_tuple(checker: &Checker, expr: &Expr) { left: Box::new(node1.into()), ops: Box::from([CmpOp::In]), comparators: Box::from([node.into()]), + identity_ops: None, range: TextRange::default(), node_index: ruff_python_ast::AtomicNodeIndex::NONE, }; diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_unary_op.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_unary_op.rs index 2bb0aa9dee..4c932cd679 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_unary_op.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_unary_op.rs @@ -158,6 +158,7 @@ pub(crate) fn negation_with_equal_op(checker: &Checker, expr: &Expr, op: UnaryOp comparators, range: _, node_index: _, + identity_ops: _, }) = operand else { return; @@ -189,6 +190,7 @@ pub(crate) fn negation_with_equal_op(checker: &Checker, expr: &Expr, op: UnaryOp left: left.clone(), ops: Box::from([CmpOp::NotEq]), comparators: comparators.clone(), + identity_ops: None, range: TextRange::default(), node_index: ruff_python_ast::AtomicNodeIndex::NONE, }; @@ -214,6 +216,7 @@ pub(crate) fn negation_with_not_equal_op( comparators, range: _, node_index: _, + identity_ops: _, }) = operand else { return; @@ -245,6 +248,7 @@ pub(crate) fn negation_with_not_equal_op( left: left.clone(), ops: Box::from([CmpOp::Eq]), comparators: comparators.clone(), + identity_ops: None, range: TextRange::default(), node_index: ruff_python_ast::AtomicNodeIndex::NONE, }; diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/if_else_block_instead_of_dict_get.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/if_else_block_instead_of_dict_get.rs index 41f3703a57..a484d18333 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/rules/if_else_block_instead_of_dict_get.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/rules/if_else_block_instead_of_dict_get.rs @@ -136,6 +136,7 @@ pub(crate) fn if_else_block_instead_of_dict_get(checker: &Checker, stmt_if: &ast comparators: test_dict, range: _, node_index: _, + identity_ops: _, }) = &**test else { return; @@ -268,6 +269,7 @@ pub(crate) fn if_exp_instead_of_dict_get( comparators: test_dict, range: _, node_index: _, + identity_ops: _, }) = test else { return; diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/if_else_block_instead_of_dict_lookup.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/if_else_block_instead_of_dict_lookup.rs index 6d5d387002..bdfe4d7626 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/rules/if_else_block_instead_of_dict_lookup.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/rules/if_else_block_instead_of_dict_lookup.rs @@ -65,6 +65,7 @@ pub(crate) fn if_else_block_instead_of_dict_lookup(checker: &Checker, stmt_if: & comparators, range: _, node_index: _, + identity_ops: _, }) = test.as_ref() else { return; @@ -152,6 +153,7 @@ pub(crate) fn if_else_block_instead_of_dict_lookup(checker: &Checker, stmt_if: & comparators, range: _, node_index: _, + identity_ops: _, })) => { let Expr::Name(ast::ExprName { id, .. }) = left.as_ref() else { return; diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/needless_bool.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/needless_bool.rs index 08db5e741a..1cdc999113 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/rules/needless_bool.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/rules/needless_bool.rs @@ -250,6 +250,7 @@ pub(crate) fn needless_bool(checker: &Checker, stmt: &Stmt) { ops, left, comparators, + identity_ops, .. }) if matches!( ops.as_ref(), @@ -269,6 +270,10 @@ pub(crate) fn needless_bool(checker: &Checker, stmt: &Stmt) { ops: Box::new([op.negate()]), left: left.clone(), comparators: Box::new([right.clone()]), + // negating `is` gives `is not`, which is still the + // basedpython type test the source spelled; dropping the + // flag would rewrite it to the `!==` identity operator + identity_ops: identity_ops.clone(), range: TextRange::default(), node_index: ruff_python_ast::AtomicNodeIndex::NONE, })) diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/reimplemented_builtin.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/reimplemented_builtin.rs index 20bc7e42f8..d33cb98720 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/rules/reimplemented_builtin.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/rules/reimplemented_builtin.rs @@ -155,6 +155,7 @@ pub(crate) fn convert_for_loop_to_any_all(checker: &Checker, stmt: &Stmt) { comparators, range: _, node_index: _, + identity_ops, }) = &loop_.test { if let ([op], [comparator]) = (&**ops, &**comparators) { @@ -174,6 +175,9 @@ pub(crate) fn convert_for_loop_to_any_all(checker: &Checker, stmt: &Stmt) { left: left.clone(), ops: Box::from([op]), comparators: Box::from([comparator.clone()]), + // the negation keeps the spelling: `is` becomes + // `is not`, never the `!==` identity operator + identity_ops: identity_ops.clone(), range: TextRange::default(), node_index: ruff_python_ast::AtomicNodeIndex::NONE, }; diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/literal_comparisons.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/literal_comparisons.rs index c7b0cde1a1..6ee27706ff 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/literal_comparisons.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/literal_comparisons.rs @@ -403,6 +403,7 @@ pub(crate) fn literal_comparisons(checker: &Checker, compare: &ast::ExprCompare) &compare.left, &ops, &compare.comparators, + compare.identity_ops.as_deref(), compare.into(), tokens, source, @@ -413,6 +414,7 @@ pub(crate) fn literal_comparisons(checker: &Checker, compare: &ast::ExprCompare) &compare.left, &ops, &compare.comparators, + compare.identity_ops.as_deref(), compare.into(), tokens, source, diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/not_tests.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/not_tests.rs index 2f1eb95385..2d37ba4038 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/not_tests.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/not_tests.rs @@ -92,6 +92,7 @@ pub(crate) fn not_tests(checker: &Checker, unary_op: &ast::ExprUnaryOp) { comparators, range: _, node_index: _, + identity_ops, }) = unary_op.operand.as_ref() else { return; @@ -106,6 +107,7 @@ pub(crate) fn not_tests(checker: &Checker, unary_op: &ast::ExprUnaryOp) { left, &[CmpOp::NotIn], comparators, + identity_ops.as_deref(), unary_op.into(), checker.tokens(), checker.source(), @@ -124,6 +126,7 @@ pub(crate) fn not_tests(checker: &Checker, unary_op: &ast::ExprUnaryOp) { left, &[CmpOp::IsNot], comparators, + identity_ops.as_deref(), unary_op.into(), checker.tokens(), checker.source(), diff --git a/crates/ruff_linter/src/rules/pyflakes/rules/invalid_literal_comparisons.rs b/crates/ruff_linter/src/rules/pyflakes/rules/invalid_literal_comparisons.rs index 476a15ad7e..cc33093673 100644 --- a/crates/ruff_linter/src/rules/pyflakes/rules/invalid_literal_comparisons.rs +++ b/crates/ruff_linter/src/rules/pyflakes/rules/invalid_literal_comparisons.rs @@ -3,7 +3,7 @@ use anyhow::{Error, bail}; use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast::helpers; use ruff_python_ast::token::{TokenKind, Tokens}; -use ruff_python_ast::{CmpOp, Expr}; +use ruff_python_ast::{self as ast, CmpOp}; use ruff_text_size::{Ranged, TextRange}; use crate::checkers::ast::Checker; @@ -81,26 +81,15 @@ impl AlwaysFixableViolation for IsLiteral { } /// F632 -pub(crate) fn invalid_literal_comparison( - checker: &Checker, - left: &Expr, - ops: &[CmpOp], - comparators: &[Expr], - expr: &Expr, -) { +pub(crate) fn invalid_literal_comparison(checker: &Checker, compare: &ast::ExprCompare) { // basedpython keeps `is` as python identity only where its right-hand side is a // literal; anywhere else `is` is a type test, and `===` is the spelling that always - // compares identity. telling `===` from `is` takes the tokens, which are worth - // locating up front once the file can contain either - let mut lazy_located = (checker.source_type.is_basedpython() - && ops.iter().any(|op| matches!(op, CmpOp::Is | CmpOp::IsNot))) - .then(|| locate_cmp_ops(expr, checker.tokens())); - let mut left = left; - for (index, (op, right)) in ops.iter().zip(comparators).enumerate() { - let spells_identity = lazy_located - .as_ref() - .and_then(|located| located.get(index)) - .is_some_and(|located_op| located_op.op == *op && located_op.spells_identity); + // compares identity. the parser records which of the two was written, so the tokens + // are needed only to place a fix + let mut lazy_located = None; + let mut left = &*compare.left; + for (index, (op, right)) in compare.ops.iter().zip(&compare.comparators).enumerate() { + let spells_identity = compare.is_identity_operator(index); if matches!(op, CmpOp::Is | CmpOp::IsNot) && (helpers::is_constant_non_singleton(left) @@ -114,20 +103,19 @@ pub(crate) fn invalid_literal_comparison( cmp_op: op.into(), spells_identity, }, - expr.range(), + compare.range(), ); if lazy_located.is_none() { - lazy_located = Some(locate_cmp_ops(expr, checker.tokens())); + lazy_located = Some(locate_cmp_ops(compare.range(), checker.tokens())); } diagnostic.try_set_optional_fix(|| { let located_op = lazy_located.as_ref().and_then(|located| located.get(index)); - // the tokens and the operators come from one parse, so they line up. a - // basedpython file is the one place the two can legitimately disagree, - // because a spelling the token scan does not know reads as a different - // operator there — hence a dropped fix rather than a panic + // the tokens and the operators come from one parse, so they line up in + // either language: basedpython's `===` and `!==` scan to the same + // `CmpOp` the parser recorded for them. a dropped fix rather than a + // panic if that ever stops holding debug_assert!( - checker.source_type.is_basedpython() - || located_op.is_none_or(|located_op| located_op.op == *op), + located_op.is_none_or(|located_op| located_op.op == *op), "located `{:?}` where the comparison has `{op:?}`", located_op.map(|located_op| located_op.op) ); @@ -175,9 +163,9 @@ impl From<&CmpOp> for IsCmpOp { /// /// This method iterates over the token stream and re-identifies [`CmpOp`] nodes, annotating them /// with valid ranges. -fn locate_cmp_ops(expr: &Expr, tokens: &Tokens) -> Vec { +fn locate_cmp_ops(range: TextRange, tokens: &Tokens) -> Vec { let mut tok_iter = tokens - .in_range(expr.range()) + .in_range(range) .iter() .filter(|token| !token.kind().is_trivia()) .peekable(); @@ -227,11 +215,14 @@ fn locate_cmp_ops(expr: &Expr, tokens: &Tokens) -> Vec { }; ops.push(op); } + // basedpython's identity operators, which parse to the same `CmpOp` as + // the `is` keyword — scanned so the operators keep lining up with the + // comparison's own, whichever spelling the source used TokenKind::EqEqEqual => { - ops.push(LocatedCmpOp::identity(token.range(), CmpOp::Is)); + ops.push(LocatedCmpOp::new(token.range(), CmpOp::Is)); } TokenKind::BangEqEqual => { - ops.push(LocatedCmpOp::identity(token.range(), CmpOp::IsNot)); + ops.push(LocatedCmpOp::new(token.range(), CmpOp::IsNot)); } TokenKind::NotEqual => { ops.push(LocatedCmpOp::new(token.range(), CmpOp::NotEq)); @@ -261,11 +252,6 @@ fn locate_cmp_ops(expr: &Expr, tokens: &Tokens) -> Vec { struct LocatedCmpOp { range: TextRange, op: CmpOp, - /// Whether the operator was written with basedpython's `===` / `!==`, the - /// spellings that compare identity there. A plain `is` in a basedpython file - /// is a [parametric type test](https://docs.basedpython.org/features/parametric-type-tests) - /// and parses to the same [`CmpOp`], so the two can only be told apart here. - spells_identity: bool, } impl LocatedCmpOp { @@ -273,14 +259,6 @@ impl LocatedCmpOp { Self { range: range.into(), op, - spells_identity: false, - } - } - - fn identity>(range: T, op: CmpOp) -> Self { - Self { - spells_identity: true, - ..Self::new(range, op) } } } @@ -291,13 +269,13 @@ mod tests { use ruff_python_ast::CmpOp; use ruff_python_parser::parse_expression; - use ruff_text_size::TextSize; + use ruff_text_size::{Ranged, TextSize}; use super::{LocatedCmpOp, locate_cmp_ops}; fn extract_cmp_op_locations(source: &str) -> Result> { let parsed = parse_expression(source)?; - Ok(locate_cmp_ops(parsed.expr(), parsed.tokens())) + Ok(locate_cmp_ops(parsed.expr().range(), parsed.tokens())) } #[test] diff --git a/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__is-literal_F632_basedpython.by.snap b/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__is-literal_F632_basedpython.by.snap index 7473174a37..b6e703c28f 100644 --- a/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__is-literal_F632_basedpython.by.snap +++ b/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__is-literal_F632_basedpython.by.snap @@ -63,3 +63,120 @@ help: Replace `is not` with `!=` 11 + print(x != "a") # F632 12 | | + +F632 [*] Use `==` to compare constant literals + --> F632_basedpython.by:25:9 + | +23 | # source between the operands, so it survives everything that can sit there +24 | def spelling_survives_what_sits_between(x: object) -> None: +25 | a = (1) === (1) # F632 + | ^^^^^^^^^^^ +26 | b = 1 \ +27 | === 1 # F632 + | +help: Replace `===` with `==` + | +24 | def spelling_survives_what_sits_between(x: object) -> None: + - a = (1) === (1) # F632 +25 + a = (1) == (1) # F632 +26 | b = 1 \ + | + +F632 [*] Use `==` to compare constant literals + --> F632_basedpython.by:26:9 + | +24 | def spelling_survives_what_sits_between(x: object) -> None: +25 | a = (1) === (1) # F632 +26 | b = 1 \ + | _________^ +27 | | === 1 # F632 + | |_____________^ +28 | c = (1 === # a comment between the operands +29 | 1) # F632 + | +help: Replace `===` with `==` + | +26 | b = 1 \ + - === 1 # F632 +27 + == 1 # F632 +28 | c = (1 === # a comment between the operands + | + +F632 [*] Use `==` to compare constant literals + --> F632_basedpython.by:28:10 + | +26 | b = 1 \ +27 | === 1 # F632 +28 | c = (1 === # a comment between the operands + | __________^ +29 | | 1) # F632 + | |__________^ +30 | d = 1===1 # F632 + | +help: Replace `===` with `==` + | +27 | === 1 # F632 + - c = (1 === # a comment between the operands +28 + c = (1 == # a comment between the operands +29 | 1) # F632 + | + +F632 [*] Use `==` to compare constant literals + --> F632_basedpython.by:30:9 + | +28 | c = (1 === # a comment between the operands +29 | 1) # F632 +30 | d = 1===1 # F632 + | ^^^^^ +help: Replace `===` with `==` + | +29 | 1) # F632 + - d = 1===1 # F632 +30 + d = 1==1 # F632 +31 | + | + +F632 [*] Use `==` to compare constant literals + --> F632_basedpython.by:35:9 + | +33 | # each operator of a chain answers for its own spelling +34 | def chained(x: object) -> None: +35 | e = x < 1 === 1 # F632 + | ^^^^^^^^^^^ +36 | f = 1 === 1 !== 2 # F632 + | +help: Replace `===` with `==` + | +34 | def chained(x: object) -> None: + - e = x < 1 === 1 # F632 +35 + e = x < 1 == 1 # F632 +36 | f = 1 === 1 !== 2 # F632 + | + +F632 [*] Use `==` to compare constant literals + --> F632_basedpython.by:36:9 + | +34 | def chained(x: object) -> None: +35 | e = x < 1 === 1 # F632 +36 | f = 1 === 1 !== 2 # F632 + | ^^^^^^^^^^^^^ +help: Replace `===` with `==` + | +35 | e = x < 1 === 1 # F632 + - f = 1 === 1 !== 2 # F632 +36 + f = 1 == 1 !== 2 # F632 + | + +F632 [*] Use `!=` to compare constant literals + --> F632_basedpython.by:36:9 + | +34 | def chained(x: object) -> None: +35 | e = x < 1 === 1 # F632 +36 | f = 1 === 1 !== 2 # F632 + | ^^^^^^^^^^^^^ +help: Replace `!==` with `!=` + | +35 | e = x < 1 === 1 # F632 + - f = 1 === 1 !== 2 # F632 +36 + f = 1 === 1 != 2 # F632 + | diff --git a/crates/ruff_linter/src/rules/pylint/rules/repeated_equality_comparison.rs b/crates/ruff_linter/src/rules/pylint/rules/repeated_equality_comparison.rs index 3dc4a2a1a5..a63cdd0d73 100644 --- a/crates/ruff_linter/src/rules/pylint/rules/repeated_equality_comparison.rs +++ b/crates/ruff_linter/src/rules/pylint/rules/repeated_equality_comparison.rs @@ -212,6 +212,7 @@ pub(crate) fn repeated_equality_comparison(checker: &Checker, bool_op: &ast::Exp BoolOp::And => Box::from([CmpOp::NotIn]), }, comparators: Box::from([comparator]), + identity_ops: None, range: bool_op.range(), node_index: AtomicNodeIndex::NONE, }))) diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/outdated_version_block.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/outdated_version_block.rs index 10e9cd34f2..acbb3ca041 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/outdated_version_block.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/outdated_version_block.rs @@ -97,6 +97,7 @@ pub(crate) fn outdated_version_block(checker: &Checker, stmt_if: &StmtIf) { comparators, range: _, node_index: _, + identity_ops: _, }) = &branch.test else { continue; diff --git a/crates/ruff_linter/src/rules/refurb/helpers.rs b/crates/ruff_linter/src/rules/refurb/helpers.rs index f6375410a3..a48766f3a1 100644 --- a/crates/ruff_linter/src/rules/refurb/helpers.rs +++ b/crates/ruff_linter/src/rules/refurb/helpers.rs @@ -72,6 +72,10 @@ pub(super) fn replace_with_identity_check( left: left.clone().into(), ops: [op].into(), comparators: [ast::ExprNoneLiteral::default().into()].into(), + // basedpython writes this test as `is None`, whose target is the type + // `None`: the same runtime check, and the spelling a reader expects. + // recording no `===` is what prints it that way in both languages + identity_ops: None, range: TextRange::default(), node_index: ruff_python_ast::AtomicNodeIndex::NONE, }); diff --git a/crates/ruff_linter/src/rules/refurb/rules/single_item_membership_test.rs b/crates/ruff_linter/src/rules/refurb/rules/single_item_membership_test.rs index feceeb5c89..1b58817e2e 100644 --- a/crates/ruff_linter/src/rules/refurb/rules/single_item_membership_test.rs +++ b/crates/ruff_linter/src/rules/refurb/rules/single_item_membership_test.rs @@ -95,6 +95,8 @@ pub(crate) fn single_item_membership_test( left, &[membership_test.replacement_op()], std::slice::from_ref(item), + // only `in` / `not in`, which have one spelling + None, expr.into(), checker.tokens(), checker.source(), diff --git a/crates/ruff_linter/src/rules/ruff/rules/unnecessary_regular_expression.rs b/crates/ruff_linter/src/rules/ruff/rules/unnecessary_regular_expression.rs index 3342a3bb37..41ee0cd368 100644 --- a/crates/ruff_linter/src/rules/ruff/rules/unnecessary_regular_expression.rs +++ b/crates/ruff_linter/src/rules/ruff/rules/unnecessary_regular_expression.rs @@ -334,6 +334,8 @@ impl<'a> ReFunc<'a> { left: Box::new(left.clone()), ops: Box::new([op]), comparators: Box::new([right.clone()]), + // only ever `==` / `!=` here, which have one spelling + identity_ops: None, range: TextRange::default(), node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) diff --git a/crates/ruff_python_ast/ast.toml b/crates/ruff_python_ast/ast.toml index 0c6859a902..2f385f33f1 100644 --- a/crates/ruff_python_ast/ast.toml +++ b/crates/ruff_python_ast/ast.toml @@ -485,6 +485,20 @@ fields = [ { name = "left", type = "Expr" }, { name = "ops", type = "Box<[CmpOp]>" }, { name = "comparators", type = "Box<[Expr]>" }, + { name = "identity_ops", type = "Box?", skip_visit = true, doc = """basedpython: which operators were written `===` / `!==`. + +basedpython gives the `is` keyword to the type test and spells Python's identity +comparison `===` / `!==`. Both parse to [`CmpOp::Is`](crate::CmpOp::Is) / [`CmpOp::IsNot`](crate::CmpOp::IsNot) so the +AST keeps Python's shape, and the spelling is recorded here instead. `None` when +nothing was written `===` / `!==`, which is every `.py` file — and boxed, because +paying two more words on every comparison would widen `Expr` itself. + +The lexer emits `===` / `!==` for any source that spells them, so a renderer can +print them back from this field alone without knowing which language it is +emitting — a `.py` file that contains them was never valid Python anyway. Read it +through [`ExprCompare::is_identity_operator`], and ask +[`ExprCompare::is_type_test`] for the other half of the question, which is the +one that needs the source type.""" }, ] # The fields must be visited simultaneously custom_source_order = true diff --git a/crates/ruff_python_ast/src/comparable.rs b/crates/ruff_python_ast/src/comparable.rs index dca4ec069b..765c7cb2aa 100644 --- a/crates/ruff_python_ast/src/comparable.rs +++ b/crates/ruff_python_ast/src/comparable.rs @@ -971,6 +971,10 @@ pub struct ExprCompare<'a> { left: Box>, ops: Vec, comparators: Vec>, + /// basedpython: `a is int` and `a === int` share an operator but not a + /// meaning, so two comparisons are only equal when they were spelled the + /// same way + identity_ops: Vec, } #[derive(Debug, PartialEq, Eq, Hash)] @@ -1282,12 +1286,17 @@ impl<'a> From<&'a ast::Expr> for ComparableExpr<'a> { left, ops, comparators, + identity_ops, range: _, node_index: _, }) => Self::Compare(ExprCompare { left: left.into(), ops: ops.iter().copied().map(Into::into).collect(), comparators: comparators.iter().map(Into::into).collect(), + identity_ops: identity_ops + .as_ref() + .map(|identity| identity.ops.to_vec()) + .unwrap_or_default(), }), ast::Expr::Call(ast::ExprCall { func, diff --git a/crates/ruff_python_ast/src/generated.rs b/crates/ruff_python_ast/src/generated.rs index 65886f1a10..086f1078ba 100644 --- a/crates/ruff_python_ast/src/generated.rs +++ b/crates/ruff_python_ast/src/generated.rs @@ -10440,6 +10440,21 @@ pub struct ExprCompare { pub left: Box, pub ops: Box<[crate::CmpOp]>, pub comparators: Box<[Expr]>, + /// basedpython: which operators were written `===` / `!==`. + /// + /// basedpython gives the `is` keyword to the type test and spells Python's identity + /// comparison `===` / `!==`. Both parse to [`CmpOp::Is`](crate::CmpOp::Is) / [`CmpOp::IsNot`](crate::CmpOp::IsNot) so the + /// AST keeps Python's shape, and the spelling is recorded here instead. `None` when + /// nothing was written `===` / `!==`, which is every `.py` file — and boxed, because + /// paying two more words on every comparison would widen `Expr` itself. + /// + /// The lexer emits `===` / `!==` for any source that spells them, so a renderer can + /// print them back from this field alone without knowing which language it is + /// emitting — a `.py` file that contains them was never valid Python anyway. Read it + /// through [`ExprCompare::is_identity_operator`], and ask + /// [`ExprCompare::is_type_test`] for the other half of the question, which is the + /// one that needs the source type. + pub identity_ops: Option>, } /// A call expression whose end offset is derived from its arguments. diff --git a/crates/ruff_python_ast/src/helpers.rs b/crates/ruff_python_ast/src/helpers.rs index 9733d556bc..165e1f08ad 100644 --- a/crates/ruff_python_ast/src/helpers.rs +++ b/crates/ruff_python_ast/src/helpers.rs @@ -2766,6 +2766,7 @@ pub fn generate_comparison( left: &Expr, ops: &[CmpOp], comparators: &[Expr], + identity_ops: Option<&crate::IdentityOperators>, parent: AnyNodeRef, tokens: &Tokens, source: &str, @@ -2779,7 +2780,14 @@ pub fn generate_comparison( &source[parenthesized_range(left.into(), parent, tokens).unwrap_or(left.range())], ); - for (op, comparator) in ops.iter().zip(comparators) { + for (index, (op, comparator)) in ops.iter().zip(comparators).enumerate() { + // basedpython writes python's identity comparison `===` / `!==` and + // gives the `is` keyword to a type test. printing `is` for an operator + // the source wrote `===` would rewrite one into the other + let identity = identity_ops + .and_then(|identity| identity.ops.get(index)) + .copied() + .unwrap_or(false); // Add the operator. contents.push_str(match op { CmpOp::Eq => " == ", @@ -2790,6 +2798,8 @@ pub fn generate_comparison( CmpOp::GtE => " >= ", CmpOp::In => " in ", CmpOp::NotIn => " not in ", + CmpOp::Is if identity => " === ", + CmpOp::IsNot if identity => " !== ", CmpOp::Is => " is ", CmpOp::IsNot => " is not ", }); diff --git a/crates/ruff_python_ast/src/node.rs b/crates/ruff_python_ast/src/node.rs index 929e51c464..34716b388f 100644 --- a/crates/ruff_python_ast/src/node.rs +++ b/crates/ruff_python_ast/src/node.rs @@ -96,6 +96,7 @@ impl ast::ExprCompare { left, ops, comparators, + identity_ops: _, range: _, node_index: _, } = self; diff --git a/crates/ruff_python_ast/src/nodes.rs b/crates/ruff_python_ast/src/nodes.rs index a6adbd4aad..9c1e506098 100644 --- a/crates/ruff_python_ast/src/nodes.rs +++ b/crates/ruff_python_ast/src/nodes.rs @@ -2799,6 +2799,83 @@ impl ExprNamed { } } +/// basedpython: which operators of an [`ExprCompare`](crate::ExprCompare) were +/// written `===` / `!==`, python's identity comparison. +/// +/// basedpython gives the `is` keyword to the type test and spells identity +/// `===` / `!==`. Both parse to the same [`CmpOp`](crate::CmpOp), so the AST +/// keeps this alongside to say which was written. +/// +/// Boxed behind an `Option` on the node: a comparison written any other way +/// carries none of this, and paying two words for it on every comparison would +/// widen `Expr` itself. +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(feature = "get-size", derive(get_size2::GetSize))] +pub struct IdentityOperators { + /// one entry per operator, in the same order as + /// [`ExprCompare::ops`](crate::ExprCompare::ops) + pub ops: Box<[bool]>, +} + +impl IdentityOperators { + /// the side table to store on a node, or `None` when no operator was + /// written `===` / `!==`. + /// + /// `operators` is the comparison's own operator count: an entry past it + /// would never be read, and one short of it would answer `false` for an + /// operator that was written `===`. + pub fn into_stored(ops: Vec, operators: usize) -> Option> { + debug_assert!( + ops.len() <= operators, + "identity spellings outnumber the operators they belong to" + ); + ops.iter().any(|identity| *identity).then(|| { + Box::new(Self { + ops: ops.into_boxed_slice(), + }) + }) + } +} + +impl crate::ExprCompare { + /// basedpython: whether the operator at `index` was written `===` / `!==`, + /// which is Python's identity comparison. + /// + /// `false` for every operator that is not an identity comparison and for an + /// index past the end, so a caller never has to check the encoding or the + /// length first. A renderer can print `===` from this answer alone, without + /// knowing which language it is emitting: the lexer produces the token for + /// any source that spells it, and a `.py` file that does was never valid + /// Python. + pub fn is_identity_operator(&self, index: usize) -> bool { + self.identity_ops + .as_ref() + .and_then(|identity| identity.ops.get(index)) + .copied() + .unwrap_or(false) + } + + /// basedpython: whether the operator at `index` is a *type test* — `is` or + /// `is not` spelled with the keyword, which asks whether the left operand + /// has the type its right-hand side names. + /// + /// The source type is required because the same operator means Python's + /// identity comparison in a `.py` file, where there is no type test at all. + pub fn is_type_test(&self, index: usize, source_type: crate::PySourceType) -> bool { + source_type.is_basedpython() + && matches!( + self.ops.get(index), + Some(crate::CmpOp::Is | crate::CmpOp::IsNot) + ) + && !self.is_identity_operator(index) + } + + /// basedpython: whether any operator of this comparison is a type test. + pub fn has_type_test(&self, source_type: crate::PySourceType) -> bool { + (0..self.ops.len()).any(|index| self.is_type_test(index, source_type)) + } +} + impl ExprList { pub fn iter(&self) -> std::slice::Iter<'_, Expr> { self.elts.iter() @@ -4552,7 +4629,10 @@ mod tests { assert_eq!(std::mem::size_of::(), 16); assert_eq!(std::mem::size_of::(), 48); assert_eq!(std::mem::size_of::(), 64); - assert_eq!(std::mem::size_of::(), 56); + // basedpython: a comparison carries which of its operators were written + // `===` / `!==`, boxed so this stays inside `ExprCall`'s width and + // `Expr` does not grow + assert_eq!(std::mem::size_of::(), 64); assert_eq!(std::mem::size_of::(), 40); assert_eq!(std::mem::size_of::(), 56); assert_eq!(std::mem::size_of::(), 12); diff --git a/crates/ruff_python_ast/src/visitor.rs b/crates/ruff_python_ast/src/visitor.rs index caac8b36ee..b22b57ca74 100644 --- a/crates/ruff_python_ast/src/visitor.rs +++ b/crates/ruff_python_ast/src/visitor.rs @@ -569,6 +569,7 @@ pub fn walk_expr<'a, V: Visitor<'a> + ?Sized>(visitor: &mut V, expr: &'a Expr) { left, ops, comparators, + identity_ops: _, range: _, node_index: _, }) => { diff --git a/crates/ruff_python_ast/src/visitor/transformer.rs b/crates/ruff_python_ast/src/visitor/transformer.rs index 9b3221a7af..f189d53375 100644 --- a/crates/ruff_python_ast/src/visitor/transformer.rs +++ b/crates/ruff_python_ast/src/visitor/transformer.rs @@ -556,6 +556,7 @@ pub fn walk_expr(visitor: &V, expr: &mut Expr) { left, ops, comparators, + identity_ops: _, range: _, node_index: _, }) => { diff --git a/crates/ruff_python_codegen/src/generator.rs b/crates/ruff_python_codegen/src/generator.rs index f56a6f497e..887832b328 100644 --- a/crates/ruff_python_codegen/src/generator.rs +++ b/crates/ruff_python_codegen/src/generator.rs @@ -1478,17 +1478,26 @@ impl<'a> Generator<'a> { self.unparse_expr(value, precedence::MAX); }); } - Expr::Compare(ast::ExprCompare { - left, - ops, - comparators, - range: _, - node_index: _, - }) => { + Expr::Compare(compare) => { + let ast::ExprCompare { + left, + ops, + comparators, + identity_ops: _, + range: _, + node_index: _, + } = compare; group_if!(precedence::CMP, { let new_lvl = precedence::CMP + 1; self.unparse_expr(left, new_lvl); - for (op, cmp) in ops.iter().zip(comparators) { + for (index, (op, cmp)) in ops.iter().zip(comparators).enumerate() { + // basedpython gives the `is` keyword to the type test and + // spells python's identity comparison `===` / `!==`. both + // parse to the same operator, so printing `is` for one + // the source wrote `===` would quietly turn an identity + // check into a type test. only basedpython records the + // spelling, so this needs no mode of its own + let identity = compare.is_identity_operator(index); let op = match op { CmpOp::Eq => " == ", CmpOp::NotEq => " != ", @@ -1496,6 +1505,8 @@ impl<'a> Generator<'a> { CmpOp::LtE => " <= ", CmpOp::Gt => " > ", CmpOp::GtE => " >= ", + CmpOp::Is if identity => " === ", + CmpOp::IsNot if identity => " !== ", CmpOp::Is => " is ", CmpOp::IsNot => " is not ", CmpOp::In => " in ", diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/identity_operators.by b/crates/ruff_python_formatter/resources/test/fixtures/ruff/identity_operators.by index c10a156483..566905d4c4 100644 --- a/crates/ruff_python_formatter/resources/test/fixtures/ruff/identity_operators.by +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/identity_operators.by @@ -11,8 +11,10 @@ b = xs !== xs c = xs is list[int] d = xs is not list[int] -# chained comparisons mix the spellings freely -e = xs === xs !== None is list[int] is not list[str] +# a chain of identity comparisons is ordinary python, and each operator keeps +# the spelling it was written with. a type test may not join a chain at all — +# python's chaining rule would ask whether the *class* has the next type +e = xs === xs !== None # mixed with value comparisons f = 1 === 1 == 1 @@ -21,6 +23,16 @@ f = 1 === 1 == 1 g = (xs) === (xs) h = (xs) !== (xs) +# a comment between the operands, and a line continuation — the two the old +# source-gap scan read wrong +i = ( + xs + # which one is this + === xs +) +j = xs \ + === xs + # inside a condition if xs === xs: pass diff --git a/crates/ruff_python_formatter/src/expression/binary_like.rs b/crates/ruff_python_formatter/src/expression/binary_like.rs index ec658172be..77738c02aa 100644 --- a/crates/ruff_python_formatter/src/expression/binary_like.rs +++ b/crates/ruff_python_formatter/src/expression/binary_like.rs @@ -8,7 +8,7 @@ use ruff_python_ast::{ Expr, ExprAttribute, ExprBinOp, ExprBoolOp, ExprCompare, ExprUnaryOp, StringLike, UnaryOp, }; use ruff_python_trivia::{SimpleToken, SimpleTokenKind, SimpleTokenizer, TriviaRanges}; -use ruff_text_size::{Ranged, TextLen, TextRange}; +use ruff_text_size::{Ranged, TextRange}; use crate::comments::{Comments, SourceComment, leading_comments, trailing_comments}; use crate::expression::OperatorPrecedence; @@ -65,20 +65,16 @@ impl<'a> BinaryLike<'a> { if let Some((last_expression, middle_expressions)) = compare.comparators.split_last() { let (last_operator, middle_operators) = compare.ops.split_last().unwrap(); - // the source gap preceding each operator, so a basedpython - // `===` / `!==` can be told apart from the `is` / `is not` it - // shares a `CmpOp` with - let mut gap_start = compare.left.end(); - - for (operator, expression) in middle_operators.iter().zip(middle_expressions) { + for (index, (operator, expression)) in + middle_operators.iter().zip(middle_expressions).enumerate() + { parts.push(OperandOrOperator::Operator(Operator { symbol: OperatorSymbol::Comparator( *operator, - TextRange::new(gap_start, expression.start()), + compare.is_identity_operator(index), ), trailing_comments: &[], })); - gap_start = expression.end(); rec(Operand::Middle { expression }, comments, trivia, parts); } @@ -86,7 +82,7 @@ impl<'a> BinaryLike<'a> { parts.push(OperandOrOperator::Operator(Operator { symbol: OperatorSymbol::Comparator( *last_operator, - TextRange::new(gap_start, last_expression.start()), + compare.is_identity_operator(middle_operators.len()), ), trailing_comments: &[], })); @@ -1022,12 +1018,11 @@ impl Format> for Operator<'_> { #[derive(Copy, Clone, Debug)] enum OperatorSymbol { Binary(ruff_python_ast::Operator), - /// The comparison operator together with the source gap that precedes it — - /// the span between the end of the left operand and the start of the right - /// one. basedpython spells identity as `===` / `!==` but parses both to the - /// `CmpOp` of `is` / `is not` (whose surface form is instead a parametric - /// type test), so the spelling can only be recovered from that source. - Comparator(ruff_python_ast::CmpOp, TextRange), + /// The comparison operator, and whether it was written `===` / `!==`. + /// basedpython spells python's identity comparison that way and parses it + /// to the same `CmpOp` as the `is` keyword, so the operator alone does not + /// say which one to print back. + Comparator(ruff_python_ast::CmpOp, bool), Bool(ruff_python_ast::BoolOp), } @@ -1045,40 +1040,33 @@ impl OperatorSymbol { } } -/// The source range of a basedpython `===` / `!==` identity operator written in -/// `gap` — the span between the two operands it joins — or `None` when the -/// operator is spelled `is` / `is not`, or is not an identity comparison at all. -fn identity_operator_range( - context: &PyFormatContext, +/// The basedpython spelling of an identity comparison — `===` / `!==` — or +/// `None` when the operator is written `is` / `is not`, or is not an identity +/// comparison at all. +/// +/// Only basedpython records the spelling, so a `.py` file always takes the +/// `None` arm and prints python's `is`. +fn identity_operator_symbol( operator: ruff_python_ast::CmpOp, - gap: TextRange, -) -> Option { - let symbol = match operator { - ruff_python_ast::CmpOp::Is => "===", - ruff_python_ast::CmpOp::IsNot => "!==", - _ => return None, - }; - - let source = context.source(); - // the operator is the first token in the gap that is neither trivia nor a - // closing parenthesis of the left operand (`(a) === b`) - let start = SimpleTokenizer::new(source, gap) - .skip_trivia() - .find(|token| token.kind() != SimpleTokenKind::RParen)? - .start(); - - source[usize::from(start)..] - .starts_with(symbol) - .then(|| TextRange::at(start, symbol.text_len())) + identity: bool, +) -> Option<&'static str> { + if !identity { + return None; + } + match operator { + ruff_python_ast::CmpOp::Is => Some("==="), + ruff_python_ast::CmpOp::IsNot => Some("!=="), + _ => None, + } } impl Format> for OperatorSymbol { fn fmt(&self, f: &mut Formatter>) -> FormatResult<()> { match self { OperatorSymbol::Binary(operator) => operator.format().fmt(f), - OperatorSymbol::Comparator(operator, gap) => { - match identity_operator_range(f.context(), *operator, *gap) { - Some(range) => source_text_slice(range).fmt(f), + OperatorSymbol::Comparator(operator, identity) => { + match identity_operator_symbol(*operator, *identity) { + Some(symbol) => token(symbol).fmt(f), None => operator.format().fmt(f), } } diff --git a/crates/ruff_python_formatter/src/expression/mod.rs b/crates/ruff_python_formatter/src/expression/mod.rs index a88e740e26..39f21dfd33 100644 --- a/crates/ruff_python_formatter/src/expression/mod.rs +++ b/crates/ruff_python_formatter/src/expression/mod.rs @@ -743,6 +743,7 @@ impl<'input> CanOmitOptionalParenthesesVisitor<'input> { left: _, ops, comparators: _, + identity_ops: _, }) => { self.update_max_precedence_with_count( OperatorPrecedence::Comparator, diff --git a/crates/ruff_python_formatter/tests/snapshots/format@identity_operators.by.snap b/crates/ruff_python_formatter/tests/snapshots/format@identity_operators.by.snap index 098d728222..7605789ec4 100644 --- a/crates/ruff_python_formatter/tests/snapshots/format@identity_operators.by.snap +++ b/crates/ruff_python_formatter/tests/snapshots/format@identity_operators.by.snap @@ -17,8 +17,10 @@ b = xs !== xs c = xs is list[int] d = xs is not list[int] -# chained comparisons mix the spellings freely -e = xs === xs !== None is list[int] is not list[str] +# a chain of identity comparisons is ordinary python, and each operator keeps +# the spelling it was written with. a type test may not join a chain at all — +# python's chaining rule would ask whether the *class* has the next type +e = xs === xs !== None # mixed with value comparisons f = 1 === 1 == 1 @@ -27,6 +29,16 @@ f = 1 === 1 == 1 g = (xs) === (xs) h = (xs) !== (xs) +# a comment between the operands, and a line continuation — the two the old +# source-gap scan read wrong +i = ( + xs + # which one is this + === xs +) +j = xs \ + === xs + # inside a condition if xs === xs: pass @@ -53,8 +65,10 @@ b = xs !== xs c = xs is list[int] d = xs is not list[int] -# chained comparisons mix the spellings freely -e = xs === xs !== None is list[int] is not list[str] +# a chain of identity comparisons is ordinary python, and each operator keeps +# the spelling it was written with. a type test may not join a chain at all — +# python's chaining rule would ask whether the *class* has the next type +e = xs === xs !== None # mixed with value comparisons f = 1 === 1 == 1 @@ -63,6 +77,15 @@ f = 1 === 1 == 1 g = (xs) === (xs) h = (xs) !== (xs) +# a comment between the operands, and a line continuation — the two the old +# source-gap scan read wrong +i = ( + xs + # which one is this + === xs +) +j = xs === xs + # inside a condition if xs === xs: pass diff --git a/crates/ruff_python_parser/resources/valid/expressions/identity_compare.py b/crates/ruff_python_parser/resources/valid/expressions/identity_compare.py index e456a03c87..e460bb77b6 100644 --- a/crates/ruff_python_parser/resources/valid/expressions/identity_compare.py +++ b/crates/ruff_python_parser/resources/valid/expressions/identity_compare.py @@ -1,7 +1,10 @@ -# basedpython identity-comparison operators +# basedpython identity-comparison operators. this fixture parses in python +# mode, where the lexer still produces the tokens — a `.py` file spelling them +# was never valid python, and the parser's job here is only to keep the shape x === y x !== y -# chained with other comparisons -a === b is not c +# chained with each other. a chain mixing in the `is` keyword is rejected in +# basedpython mode, and has its own test in `parser::tests` +a === b !== c a !== b diff --git a/crates/ruff_python_parser/src/error.rs b/crates/ruff_python_parser/src/error.rs index 30f86d7a28..91a33e91ff 100644 --- a/crates/ruff_python_parser/src/error.rs +++ b/crates/ruff_python_parser/src/error.rs @@ -139,6 +139,8 @@ pub enum ParseErrorType { DuplicateTypeParamSeparator(&'static str), /// basedpython: a bound range `T: Lower..Upper` was missing one of its ends. IncompleteTypeParamBoundRange, + /// basedpython: an `is` type test appeared in a chained comparison. + ChainedTypeTest, /// An unparenthesized named expression was found where it is not allowed. UnparenthesizedNamedExpression, @@ -308,6 +310,10 @@ impl std::fmt::Display for ParseErrorType { "Type parameter list cannot have two `{separator}` separators" ) } + ParseErrorType::ChainedTypeTest => f.write_str( + "`is` type test cannot be chained with another comparison; \ + split it into separate tests joined with `and`", + ), ParseErrorType::IncompleteTypeParamBoundRange => f.write_str( "Type parameter bound range requires both a lower and an upper bound, as in `T: int..object`", ), diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index 9d111825ad..ca974a91d0 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -2259,6 +2259,17 @@ impl<'src> Parser<'src> { op: CmpOp, context: ExpressionContext, ) -> ast::ExprCompare { + // built only once an operator is actually written `===` / `!==`, so an + // ordinary comparison — every one in a `.py` file — allocates nothing + let mut identity_ops: Vec = Vec::new(); + let record_identity = |identity_ops: &mut Vec, index: usize, identity: bool| { + if identity_ops.is_empty() && !identity { + return; + } + identity_ops.resize(index, false); + identity_ops.push(identity); + }; + record_identity(&mut identity_ops, 0, self.at_identity_operator(op)); self.bump_cmp_op(op); let comparators_snapshot = self.expr_scratch.snapshot(); @@ -2288,19 +2299,58 @@ impl<'src> Parser<'src> { break; }; + record_identity( + &mut identity_ops, + operators.len(), + self.at_identity_operator(next_op), + ); self.bump_cmp_op(next_op); operators.push(next_op); } + let range = self.node_range(start); + // a type test's right-hand side is a type expression, and python's + // chaining rule makes each operand the *next* comparison's left operand + // — so `a is int is str` becomes `a is int and int is str`, where the + // class `int` is asked whether it has the type `str`. only a type test + // that something follows creates that conflict; a trailing one + // (`a < b is None`) hands nothing on, and chains as it always did + if self.options.is_basedpython + && operators + .iter() + .enumerate() + .take(operators.len().saturating_sub(1)) + .any(|(index, op)| { + matches!(op, CmpOp::Is | CmpOp::IsNot) + && !identity_ops.get(index).copied().unwrap_or(false) + }) + { + self.add_error(ParseErrorType::ChainedTypeTest, range); + } + let operator_count = operators.len(); ast::ExprCompare { left: Box::new(lhs), ops: operators.into_boxed_slice(), comparators: self.expr_scratch.take(comparators_snapshot), - range: self.node_range(start), + // the common case — a `.py` file, or basedpython written with the + // `is` keyword — stores nothing at all + identity_ops: ast::IdentityOperators::into_stored(identity_ops, operator_count), + range, node_index: AtomicNodeIndex::NONE, } } + /// basedpython: whether the comparison operator the parser is positioned at + /// was written `===` / `!==` — python's identity comparison, which the lexer + /// emits as its own token and which shares a [`CmpOp`] with the `is` keyword. + fn at_identity_operator(&self, op: CmpOp) -> bool { + matches!(op, CmpOp::Is | CmpOp::IsNot) + && self.at_ts(TokenSet::new([ + TokenKind::EqEqEqual, + TokenKind::BangEqEqual, + ])) + } + /// Parses all kinds of strings and implicitly concatenated strings. /// /// # Panics diff --git a/crates/ruff_python_parser/src/parser/helpers.rs b/crates/ruff_python_parser/src/parser/helpers.rs index e48886b996..05074e6ac9 100644 --- a/crates/ruff_python_parser/src/parser/helpers.rs +++ b/crates/ruff_python_parser/src/parser/helpers.rs @@ -44,14 +44,13 @@ pub(super) const fn token_kind_to_cmp_op( (TokenKind::In, _) => CmpOp::In, (TokenKind::EqEqual, _) => CmpOp::Eq, // basedpython: `===` is the identity comparison (Python's `is`). - // we map it to CmpOp::Is so the AST matches Python semantics; the - // forward transpile distinguishes it from the basedpython `is` - // keyword by looking at the source text + // we map it to CmpOp::Is so the AST matches Python semantics, and the + // parser records the spelling in `ExprCompare::identity_ops` so nothing + // downstream has to read it back out of the source (TokenKind::EqEqEqual, _) => CmpOp::Is, // basedpython: `!==` is the negated identity comparison (Python's - // `is not`). mapped to CmpOp::IsNot; the forward transpile - // distinguishes it from the basedpython `is not` keyword pair by - // looking at the source text + // `is not`). mapped to CmpOp::IsNot, with the spelling recorded the + // same way (TokenKind::BangEqEqual, _) => CmpOp::IsNot, (TokenKind::NotEqual, _) => CmpOp::NotEq, (TokenKind::Less, _) => CmpOp::Lt, diff --git a/crates/ruff_python_parser/src/parser/snapshots/ruff_python_parser__parser__tests__ipython_escape_commands.snap b/crates/ruff_python_parser/src/parser/snapshots/ruff_python_parser__parser__tests__ipython_escape_commands.snap index b5b94a8af2..99c2c06bcf 100644 --- a/crates/ruff_python_parser/src/parser/snapshots/ruff_python_parser__parser__tests__ipython_escape_commands.snap +++ b/crates/ruff_python_parser/src/parser/snapshots/ruff_python_parser__parser__tests__ipython_escape_commands.snap @@ -194,6 +194,7 @@ Module( }, ), ], + identity_ops: None, }, ), ), diff --git a/crates/ruff_python_parser/src/parser/tests.rs b/crates/ruff_python_parser/src/parser/tests.rs index 82f693be85..c9e126d609 100644 --- a/crates/ruff_python_parser/src/parser/tests.rs +++ b/crates/ruff_python_parser/src/parser/tests.rs @@ -39,6 +39,73 @@ fn parse_basedpython_module_with_errors(source: &str) -> Parsed { .unwrap() } +/// the identity spelling recorded for each operator of the single comparison in +/// `source`, parsed as basedpython +fn identity_spellings(source: &str) -> Vec { + let parsed = parse_basedpython_module(source); + let [Stmt::Expr(statement)] = parsed.syntax().body.as_slice() else { + panic!("expected a single expression statement: {source}"); + }; + let Expr::Compare(compare) = statement.value.as_ref() else { + panic!("expected a comparison: {source}"); + }; + (0..compare.ops.len()) + .map(|index| compare.is_identity_operator(index)) + .collect() +} + +#[test] +fn identity_operators_are_recorded_per_operator() { + // the two spellings share a `CmpOp`, so the parser records which it saw — + // and records nothing at all when the answer is "none of them" + assert_eq!(identity_spellings("a === b"), vec![true]); + assert_eq!(identity_spellings("a !== b"), vec![true]); + assert_eq!(identity_spellings("a is b"), vec![false]); + assert_eq!(identity_spellings("a == b"), vec![false]); + // a spelling recorded for one operator does not shift the others + assert_eq!(identity_spellings("a == b === c"), vec![false, true]); + assert_eq!(identity_spellings("a === b == c"), vec![true, false]); + assert_eq!( + identity_spellings("a === b !== c === d"), + vec![true, true, true] + ); +} + +#[test] +fn a_type_test_may_not_be_chained() { + // python chains `a is int is str` into `a is int and int is str`, whose + // second half asks whether the class `int` has the type `str` + let parsed = parse_basedpython_module_with_errors("a is int is str\n"); + let errors = parsed.errors(); + assert!( + errors + .iter() + .any(|error| matches!(error.error, ParseErrorType::ChainedTypeTest)), + "expected a chained-type-test error, got {errors:?}" + ); + // the reported range covers the whole comparison, which is what has to change + let error = errors + .iter() + .find(|error| matches!(error.error, ParseErrorType::ChainedTypeTest)) + .unwrap(); + assert_eq!(&"a is int is str\n"[error.location], "a is int is str"); +} + +#[test] +fn a_trailing_type_test_still_chains() { + // nothing follows it, so no type expression becomes the next comparison's + // left operand — the conflict the rejection exists for + let parsed = parse_basedpython_module_with_errors("a < b is None\n"); + assert!( + !parsed + .errors() + .iter() + .any(|error| matches!(error.error, ParseErrorType::ChainedTypeTest)), + "a trailing type test is not a chained one: {:?}", + parsed.errors() + ); +} + /// The keyword-argument names of the single call statement in `source`, each /// with the way it was spelled. fn keyword_names(source: &str) -> Vec<(String, ruff_python_ast::KeywordKey)> { diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_equals.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_equals.snap index 7a9edf207f..9b727a1956 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_equals.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_equals.snap @@ -49,6 +49,7 @@ expression: suite }, ), ], + identity_ops: None, }, ), debug_text: None, diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_not_equals.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_not_equals.snap index 3ab964282e..499e7dd3d5 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_not_equals.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_not_equals.snap @@ -49,6 +49,7 @@ expression: suite }, ), ], + identity_ops: None, }, ), debug_text: None, diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_equals.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_equals.snap index c7baa8a513..70f5bae3ff 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_equals.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_equals.snap @@ -48,6 +48,7 @@ expression: suite }, ), ], + identity_ops: None, }, ), debug_text: None, diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_not_equals.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_not_equals.snap index bf1180fe07..b8c3e457ae 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_not_equals.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_not_equals.snap @@ -48,6 +48,7 @@ expression: suite }, ), ], + identity_ops: None, }, ), debug_text: None, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__invalid_order.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__invalid_order.py.snap index 17ed357836..cfd2db5e3d 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__invalid_order.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__invalid_order.py.snap @@ -46,6 +46,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -90,6 +91,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__invalid_rhs_expression.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__invalid_rhs_expression.py.snap index 642c1d4402..5506702509 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__invalid_rhs_expression.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__invalid_rhs_expression.py.snap @@ -76,6 +76,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -117,6 +118,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__missing_rhs_0.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__missing_rhs_0.py.snap index b40badc953..5b39fbc339 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__missing_rhs_0.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__missing_rhs_0.py.snap @@ -39,6 +39,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__missing_rhs_2.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__missing_rhs_2.py.snap index a9019dc09e..95295d193a 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__missing_rhs_2.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__missing_rhs_2.py.snap @@ -39,6 +39,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__named_expression.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__named_expression.py.snap index 7b1a47a58d..691867e8b2 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__named_expression.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__named_expression.py.snap @@ -39,6 +39,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -110,6 +111,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__starred_expression.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__starred_expression.py.snap index 4b546bf98c..053c2434fa 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__starred_expression.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__starred_expression.py.snap @@ -46,6 +46,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -86,6 +87,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -123,6 +125,7 @@ Module( }, ), ], + identity_ops: None, }, ), ctx: Load, @@ -163,6 +166,7 @@ Module( }, ), ], + identity_ops: None, }, ), ctx: Load, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__dict__double_star.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__dict__double_star.py.snap index d8ba224fd8..e15f5a3659 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__dict__double_star.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__dict__double_star.py.snap @@ -442,6 +442,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -486,6 +487,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -530,6 +532,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__list__star_expression_precedence.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__list__star_expression_precedence.py.snap index 52c1ef0432..429b8d5fdf 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__list__star_expression_precedence.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__list__star_expression_precedence.py.snap @@ -86,6 +86,7 @@ Module( }, ), ], + identity_ops: None, }, ), ctx: Load, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__generator.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__generator.py.snap index 617832fc69..b5ff137f49 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__generator.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__generator.py.snap @@ -90,6 +90,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__tuple_starred_expr.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__tuple_starred_expr.py.snap index 3dcdb1c7b9..4593ae60d8 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__tuple_starred_expr.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__tuple_starred_expr.py.snap @@ -48,6 +48,7 @@ Module( }, ), ], + identity_ops: None, }, ), ctx: Load, @@ -90,6 +91,7 @@ Module( }, ), ], + identity_ops: None, }, ), ctx: Load, @@ -720,6 +722,7 @@ Module( }, ), ], + identity_ops: None, }, ), ctx: Load, @@ -762,6 +765,7 @@ Module( }, ), ], + identity_ops: None, }, ), ctx: Load, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__set__star_expression_precedence.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__set__star_expression_precedence.py.snap index 5059784a2b..f4ab2b2c11 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__set__star_expression_precedence.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__set__star_expression_precedence.py.snap @@ -85,6 +85,7 @@ Module( }, ), ], + identity_ops: None, }, ), ctx: Load, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target.py.snap index 6da0d01b1f..4597ac6bc8 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target.py.snap @@ -307,6 +307,7 @@ Module( }, ), ], + identity_ops: None, }, ), ), diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_binary_expr.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_binary_expr.py.snap index ffcc5f43b8..4fcc9fb2a7 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_binary_expr.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_binary_expr.py.snap @@ -40,6 +40,7 @@ Module( }, ), ], + identity_ops: None, }, ), pattern: None, @@ -98,6 +99,7 @@ Module( }, ), ], + identity_ops: None, }, ), pattern: None, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_in_keyword.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_in_keyword.py.snap index d5ef96f286..b198947628 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_in_keyword.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_in_keyword.py.snap @@ -56,6 +56,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], @@ -125,6 +126,7 @@ Module( }, ), ], + identity_ops: None, }, ), arguments: Arguments { @@ -193,6 +195,7 @@ Module( }, ), ], + identity_ops: None, }, ), pattern: None, @@ -256,6 +259,7 @@ Module( }, ), ], + identity_ops: None, }, ), Name( @@ -336,6 +340,7 @@ Module( }, ), ], + identity_ops: None, }, ), Name( @@ -411,6 +416,7 @@ Module( }, ), ], + identity_ops: None, }, ), Name( diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@statements__invalid_assignment_targets.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@statements__invalid_assignment_targets.py.snap index d9762e8efe..477401a8d4 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@statements__invalid_assignment_targets.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@statements__invalid_assignment_targets.py.snap @@ -853,6 +853,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@statements__invalid_augmented_assignment_target.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@statements__invalid_augmented_assignment_target.py.snap index 84ed5916fe..6cc5335145 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@statements__invalid_augmented_assignment_target.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@statements__invalid_augmented_assignment_target.py.snap @@ -732,6 +732,7 @@ Module( }, ), ], + identity_ops: None, }, ), op: Add, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@while_stmt_missing_colon.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@while_stmt_missing_colon.py.snap index 9ad6e62132..1555b0189c 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@while_stmt_missing_colon.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@while_stmt_missing_colon.py.snap @@ -40,6 +40,7 @@ Module( }, ), ], + identity_ops: None, }, ), body: [ diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@ambiguous_lpar_with_items_binary_expr.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@ambiguous_lpar_with_items_binary_expr.py.snap index 279f8ac0b9..dac46093f8 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@ambiguous_lpar_with_items_binary_expr.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@ambiguous_lpar_with_items_binary_expr.py.snap @@ -98,6 +98,7 @@ Module( }, ), ], + identity_ops: None, }, ), optional_vars: None, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__await.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__await.py.snap index ce65176739..f2057516eb 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__await.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__await.py.snap @@ -390,6 +390,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__compare.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__compare.py.snap index b81211f86c..db6cc2c01e 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__compare.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__compare.py.snap @@ -39,6 +39,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -72,6 +73,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -105,6 +107,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -138,6 +141,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -171,6 +175,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -204,6 +209,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -237,6 +243,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -270,6 +277,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -303,6 +311,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -336,6 +345,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -405,6 +415,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -492,6 +503,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -530,6 +542,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -585,6 +598,7 @@ Module( }, ), ], + identity_ops: None, }, ), Name( @@ -639,6 +653,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -679,6 +694,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -784,6 +800,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__dictionary_comprehension.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__dictionary_comprehension.py.snap index f97cfa5d14..f0d0213c9f 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__dictionary_comprehension.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__dictionary_comprehension.py.snap @@ -336,6 +336,7 @@ Module( }, ), ], + identity_ops: None, }, ), BoolOp( @@ -515,6 +516,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], @@ -645,6 +647,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__f_string.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__f_string.py.snap index f289d34a95..4b981ac072 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__f_string.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__f_string.py.snap @@ -370,6 +370,7 @@ Module( }, ), ], + identity_ops: None, }, ), debug_text: None, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__generator.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__generator.py.snap index 3ff8b3044d..1087249e6a 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__generator.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__generator.py.snap @@ -162,6 +162,7 @@ Module( }, ), ], + identity_ops: None, }, ), BoolOp( @@ -316,6 +317,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], @@ -437,6 +439,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__identity_compare.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__identity_compare.py.snap index 82a85e4db9..cae10a971b 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__identity_compare.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__identity_compare.py.snap @@ -8,20 +8,20 @@ input_file: crates/ruff_python_parser/resources/valid/expressions/identity_compa Module( ModModule { node_index: NodeIndex(None), - range: 0..119, + range: 0..410, body: [ Expr( StmtExpr { node_index: NodeIndex(None), - range: 44..51, + range: 234..241, value: Compare( ExprCompare { node_index: NodeIndex(None), - range: 44..51, + range: 234..241, left: Name( ExprName { node_index: NodeIndex(None), - range: 44..45, + range: 234..235, id: Name("x"), ctx: Load, }, @@ -33,12 +33,19 @@ Module( Name( ExprName { node_index: NodeIndex(None), - range: 50..51, + range: 240..241, id: Name("y"), ctx: Load, }, ), ], + identity_ops: Some( + IdentityOperators { + ops: [ + true, + ], + }, + ), }, ), }, @@ -46,15 +53,15 @@ Module( Expr( StmtExpr { node_index: NodeIndex(None), - range: 52..59, + range: 242..249, value: Compare( ExprCompare { node_index: NodeIndex(None), - range: 52..59, + range: 242..249, left: Name( ExprName { node_index: NodeIndex(None), - range: 52..53, + range: 242..243, id: Name("x"), ctx: Load, }, @@ -66,12 +73,19 @@ Module( Name( ExprName { node_index: NodeIndex(None), - range: 58..59, + range: 248..249, id: Name("y"), ctx: Load, }, ), ], + identity_ops: Some( + IdentityOperators { + ops: [ + true, + ], + }, + ), }, ), }, @@ -79,15 +93,15 @@ Module( Expr( StmtExpr { node_index: NodeIndex(None), - range: 94..110, + range: 388..401, value: Compare( ExprCompare { node_index: NodeIndex(None), - range: 94..110, + range: 388..401, left: Name( ExprName { node_index: NodeIndex(None), - range: 94..95, + range: 388..389, id: Name("a"), ctx: Load, }, @@ -100,7 +114,7 @@ Module( Name( ExprName { node_index: NodeIndex(None), - range: 100..101, + range: 394..395, id: Name("b"), ctx: Load, }, @@ -108,12 +122,20 @@ Module( Name( ExprName { node_index: NodeIndex(None), - range: 109..110, + range: 400..401, id: Name("c"), ctx: Load, }, ), ], + identity_ops: Some( + IdentityOperators { + ops: [ + true, + true, + ], + }, + ), }, ), }, @@ -121,15 +143,15 @@ Module( Expr( StmtExpr { node_index: NodeIndex(None), - range: 111..118, + range: 402..409, value: Compare( ExprCompare { node_index: NodeIndex(None), - range: 111..118, + range: 402..409, left: Name( ExprName { node_index: NodeIndex(None), - range: 111..112, + range: 402..403, id: Name("a"), ctx: Load, }, @@ -141,12 +163,19 @@ Module( Name( ExprName { node_index: NodeIndex(None), - range: 117..118, + range: 408..409, id: Name("b"), ctx: Load, }, ), ], + identity_ops: Some( + IdentityOperators { + ops: [ + true, + ], + }, + ), }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__if.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__if.py.snap index c321bfbe65..0c796aca94 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__if.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__if.py.snap @@ -186,6 +186,7 @@ Module( }, ), ], + identity_ops: None, }, ), body: BinOp( @@ -325,6 +326,7 @@ Module( }, ), ], + identity_ops: None, }, ), orelse: Name( diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__list.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__list.py.snap index f39ea8650c..7f4b3f51ae 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__list.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__list.py.snap @@ -964,6 +964,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__list_comprehension.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__list_comprehension.py.snap index b5e3275e91..02ee04f199 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__list_comprehension.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__list_comprehension.py.snap @@ -228,6 +228,7 @@ Module( }, ), ], + identity_ops: None, }, ), BoolOp( @@ -381,6 +382,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], @@ -501,6 +503,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], @@ -565,6 +568,7 @@ Module( }, ), ], + identity_ops: None, }, ), ifs: [], @@ -781,6 +785,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__set_comprehension.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__set_comprehension.py.snap index c6ba397fb6..7d0e3fa7bc 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__set_comprehension.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__set_comprehension.py.snap @@ -116,6 +116,7 @@ Module( }, ), ], + identity_ops: None, }, ), BoolOp( @@ -269,6 +270,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], @@ -389,6 +391,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__t_string.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__t_string.py.snap index dc3f7f79a3..78f3e21ac2 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__t_string.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__t_string.py.snap @@ -353,6 +353,7 @@ Module( }, ), ], + identity_ops: None, }, ), debug_text: None, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__yield.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__yield.py.snap index 7613e48055..5c7a19142a 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__yield.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__yield.py.snap @@ -404,6 +404,7 @@ Module( }, ), ], + identity_ops: None, }, ), ), diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__yield_from.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__yield_from.py.snap index 4eb78e54a3..fc81e1c3da 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__yield_from.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__yield_from.py.snap @@ -330,6 +330,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@for_in_target_valid_expr.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@for_in_target_valid_expr.py.snap index 788be1a067..a4a71d6e70 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@for_in_target_valid_expr.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@for_in_target_valid_expr.py.snap @@ -52,6 +52,7 @@ Module( }, ), ], + identity_ops: None, }, ), ctx: Store, @@ -119,6 +120,7 @@ Module( }, ), ], + identity_ops: None, }, ), slice: NumberLiteral( @@ -195,6 +197,7 @@ Module( }, ), ], + identity_ops: None, }, ), attr: Identifier { diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_classify_as_identifier_1.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_classify_as_identifier_1.py.snap index 7eb5caad29..1be7df9bcd 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_classify_as_identifier_1.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_classify_as_identifier_1.py.snap @@ -39,6 +39,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_classify_as_identifier_2.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_classify_as_identifier_2.py.snap index 616aeb61bb..3d0d7d6abf 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_classify_as_identifier_2.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_classify_as_identifier_2.py.snap @@ -53,6 +53,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -367,6 +368,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__ambiguous_lpar_with_items.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__ambiguous_lpar_with_items.py.snap index fc65621d8a..1a3feff163 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__ambiguous_lpar_with_items.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__ambiguous_lpar_with_items.py.snap @@ -625,6 +625,7 @@ Module( }, ), ], + identity_ops: None, }, ), optional_vars: None, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__assert.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__assert.py.snap index 23d81b8c5a..a0ad80056d 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__assert.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__assert.py.snap @@ -41,6 +41,7 @@ Module( }, ), ], + identity_ops: None, }, ), msg: None, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__for.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__for.py.snap index f57cf2cb45..2aac57fe17 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__for.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__for.py.snap @@ -272,6 +272,7 @@ Module( }, ), ], + identity_ops: None, }, ), body: [ diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__if.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__if.py.snap index 7adfc6f3f9..1cf47f0ccd 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__if.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__if.py.snap @@ -176,6 +176,7 @@ Module( }, ), ], + identity_ops: None, }, ), body: [ diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__match.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__match.py.snap index b3c84a2be2..1f38fe8c7d 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__match.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__match.py.snap @@ -3366,6 +3366,7 @@ Module( }, ), ], + identity_ops: None, }, ), ), @@ -8077,6 +8078,7 @@ Module( }, ), ], + identity_ops: None, }, ), ), @@ -9050,6 +9052,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__raise.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__raise.py.snap index ee2161cd6d..f4fafe203f 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__raise.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__raise.py.snap @@ -106,6 +106,7 @@ Module( }, ), ], + identity_ops: None, }, ), ), @@ -378,6 +379,7 @@ Module( }, ), ], + identity_ops: None, }, ), ), diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__return.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__return.py.snap index cf64239bc1..b0af82ddd0 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__return.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__return.py.snap @@ -195,6 +195,7 @@ Module( }, ), ], + identity_ops: None, }, ), ), diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__type.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__type.py.snap index 985fe73d59..a2a5b67daf 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__type.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__type.py.snap @@ -3201,6 +3201,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -3345,6 +3346,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__while.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__while.py.snap index fffc87aa97..577ba2b3bc 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__while.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__while.py.snap @@ -75,6 +75,7 @@ Module( }, ), ], + identity_ops: None, }, ), Name( diff --git a/crates/ty_ide/src/inlay_hints.rs b/crates/ty_ide/src/inlay_hints.rs index 33ffc07e98..f0265da138 100644 --- a/crates/ty_ide/src/inlay_hints.rs +++ b/crates/ty_ide/src/inlay_hints.rs @@ -1337,18 +1337,16 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// basedpython: hint `reified` on each type parameter of `function` that a /// value-position use in the body reifies without saying so. fn add_inferred_reification(&mut self, function: &ast::StmtFunctionDef) { - let inferred = |hints: &mut Self| { - inferred_reified_type_param_names(hints.source, hints.source_type, function) - }; + let inferred = + |hints: &mut Self| inferred_reified_type_param_names(hints.source_type, function); self.add_inferred_reification_of(function.type_params.as_deref(), inferred); } /// basedpython: the same hint on a class, which reifies a type parameter its /// methods read through their receiver. fn add_inferred_class_reification(&mut self, class: &ast::StmtClassDef) { - let inferred = |hints: &mut Self| { - inferred_reified_class_type_param_names(hints.source, hints.source_type, class) - }; + let inferred = + |hints: &mut Self| inferred_reified_class_type_param_names(hints.source_type, class); self.add_inferred_reification_of(class.type_params.as_deref(), inferred); } diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_enums.md b/crates/ty_python_semantic/resources/mdtest/basedpython_enums.md index ae3affd265..712801c996 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_enums.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_enums.md @@ -596,34 +596,33 @@ c: Color.RED = Color.RED reveal_type(c) # revealed: Color.RED ``` -## `is` / `is not` between members keeps identity at runtime +## `is` / `is not` against a member is identity at runtime -a payload-less variant is a singleton *instance*, not a class, so the `is`/`is not` keyword pair -keeps python identity semantics for it — the `isinstance` lowering only fires when the rhs resolves -to a variant *class*. this block is checker-clean, so the divergence harness executes it and pins -the runtime contract +a payload-less variant names the type holding exactly one object, so the test for it is identity — +and a payload-less variant is a singleton *instance*, so identity is also all the runtime has to +compare. a payload variant names a *class* instead, and its test is the `isinstance` that asks. this +block is checker-clean, so the divergence harness executes it and pins the runtime contract + +the values come from a call rather than being written inline, so the tests are not settled +statically — a settled one would fold to a constant and stop exercising the lowering ```by enum class Genre: case A, B -assert Genre.A is Genre.A -assert Genre.A is not Genre.B +def pick() -> Genre: + return Genre.A -g: Genre = Genre.A -assert g is Genre.A -assert g is not Genre.B +assert pick() is Genre.A +assert pick() is not Genre.B enum class Shape: case Circle(radius: float) case Point -assert Shape.Point is Shape.Point -p = Shape.Point -assert p is Shape.Point +def shape() -> Shape: + return Shape.Circle(1.0) -# a payload variant is a class, so the rhs of `is` lowers to `isinstance` -c = Shape.Circle(1.0) -assert c is Shape.Circle -assert c is not Shape.Point +assert shape() is Shape.Circle +assert shape() is not Shape.Point ``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_escaping_local.md b/crates/ty_python_semantic/resources/mdtest/basedpython_escaping_local.md index 2a41101fa4..162812015c 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_escaping_local.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_escaping_local.md @@ -262,7 +262,7 @@ def f(fn: (local Resource) -> None): f: tmp = it - print(tmp is None) + print(tmp === None) ``` ## a `once` block's fresh binding still escapes diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_identity_narrow.md b/crates/ty_python_semantic/resources/mdtest/basedpython_identity_narrow.md index cb286776b7..b14f1a8a1f 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_identity_narrow.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_identity_narrow.md @@ -1,23 +1,23 @@ -# basedpython: `is` / `is not` keyword narrowing +# basedpython: `is` / `is not` narrowing -In basedpython, the `is` and `is not` keyword pair perform instance checks (they transpile to -`isinstance(...)` / `not isinstance(...)`). The `===` and `!==` operators retain Python's identity -comparison semantics. Narrowing in `.by` files mirrors this swap. +In basedpython the `is` and `is not` keyword pair is a *type test*: its right-hand side is a type +expression, and the test asks whether the value has that type. The `===` and `!==` operators keep +Python's identity comparison. Narrowing mirrors that split. -## `is not` narrows to negation of the instance type +## `is` narrows to the type named ```by def f(a: object): - if a is not int: - reveal_type(a) # revealed: not int + if a is int: + reveal_type(a) # revealed: int ``` -## `is` narrows to the instance type +## `is not` narrows to the negation of it ```by def f(a: object): - if a is int: - reveal_type(a) # revealed: int + if a is not int: + reveal_type(a) # revealed: not int ``` ## `!==` keeps Python identity semantics @@ -36,11 +36,11 @@ def f(a: int | None): reveal_type(a) # revealed: None ``` -## `is` with literal RHS keeps Python identity semantics +## A literal names the type holding exactly that value -`isinstance(x, None)` is invalid at runtime, so `is`/`is not` against literal singletons (`None`, -`True`/`False`, numbers, strings, bytes, `...`) must transpile as Python `is`/`is not` rather than -`isinstance`. +`None`, `True`/`False` and a number are all type expressions, so a test against one narrows to the +literal type it names. The runtime check that comes out is the equality — or, for `None`, the +identity — that decides membership of that type. ```by def f(a: int | None): @@ -59,16 +59,17 @@ def f(a: bool | int): ``` ```by -def f(a: int | None): - if a is ...: - reveal_type(a) # revealed: Never +def f(a: int | str): + if a is 1: + reveal_type(a) # revealed: 1 + if a is "x": + reveal_type(a) # revealed: "x" ``` -## `is` with an enum member RHS keeps Python identity semantics +## An enum member names the type holding exactly that member -An enum member is a singleton *instance*, not a class — `isinstance(x, Color.RED)` would be a -runtime `TypeError` — so `is`/`is not` against a member keeps Python identity semantics and narrows -by identity, the same as literal singletons. +An enum member is a singleton, so `Literal[Color.RED]` holds one object and the test for it is +identity — which is also what the runtime compares. ```by import enum @@ -97,11 +98,11 @@ def g(x: Genre): reveal_type(x) # revealed: Literal[Genre.B] ``` -## An instance check yields `bool`, never an identity fold +## An undecidable test is `bool` -The keyword form is an instance check, so Python's identity folds (an instance is never identical to -a class object, so plain Python would type `x is int` as `Literal[False]`) must not apply — -otherwise everything after `assert x is int` would be unreachable. +The identity folds Python applies to the same operator have no place here: the right-hand side names +a type rather than the class object the same source spells as a value, so `x is int` is not "an +instance compared to a class" and must not collapse to `Literal[False]`. ```by def f(x: object): @@ -111,10 +112,21 @@ def f(x: object): reveal_type(x) # revealed: int ``` +## A test the types settle is its answer + +Where the value's type decides the question, the test *is* that answer — which is what lets a reader +see that the branch it guards is already decided. + +```by +def f(x: int): + reveal_type(x is int) # revealed: True + reveal_type(x is not int) # revealed: False +``` + ## A test against a disjoint type is reported -An instance check whose value can never have the tested type is a constant: `is` never holds and -`is not` always does. Either the guarded branch is dead or the wrong type was named. +A test whose value can never have the type named is a constant: `is` never holds and `is not` always +does. Either the guarded branch is dead or the wrong type was named. ```by def f(x: None): @@ -189,11 +201,30 @@ def g(x: int): ## Identity comparisons are left alone -The `===` operators and the literal/enum-member forms keep Python identity semantics, where an -always-`False` comparison is already typed `Literal[False]`. +`===` and `!==` keep Python identity semantics, where an always-`False` comparison is already typed +`Literal[False]`. ```by def f(x: None): b = x === 1 reveal_type(b) # revealed: False ``` + +## A chained type test is rejected + +Python chains `a is int is str` into `a is int and int is str`, whose second half asks whether the +*class* `int` has the type `str`. That is never what the writer meant, so the chain is refused +rather than given a meaning. + +```by +def f(a: object): + # error: [invalid-syntax] "`is` type test cannot be chained with another comparison; split it into separate tests joined with `and`" + b = a is int is str +``` + +A chain of identity comparisons is ordinary Python and stays legal. + +```by +def f(a: object, b: object): + c = a === b !== None +``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_inferred_narrowing.md b/crates/ty_python_semantic/resources/mdtest/basedpython_inferred_narrowing.md index 0d4f6d2b4d..40ea39566f 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_inferred_narrowing.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_inferred_narrowing.md @@ -384,8 +384,8 @@ def f(x: str): ## a parameter the body puts something else in is not a guard A guard names the argument a call passed, so it says nothing once the body puts something else where -that argument was. `rebound` hands back `True` whatever it is given, and reading that as a claim -about the argument would narrow it to `Never`. +that argument was. `rebound` hands back `True` whatever it is given — the test is settled by the `1` +the body assigned — and reading that as a claim about the argument would narrow it to `Never`. ```by def rebound(a: object): @@ -393,7 +393,7 @@ def rebound(a: object): return a is int def f(x: str): - reveal_type(rebound) # revealed: def rebound(a: object) -> bool + reveal_type(rebound) # revealed: def rebound(a: object) -> True if rebound(x): reveal_type(x) # revealed: str ``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_parametric_is.md b/crates/ty_python_semantic/resources/mdtest/basedpython_parametric_is.md index ed0ad3d5b9..a39a7731cc 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_parametric_is.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_parametric_is.md @@ -384,7 +384,7 @@ class C: a: bool def f(c: C) -> bool: - reveal_type(c is HasA[bool]) # revealed: bool + reveal_type(c is HasA[bool]) # revealed: True return c is HasA[bool] ``` @@ -840,3 +840,205 @@ def outer(data: list[int] | list[str]) -> str: print(outer(list[int]())) # ints print(outer(list[str]())) # strs ``` + +## the target is a type expression + +The right-hand side of a type test is a type expression, so it names its target the way an +annotation does — and an unusable target is reported the same way too. + +```by +import os + +def fn() -> None: ... + +def f(v: object) -> None: + # error: [invalid-type-form] "Module `os` is not valid in a type expression" + print(v is os) + # error: [invalid-type-form] "Function `fn` is not valid in a type expression" + print(v is fn) +``` + +A bare generic class is complete without arguments: the test asks whether the value is one, and the +arguments are what the runtime cannot see either way. + +```by +def f(v: object) -> None: + print(v is list) + print(v is dict) +``` + +A tuple names the tuple *type*, not the "any of these" that `isinstance` spells with one. Write a +union for that. + +```by +def f(v: int) -> None: + # error: [non-overlapping-type-test] "`int` and `(int, str)` are non-overlapping types, so this test is always `False`" + print(v is (int, str)) + print(v is int | str) +``` + +## a target with no runtime form is rejected + +A test must *earn* its `True` — it narrows — so a target the runtime can only partly check is +rejected rather than approximated. `Any` admits every value, and a test that can only answer `True` +narrows nothing. + +```by +from typing import Any + +def f(v: object) -> None: + # error: [erased-type-check] "`is Any` cannot be checked at runtime: `Any` admits every value, so there is nothing for the test to look for" + print(v is Any) +``` + +A callable type's parameter and return types are not recorded on the value, so a runtime check would +assume the signature rather than test it. A bare `Callable` asks only what the runtime does record, +and is the test `callable()` performs. + +```by +from typing import Callable + +def f(v: object) -> None: + # error: [erased-type-check] "`is Callable[[], int]` cannot be checked at runtime: a callable's parameter and return types are not recorded on the value" + print(v is Callable[[], int]) + print(v is Callable) +``` + +A `float` or `complex` literal names a type equality cannot decide — `0.0 == -0.0` and `nan != nan` +— and `LiteralString` is a property of how a value was written rather than of the value. + +```by +from typing import LiteralString + +def f(v: object) -> None: + # error: [erased-type-check] "`is 1.5` cannot be checked at runtime: equality does not decide membership of `1.5`" + print(v is 1.5) + # error: [erased-type-check] "`is LiteralString` cannot be checked at runtime: equality does not decide membership of `LiteralString`" + print(v is LiteralString) +``` + +An intersection has no single runtime form, and `issubclass` cannot be asked what a `type[Any]` is +parameterized by. + +```by +from typing import Any + +class A: ... + +class B: ... + +def f(v: object) -> None: + # error: [erased-type-check] "`is A & B` cannot be checked at runtime: an intersection has no single runtime form" + print(v is A & B) + # error: [erased-type-check] "`is type[Any]` cannot be checked at runtime: `issubclass` cannot take what `type[Any]` is parameterized by" + print(v is type[Any]) +``` + +A `TypedDict`'s instances are plain dicts, so the test could only ask whether the value is a `dict` +and assume every key. + +```by +from typing import TypedDict + +class Movie(TypedDict): + title: str + +def f(v: dict[str, str]) -> None: + # error: [erased-type-check] "`is Movie` cannot be checked at runtime: a `TypedDict`'s instances are plain dicts" + print(v is Movie) +``` + +A protocol whose members all have a runtime spelling is checked against the value's own reified +annotations, member by member — `isinstance` could not, since a conforming class need not be a +subclass. One with a member the emitted python cannot name leaves nothing to check against. + +```by +from typing import Callable, Protocol + +class Handler(Protocol): + on_event: Callable[[], int] + +def f(v: object) -> None: + # error: [erased-type-check] "`is Handler` cannot be checked at runtime: `Handler` has a member with no runtime spelling" + print(v is Handler) +``` + +## a `@runtime_checkable` protocol target is an ordinary instance test + +Decorating the protocol is the author's own statement that `isinstance` may take it, so the test is +the check python itself performs — that the members are present. + +```by +from typing import Protocol, runtime_checkable + +@runtime_checkable +class HasName(Protocol): + name: str + +def f(v: object) -> None: + print(v is HasName) +``` + +## a literal target tests the value, not its class + +A literal names the type holding exactly the values equal to it, so the runtime check is that +equality — with the class pinned, because python's `1 == True` would otherwise let a `bool` satisfy +`Literal[1]`. + +```by +from typing import Literal + +def f(v: object) -> None: + print(v is Literal[1]) + print(v is Literal["a", "b"]) +``` + +An alias to a union of literals is the same test, arm by arm — the source spells the union with one +word, and `isinstance` could take neither that word nor the literals it stands for. + +```by +from typing import Literal + +type Small = Literal[1, 2] + +def f(v: object) -> None: + print(v is Small) +``` + +## a `type[C]` target tests the class + +`type[C]` is one of the few parameterized types the runtime can check in full: the value must be a +class, and a subclass of `C`. + +```by +class A: ... + +def f(v: object) -> None: + print(v is type[A]) +``` + +## a template literal type tests the string + +A template literal type is the set of strings its pattern produces, and a value is one of them or it +is not — so a test against it is decided by the pattern. + +```by +def f(s: str) -> None: + print(s is f"item-{int}") + +reveal_type("item-12" is f"item-{int}") # revealed: True +# error: [non-overlapping-type-test] "`"item-ab"` and `f"item-{int}"` are non-overlapping types, so this test is always `False`" +reveal_type("item-ab" is f"item-{int}") # revealed: False +``` + +## a union target the source spells with one word + +A PEP 695 alias names a whole union with a single identifier, which `isinstance` cannot take — the +test is the disjunction of the arms all the same. + +```by +type Key = int | str + +def f(v: object) -> None: + print(v is Key) +``` diff --git a/crates/ty_python_semantic/src/lib.rs b/crates/ty_python_semantic/src/lib.rs index 6a5f6c2f2f..d012b52aeb 100644 --- a/crates/ty_python_semantic/src/lib.rs +++ b/crates/ty_python_semantic/src/lib.rs @@ -46,9 +46,7 @@ pub use ty_site_packages::{ SitePackagesDiscoveryError, SitePackagesPaths, SysPrefixPathOrigin, }; pub use types::conformance::declares_conformances; -pub use types::conformance::{ - ConformanceRegistration, ConformanceTest, WitnessDispatch, WitnessKind, -}; +pub use types::conformance::{ConformanceRegistration, WitnessDispatch, WitnessKind}; pub use types::conversions::{ ConversionImport, ConversionInfo, ConversionRuntime, DISCARD_ADAPTER, }; @@ -64,6 +62,7 @@ pub use types::ide_support::{ pub use types::implicit_names::implicit_names; pub use types::reified_infer::{ ArgVariance, ErasedTargetReason, ErasedUnion, ParametricIsPlan, ProtocolMemberCheck, + TargetSpelling, }; pub use types::static_resource::{ResourceError, render_as, resolve_static_resource}; pub use types::template::finite_string_set; diff --git a/crates/ty_python_semantic/src/reified.rs b/crates/ty_python_semantic/src/reified.rs index 0af235c034..00ec3545c0 100644 --- a/crates/ty_python_semantic/src/reified.rs +++ b/crates/ty_python_semantic/src/reified.rs @@ -17,7 +17,7 @@ use ruff_python_ast::name::Name; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{self as ast, CmpOp, Expr, PySourceType, Stmt}; +use ruff_python_ast::{self as ast, Expr, PySourceType, Stmt}; use ruff_text_size::{Ranged, TextRange}; use rustc_hash::{FxHashMap, FxHashSet}; @@ -30,7 +30,6 @@ use rustc_hash::{FxHashMap, FxHashSet}; /// `ParamSpec`, a parameter list with no runtime object to bind — so /// `source_type` decides whether it is a candidate pub fn reified_type_param_names( - source: &str, source_type: PySourceType, function: &ast::StmtFunctionDef, ) -> Vec { @@ -68,7 +67,7 @@ pub fn reified_type_param_names( shadow_bound_names(&function.body, &mut active); let param_typevars = param_annotation_typevars(&function.parameters, &active); let mut finder = ValueUseFinder { - source, + source_type, active, param_typevars, found: Vec::new(), @@ -147,11 +146,10 @@ pub enum UnansweredReason { /// a `**Kwargs` keyword pack is never a candidate. a class writes its /// specialization as a subscript, and a subscript takes no keyword arguments, /// so there is no way to supply one -pub fn reified_class_reads<'ast>( - source: &str, +pub fn reified_class_reads( source_type: PySourceType, - class: &'ast ast::StmtClassDef, -) -> ReifiedClassReads<'ast> { + class: &ast::StmtClassDef, +) -> ReifiedClassReads<'_> { let Some(type_params) = class.type_params.as_deref() else { return ReifiedClassReads::default(); }; @@ -199,7 +197,7 @@ pub fn reified_class_reads<'ast>( // wherever the body writes the `def` — guarded by a version check, say — // while a read in the class body itself, in a method's header, or inside a // class nested in the class body has no method around it - let mut finder = ValueUseFinder::new(source, active); + let mut finder = ValueUseFinder::new(source_type, active); for stmt in &class.body { finder.visit_stmt(stmt); } @@ -347,18 +345,16 @@ fn body_span(function: &ast::StmtFunctionDef) -> Option { /// names of the class's type parameters that are reified, in declaration order pub(crate) fn reified_class_type_param_names( - source: &str, source_type: PySourceType, class: &ast::StmtClassDef, ) -> Vec { - reified_class_reads(source, source_type, class).names + reified_class_reads(source_type, class).names } /// names of the class's type parameters that are reified *only* because the /// class reads them in a value position, in declaration order. this is what an /// editor hints, where the keyword would be written pub fn inferred_reified_class_type_param_names( - source: &str, source_type: PySourceType, class: &ast::StmtClassDef, ) -> Vec { @@ -370,7 +366,7 @@ pub fn inferred_reified_class_type_param_names( .filter(|param| param.is_reified()) .map(|param| param.name().id.as_str()) .collect(); - let mut names = reified_class_type_param_names(source, source_type, class); + let mut names = reified_class_type_param_names(source_type, class); names.retain(|name| !declared.contains(name.as_str())); names } @@ -419,7 +415,6 @@ fn own_bindings(function: &ast::StmtFunctionDef) -> Vec<&str> { /// [`reified_type_param_names`] finds that does not already say so itself. /// this is what an editor hints, where the keyword would be written pub fn inferred_reified_type_param_names( - source: &str, source_type: PySourceType, function: &ast::StmtFunctionDef, ) -> Vec { @@ -431,24 +426,11 @@ pub fn inferred_reified_type_param_names( .filter(|param| param.is_reified()) .map(|param| param.name().id.as_str()) .collect(); - let mut names = reified_type_param_names(source, source_type, function); + let mut names = reified_type_param_names(source_type, function); names.retain(|name| !declared.contains(name.as_str())); names } -/// whether the `is` / `is not` between two compare operands is the keyword -/// form (isinstance semantics) rather than the `===` / `!==` identity -/// operators, which the parser flattens to the same ast -pub fn is_keyword_comparison(source: &str, op: CmpOp, lhs: &Expr, rhs: &Expr) -> bool { - let between = &source[usize::from(lhs.range().end())..usize::from(rhs.range().start())]; - let trimmed = between.trim(); - match op { - CmpOp::Is => trimmed == "is", - CmpOp::IsNot => !trimmed.starts_with("!=="), - _ => false, - } -} - /// parameter name → the still-active type-param names its annotation /// mentions. a parametric `is` test on such a parameter lowers to an equality /// check of those params' reified cells, so the test is a value-position use @@ -557,7 +539,7 @@ impl<'a> Visitor<'a> for StoredNames<'a> { } struct ValueUseFinder<'a> { - source: &'a str, + source_type: PySourceType, active: FxHashSet<&'a str>, /// parameters of the innermost enclosing def whose annotations mention /// active type params — parametric `is` tests on them reify those params @@ -570,9 +552,9 @@ struct ValueUseFinder<'a> { impl<'a> ValueUseFinder<'a> { /// a finder for a region that binds no parameters of its own, so no /// annotation can carry a parametric type test into it - fn new(source: &'a str, active: FxHashSet<&'a str>) -> Self { + fn new(source_type: PySourceType, active: FxHashSet<&'a str>) -> Self { Self { - source, + source_type, active, param_typevars: FxHashMap::default(), found: Vec::new(), @@ -602,11 +584,8 @@ impl<'a> ValueUseFinder<'a> { /// reified cell against the target's type arguments fn check_parametric_tests(&mut self, compare: &'a ast::ExprCompare) { let mut lhs: &Expr = &compare.left; - for (op, rhs) in compare.ops.iter().zip(&compare.comparators) { - if matches!(op, CmpOp::Is | CmpOp::IsNot) - && matches!(rhs, Expr::Subscript(_)) - && is_keyword_comparison(self.source, *op, lhs, rhs) - { + for (index, rhs) in compare.comparators.iter().enumerate() { + if matches!(rhs, Expr::Subscript(_)) && compare.is_type_test(index, self.source_type) { self.reify_tested_param(lhs); } lhs = rhs; diff --git a/crates/ty_python_semantic/src/semantic_model.rs b/crates/ty_python_semantic/src/semantic_model.rs index 07c8e61d07..814507a729 100644 --- a/crates/ty_python_semantic/src/semantic_model.rs +++ b/crates/ty_python_semantic/src/semantic_model.rs @@ -348,35 +348,6 @@ impl<'db> SemanticModel<'db> { }) } - /// basedpython: how `x is ` is answered once conformances are in - /// play. `None` when the target is not an interface anything conforms to - /// here, where the ordinary `isinstance` lowering is still right - pub fn conformance_test( - &self, - target: &ast::Expr, - ) -> Option { - use crate::types::conformance; - - let db = self.db; - if !self.file.file(db).source_type(db).is_basedpython() { - return None; - } - let interface = target.inferred_type(self)?.to_class_type(db)?; - if !conformance::visible_conformances(db, self.file.file(db)) - .iter() - .any(|(_, declared)| declared.class_literal(db) == interface.class_literal(db)) - { - return None; - } - let members = Some( - conformance::interface_requirements(db, interface) - .iter() - .map(ToString::to_string) - .collect(), - ); - Some(conformance::ConformanceTest { members }) - } - /// basedpython: when an attribute access resolves to an `extension` /// member (this module's, or one from a module imported with a plain /// `import mod`), the backing-function rewrite the transpiler applies. @@ -1003,57 +974,72 @@ impl<'db> SemanticModel<'db> { .is_some_and(|ty| crate::types::trailing_lambda::callee_callback_is_once(self.db, ty)) } - /// basedpython: how the parametric type test `lhs is rhs` (keyword form) - /// resolves, from the operands' inferred types. `rhs` may name the target - /// specialization directly (`list[int]`) or through an alias — an implicit - /// alias whose value is a specialization (`X = list[int]`) or a PEP 695 - /// `type` alias. `None` when `rhs` does not resolve to a specialization — - /// the test is then an ordinary isinstance lowering - pub fn parametric_is_plan( + /// basedpython: whether `target`, read as a *value*, denotes a plain value + /// rather than a class — an enum member, a literal, an instance of a + /// concrete non-type class. + /// + /// A type test's target is a type expression, so its inferred type is the + /// type it names and the value it evaluates to is not recorded. That is + /// almost always the same thing, and where it is not, the type reading is + /// the one that matters. The exception is a target the type reading cannot + /// make sense of at all: the enum lowering rewrites a unit variant into a + /// singleton instance before the transpiler's passes run, so `s is + /// Shape.Point` reaches them naming a type in the source and a value in + /// what is emitted — and the test for a value is identity. + /// + /// Resolved here rather than read from the inference store, which holds the + /// type reading. Only a name or a dotted name is resolved; anything else + /// answers `false`, which leaves the ordinary instance check in place. + pub fn denotes_plain_value(&self, target: &ast::Expr) -> bool { + let env = self.program_environment(); + self.value_of(target, &env) + .is_some_and(|ty| crate::types::basedpython_is_plain_value(self.db, &env, ty)) + } + + /// the type of the value `expr` evaluates to, for a name or a dotted name + fn value_of( &self, - lhs: &ast::Expr, - rhs: &ast::Expr, - ) -> Option { - let env = &self.program_environment(); - let alias = crate::types::reified_infer::parametric_is_target( - self.db, - env, - rhs.inferred_type(self)?, - )?; - let lhs_ty = lhs.inferred_type(self)?; - Some(crate::types::reified_infer::classify_parametric_is( - self.db, - env, - self.file(), - lhs_ty, - alias, - rhs, - )) + expr: &ast::Expr, + env: &crate::types::context::ProgramEnvironment<'db>, + ) -> Option> { + match expr { + ast::Expr::Name(name) => crate::place::global_symbol( + self.db, + self.db.program_file(self.file()), + name.id.as_str(), + ) + .place + .ignore_possibly_undefined(), + ast::Expr::Attribute(attribute) => { + let base = self.value_of(&attribute.value, env)?; + base.member(self.db, env, attribute.attr.id.as_str()) + .place + .ignore_possibly_undefined() + } + _ => None, + } } - /// basedpython: [`Self::parametric_is_plan`] for a checked cast - /// (`value cast T`). The same classification engine decides both — only the - /// target's inference position differs, since a cast's target is a *type* - /// expression while an `is`-rhs is a value expression. - pub fn parametric_cast_plan( + /// basedpython: how the type test `lhs is rhs` resolves, from the value's + /// inferred type and the type its right-hand side names. + /// + /// The right-hand side is a type expression, so `rhs` may name its target + /// however a type expression can: directly (`list[int]`), through an + /// implicit alias (`X = list[int]`), or through a PEP 695 `type` alias. + /// `None` only when inference has no type for one of the operands. + pub fn parametric_is_plan( &self, - value: &ast::Expr, - target: &ast::Expr, + lhs: &ast::Expr, + rhs: &ast::Expr, ) -> Option { let env = &self.program_environment(); - let alias = crate::types::reified_infer::parametric_cast_target( - self.db, - env, - target.inferred_type(self)?, - )?; - let value_ty = value.inferred_type(self)?; - Some(crate::types::reified_infer::classify_parametric_is( + Some(crate::types::reified_infer::type_test_plan( self.db, env, self.file(), - value_ty, - alias, - target, + lhs.inferred_type(self)?, + rhs.inferred_type(self)?, + Some(rhs), )) } diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 61865adb38..d7ebc881e9 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -94,7 +94,7 @@ use crate::types::diagnostic::{ report_bad_dunder_get_call, report_bad_import_call, }; pub use crate::types::display::{DisplaySettings, SourceSpelling, TypeDetail, TypeDisplayDetails}; -pub use crate::types::enums::basedpython_is_keeps_identity; +pub(crate) use crate::types::enums::basedpython_is_plain_value; pub(crate) use crate::types::enums::{EnumClassLiteral, EnumComplementType, enum_metadata}; pub(crate) use crate::types::equality::{ComparisonSoundnessPolicy, equality_truthiness}; use crate::types::function::{ @@ -700,7 +700,7 @@ pub enum TypingModule { impl TypingModule { /// Return the module for a `TypedDict` special form, including a union of the special forms /// exported by `typing` and `typing_extensions`. - pub(crate) fn from_typed_dict_type<'db>(db: &'db dyn Db, ty: Type<'db>) -> Option { + fn from_typed_dict_type<'db>(db: &'db dyn Db, ty: Type<'db>) -> Option { match ty { Type::SpecialForm(SpecialFormType::TypedDict(module)) => Some(module), Type::Union(union) => { @@ -727,7 +727,7 @@ impl TypingModule { } } - pub(crate) const fn from_type_alias_class(class: KnownClass) -> Option { + const fn from_type_alias_class(class: KnownClass) -> Option { match class { KnownClass::TypeAliasType => Some(Self::Typing), KnownClass::ExtensionsTypeAliasType => Some(Self::TypingExtensions), diff --git a/crates/ty_python_semantic/src/types/class/static_literal.rs b/crates/ty_python_semantic/src/types/class/static_literal.rs index 317db9c390..7f8b9841e5 100644 --- a/crates/ty_python_semantic/src/types/class/static_literal.rs +++ b/crates/ty_python_semantic/src/types/class/static_literal.rs @@ -664,13 +664,8 @@ impl<'db> StaticClassLiteral<'db> { return Box::default(); } let module = parsed_module(db, self.python_file(db)).load(db); - let source = ruff_db::source::source_text(db, file); - crate::reified::reified_class_type_param_names( - source.as_str(), - source_type, - self.node(db, &module), - ) - .into_boxed_slice() + crate::reified::reified_class_type_param_names(source_type, self.node(db, &module)) + .into_boxed_slice() } /// basedpython: whether this class or anything it inherits from reifies a diff --git a/crates/ty_python_semantic/src/types/conformance.rs b/crates/ty_python_semantic/src/types/conformance.rs index 9799d1e6ac..5e178d29d3 100644 --- a/crates/ty_python_semantic/src/types/conformance.rs +++ b/crates/ty_python_semantic/src/types/conformance.rs @@ -398,16 +398,6 @@ pub enum WitnessKind { Property, } -/// how `x is ` is answered at runtime once conformances are in play -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ConformanceTest { - /// the requirement names, for a protocol target: with no registered - /// conformance for the value's class, answering the test means checking that - /// the value carries them. `None` for an abstract class, where `isinstance` - /// is already the whole answer - pub members: Option>, -} - /// declaration-site validation for a conformance extension, run from the /// post-inference static-class checks alongside the ordinary extension checks pub(crate) fn validate_conformance_declaration<'db>( diff --git a/crates/ty_python_semantic/src/types/enums.rs b/crates/ty_python_semantic/src/types/enums.rs index bf273f4f35..1fdcbb2b90 100644 --- a/crates/ty_python_semantic/src/types/enums.rs +++ b/crates/ty_python_semantic/src/types/enums.rs @@ -1735,17 +1735,19 @@ pub(crate) fn is_enum_class<'db>(db: &'db dyn Db, ty: Type<'db>) -> bool { } } -/// shared checker/transpiler contract for the basedpython `is`/`is not` -/// keyword pair: whether a comparison whose rhs has type `ty` keeps python -/// identity semantics instead of lowering to `isinstance`. true when the rhs -/// is statically a plain *value* — an enum member (`Color.RED`, a based-enum -/// unit variant like `Shape.Point`), another literal, or an instance of a -/// concrete non-type class — which `isinstance` would reject as its classinfo -/// argument at runtime. a class-like rhs (a class literal, `type[...]`, an -/// instance of a metaclass), a tuple (a valid multi-target classinfo -/// spelling), a bare `object` (which admits classes), and anything dynamic or -/// unresolved lower to `isinstance` as usual -pub fn basedpython_is_keeps_identity<'db>( +/// basedpython: whether a value of type `ty` is a plain *value* rather than a +/// class — an enum member (`Color.RED`, a based-enum unit variant), another +/// literal, or an instance of a concrete non-type class. `isinstance` rejects +/// such a value as its classinfo argument at runtime, so the test for one is +/// identity. +/// +/// This answers a lowering question, not a checking one. A type test's target +/// is a type expression, and the checker reads it as one — but the enum +/// lowering rewrites a unit variant into a singleton instance before the +/// transpiler's own pass runs, so a target that names a type in the source can +/// name a value in what is emitted. That is the case this decides, from the +/// type rather than from the shape the target was written in. +pub(crate) fn basedpython_is_plain_value<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, ty: Type<'db>, @@ -1754,13 +1756,13 @@ pub fn basedpython_is_keeps_identity<'db>( Type::Union(union) => union .elements(db) .iter() - .all(|element| basedpython_is_keeps_identity(db, env, *element)), + .all(|element| basedpython_is_plain_value(db, env, *element)), // literal values (enum members included) are never classes Type::LiteralValue(_) | Type::EnumComplement(_) => true, // a use-site modifier says nothing about whether the value is a class: // a unit enum variant is a `final _Shape_Point`, still an instance Type::Restricted(restricted) => { - basedpython_is_keeps_identity(db, env, restricted.value_type(db)) + basedpython_is_plain_value(db, env, restricted.value_type(db)) } Type::NominalInstance(instance) => { !instance.has_known_class(db, KnownClass::Object) diff --git a/crates/ty_python_semantic/src/types/function.rs b/crates/ty_python_semantic/src/types/function.rs index 49bff5c7ba..3b80ade284 100644 --- a/crates/ty_python_semantic/src/types/function.rs +++ b/crates/ty_python_semantic/src/types/function.rs @@ -573,9 +573,7 @@ impl<'db> OverloadLiteral<'db> { // below belongs to whichever one the body scope was built for let module = parsed_module(db, self.python_file(db)).load(db); let node = self.body_scope(db).node(db).expect_function().node(&module); - let source = source_text(db, file); - crate::reified::reified_type_param_names(source.as_str(), source_type, node) - .into_boxed_slice() + crate::reified::reified_type_param_names(source_type, node).into_boxed_slice() } /// basedpython: reified type parameters that a call must supply a value diff --git a/crates/ty_python_semantic/src/types/infer.rs b/crates/ty_python_semantic/src/types/infer.rs index a6787ab202..7536bf20aa 100644 --- a/crates/ty_python_semantic/src/types/infer.rs +++ b/crates/ty_python_semantic/src/types/infer.rs @@ -2540,6 +2540,15 @@ bitflags::bitflags! { /// means something: it bounds the pack as a whole rather than field by field. const IN_PACK_BOUND = 1 << 18; + /// basedpython: set while inferring the right-hand side of a type test — the + /// type expression `x is T` tests against. + /// + /// A bare generic class is complete there: the test asks whether the value is + /// one, and the type arguments are exactly what the runtime cannot see. So + /// `missing-type-argument`, which asks an annotation to say more, has nothing + /// to ask for here. + const IN_TYPE_TEST_TARGET = 1 << 19; + /// Whether the current method's explicit receiver annotation is incompatible with `Self`. const HAS_INCOMPATIBLE_SELF_RECEIVER = 1 << 15; } diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index fd1ba583b3..e8f87f036a 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -162,16 +162,15 @@ use crate::types::visibility::{private_constructor, scope_is_within_class}; use crate::types::{ BindingContext, BoundTypeVarInstance, CallDunderError, CallableBinding, CallableType, CallableTypes, ClassType, DeferredOperation, DeferredType, DynamicType, GeneratorTypeMode, - InferenceFlags, InstanceProjection, InternedConstraintSet, InternedType, IntersectionBuilder, - IntersectionType, KnownBoundMethodType, KnownClass, KnownInstanceType, KnownUnion, - LiteralValueType, LiteralValueTypeKind, MemberLookupPolicy, ParamSpecAttrKind, Parameter, - Parameters, ProgramEnvironment, PropertyDeprecations, RestrictedType, SentinelInstance, - Signature, SpecialFormType, SubclassOfType, Type, TypeAliasType, TypeAndQualifiers, - TypeContext, TypeQualifiers, TypeVarBoundOrConstraints, TypeVarKind, TypeVarVariance, - TypedDictType, TypingModule, UnionAccumulator, UnionBuilder, UnionType, any_over_type, - binding_type, extract_fixed_length_iterable_element_types, infer_complete_scope_types, - infer_scope_types, is_discarded_dict_key_assignment, report_iteration_over_character, - todo_type, + InferenceFlags, InternedConstraintSet, InternedType, IntersectionBuilder, IntersectionType, + KnownBoundMethodType, KnownClass, KnownInstanceType, KnownUnion, LiteralValueType, + LiteralValueTypeKind, MemberLookupPolicy, ParamSpecAttrKind, Parameter, Parameters, + ProgramEnvironment, PropertyDeprecations, RestrictedType, SentinelInstance, Signature, + SpecialFormType, SubclassOfType, Type, TypeAliasType, TypeAndQualifiers, TypeContext, + TypeQualifiers, TypeVarBoundOrConstraints, TypeVarKind, TypeVarVariance, TypedDictType, + TypingModule, UnionAccumulator, UnionBuilder, UnionType, any_over_type, binding_type, + extract_fixed_length_iterable_element_types, infer_complete_scope_types, infer_scope_types, + is_discarded_dict_key_assignment, report_iteration_over_character, todo_type, }; use crate::{AnalysisSettings, Db, DisplaySettings, FxIndexSet, FxOrderSet, SemanticModel}; use fluid::FluidTimeline; @@ -11023,7 +11022,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { self.file(), value_ty, alias, - type_arg, + Some(type_arg), ), crate::types::reified_infer::ParametricIsPlan::TokenEq(_) | crate::types::reified_infer::ParametricIsPlan::Fold(_) @@ -15828,6 +15827,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let ast::ExprCompare { range: _, node_index: _, + identity_ops: _, left, ops, comparators, @@ -15847,6 +15847,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // A chain like `a == True == b` is two comparisons over one literal: reporting each pair // would double up on that `True`, and "test the operand" is not the fix for the chain. let single_comparison = ops.len() == 1; + // a chained type test is a parse error; nothing here re-reports it + let chained_type_test = !single_comparison && compare.has_type_test(self.source_type()); let ChainedBooleanResult { value_type: ty, preceding_truthiness, @@ -15856,32 +15858,45 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { std::iter::once(&**left) .chain(comparators) .tuple_windows::<(_, _)>() - .zip(ops), + .zip(ops.iter().enumerate()), |_| false, - |builder, ((left, right), op), _peer_ty| { + |builder, ((left, right), (index, op)), _peer_ty| { let left_ty = builder.expression_type(left); - let right_ty = builder.infer_expression(right, TypeContext::default()); - let range = TextRange::new(left.start(), right.end()); + // basedpython: the right-hand side of a type test is a *type* + // expression — `x is list[int]` names the type `list[int]`, not + // the class object the same source spells as a value. inferring + // it that way is what makes an annotation and a type test agree + // about which targets exist, and about which ones are rejected. + // + // a chain carrying one is rejected outright by the parser, and + // checking its pairs anyway would pile a second report on the + // same mistake + if !chained_type_test && compare.is_type_test(index, builder.source_type()) { + let previously_in_target = builder + .context + .inference_flags + .replace(InferenceFlags::IN_TYPE_TEST_TARGET, true); + let target = builder.infer_type_expression(right); + builder + .context + .inference_flags + .set(InferenceFlags::IN_TYPE_TEST_TARGET, previously_in_target); + return ( + builder.check_type_test(left, right, left_ty, target, *op), + range, + ); + } + + let right_ty = builder.infer_expression(right, TypeContext::default()); + if single_comparison { builder.check_redundant_boolean_comparison( left, right, left_ty, right_ty, *op, range, ); } - // a basedpython keyword-form `is`/`is not` whose rhs is a - // class (or a parametric test like `x is list[int]`) is an - // instance check, not python identity: it always yields a - // `bool`, and its reachability is decided by narrowing (not - // by the instance-vs-class-object disjointness that would - // otherwise type it `Literal[False]` and kill a live branch) - if let Some(ty) = - builder.check_basedpython_is_test(left, right, left_ty, right_ty, *op) - { - return (ty, range); - } - let ty = comparisons::infer_binary_type_comparison( &builder.context, left_ty, @@ -15951,71 +15966,85 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { ty } - /// basedpython: type a keyword-form `is`/`is not` pair that performs an - /// *instance check* rather than python identity, and check it. Returns - /// `Some(bool)` — the runtime result type — for any such pair, so the - /// identity folds (disjointness → `Literal[False]`) never apply to it; - /// reachability is decided by narrowing instead - fn check_basedpython_is_test( + /// basedpython: check a type test — `x is T` / `x is not T` — and give it + /// the `bool` its runtime check produces. + /// + /// `target` is the *type* the right-hand side names, already inferred as a + /// type expression, so an invalid target has been reported as + /// `invalid-type-form` before this runs. What is left to decide is whether + /// that type can be tested at runtime at all, and whether the test could + /// ever hold. Reachability is decided by narrowing rather than by the + /// identity folds, so this never returns anything but `bool`. + fn check_type_test( &mut self, left: &ast::Expr, right: &ast::Expr, left_ty: Type<'db>, - right_ty: Type<'db>, + target: Type<'db>, op: ast::CmpOp, - ) -> Option> { - let (bool_ty, decision) = - self.classify_basedpython_is_test(left, right, left_ty, right_ty, op)?; - self.report_non_overlapping_type_test(left, right, left_ty, right_ty, op, decision); - Some(bool_ty) - } - - /// The type a keyword-form `is`/`is not` asks its left operand to have: the - /// instance type of the class on the right, or the union of the arms' instance - /// types for a union target. `None` for a target with no instance form. - fn is_test_target_instance(&self, right: &ast::Expr, right_ty: Type<'db>) -> Option> { - let env = self.program_environment(); - // an over-approximating projection is the safe direction here: a wider - // target can only overlap more, so it never invents a disjointness + ) -> Type<'db> { let db = self.db(); - let Some(arms) = union_target_arms(right) else { - return right_ty - .to_instance(db, env) - .map(InstanceProjection::into_inner); - }; - let mut instances = Vec::with_capacity(arms.len()); - for arm in arms { - instances.push(self.expression_type(arm).to_instance(db, env)?.into_inner()); + let env = self.program_environment(); + let plan = crate::types::reified_infer::type_test_plan( + db, + env, + self.file(), + left_ty, + target, + Some(right), + ); + if let crate::types::reified_infer::ParametricIsPlan::ErasedTarget(reason) = plan { + let source = ruff_db::source::source_text(db, self.file()); + self.report_erased_type_check( + TextRange::new(left.start(), right.end()), + &source[right.range()], + reason, + ); + } else { + // a target with no runtime test has already been rejected, and + // saying it can also never hold would pile a second report on one + // mistake + self.report_non_overlapping_type_test( + left, + right, + left_ty, + target, + op, + plan.never_holds(), + ); + } + // a test the static types settle *is* its answer, and saying so is what + // lets a reader — and the editor's data-flow view — see that the branch + // it guards is decided. an undecidable test stays `bool`: the identity + // folds python would apply to the same operator have no place here, + // since the right-hand side names a type rather than the class object + // the same source spells as a value + match plan { + crate::types::reified_infer::ParametricIsPlan::Fold(holds) => { + Type::bool_literal(holds == (op == ast::CmpOp::Is)) + } + _ => KnownClass::Bool.to_instance(db, env), } - Some(UnionType::from_elements(db, env, instances)) } - /// Warn when a keyword-form `is`/`is not` tests a value against a type it can - /// never have. The test is then a constant — `is` never holds and `is not` - /// always does — so either the guarded branch is dead or the wrong type was - /// named. `Any`/`Unknown` overlap everything, so those never fire. + /// Warn when a type test asks whether a value has a type it can never have. + /// The test is then a constant — `is` never holds and `is not` always does — + /// so either the guarded branch is dead or the wrong type was named. + /// `Any`/`Unknown` overlap everything, so those never fire. fn report_non_overlapping_type_test( &self, left: &ast::Expr, right: &ast::Expr, left_ty: Type<'db>, - right_ty: Type<'db>, + target: Type<'db>, op: ast::CmpOp, - decision: IsTestDecision, + never_holds: bool, ) { - let env = self.program_environment(); - let db = self.db(); - let Some(target) = self.is_test_target_instance(right, right_ty) else { - return; - }; - let never_holds = match decision { - IsTestDecision::Instance => left_ty.is_disjoint_from(db, env, target), - IsTestDecision::ParametricNeverHolds => true, - IsTestDecision::Undecided => false, - }; if !never_holds { return; } + let db = self.db(); + let env = self.program_environment(); let range = TextRange::new(left.start(), right.end()); let Some(builder) = self.context.report_lint(&NON_OVERLAPPING_TYPE_TEST, range) else { return; @@ -16032,118 +16061,6 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { )); } - /// Decide whether this pair is an instance check, erroring when the pair is a - /// parametric test (`x is list[int]`) against a builtin collection whose - /// runtime instances erase their type arguments, so no runtime probe of the - /// value can ever confirm the specialization. `None` when the pair keeps - /// python identity semantics (`===` spelling, a literal or other plain-value - /// rhs such as an enum member — mirroring the transpiler's lowering), so the - /// caller keeps its usual comparison typing - fn classify_basedpython_is_test( - &mut self, - left: &ast::Expr, - right: &ast::Expr, - left_ty: Type<'db>, - right_ty: Type<'db>, - op: ast::CmpOp, - ) -> Option<(Type<'db>, IsTestDecision)> { - let env = self.program_environment(); - if !matches!(op, ast::CmpOp::Is | ast::CmpOp::IsNot) || !self.is_basedpython_file() { - return None; - } - // a literal rhs (`x is None`, `x is 0`) keeps python identity - // semantics; the transpiler leaves the operator untouched - if right.is_literal_expr() { - return None; - } - let source = ruff_db::source::source_text(self.db(), self.file()); - if !crate::reified::is_keyword_comparison(source.as_str(), op, left, right) { - return None; - } - let bool_ty = KnownClass::Bool.to_instance(self.db(), env); - - // a union target `a is T1 | T2` tests each arm (`type(a) <: Ti` for any - // arm). an erased arm can't be checked at runtime and, unlike a - // standalone erased target, may not fold to a constant inside the - // disjunction — that would be unsound — so it is rejected per arm - if let Some(arms) = union_target_arms(right) { - let mut decision = IsTestDecision::Instance; - for arm in arms { - let Some(alias) = crate::types::reified_infer::parametric_is_target( - self.db(), - env, - self.expression_type(arm), - ) else { - continue; - }; - // a parametric arm is decided by the engine rather than by - // disjointness, and the test holds as soon as *any* arm does, so - // one such arm puts the whole disjunction out of the lint's reach - decision = IsTestDecision::Undecided; - if let crate::types::reified_infer::ParametricIsPlan::ErasedTarget(reason) = - crate::types::reified_infer::classify_parametric_is( - self.db(), - env, - self.file(), - left_ty, - alias, - arm, - ) - { - self.report_erased_type_check(arm.range(), &source[arm.range()], reason); - } - } - return Some((bool_ty, decision)); - } - - // a plain-value rhs (an enum member, an instance of a non-type class) - // keeps python identity semantics — the transpiler leaves `is`/`is not` - // untouched, so ty types it as an ordinary identity comparison too - if crate::types::basedpython_is_keeps_identity(self.db(), env, right_ty) { - return None; - } - - let Some(alias) = - crate::types::reified_infer::parametric_is_target(self.db(), env, right_ty) - else { - // a bare class / dynamic rhs (`x is int`, `x is SomeClass`) is an - // instance check that lowers to `isinstance`, so it always yields a - // `bool` — the identity folds (disjointness → `Literal[False]`) - // must not apply - return Some((bool_ty, IsTestDecision::Instance)); - }; - let plan = crate::types::reified_infer::classify_parametric_is( - self.db(), - env, - self.file(), - left_ty, - alias, - right, - ); - // only a probe against a runtime-erased target is an error; every - // other plan (fold, reified-cell equality, witness, or a probe of a - // user generic that carries `__orig_class__`) is a valid test - if let crate::types::reified_infer::ParametricIsPlan::ErasedTarget(reason) = plan { - self.report_erased_type_check( - TextRange::new(left.start(), right.end()), - &source[right.range()], - reason, - ); - } - let decision = if plan == crate::types::reified_infer::ParametricIsPlan::Fold(false) { - IsTestDecision::ParametricNeverHolds - } else { - IsTestDecision::Undecided - }; - Some((bool_ty, decision)) - } - - /// report an `erased-type-check` for a parametric `is`-target (or one arm - /// of a union target) that has no runtime residue — either because the - /// target records no specialization to probe, or because the target cannot - /// be spelled at runtime at all. every other concrete class records its - /// specialization on the instance or across its mro, so the runtime probe - /// unwinds it instead fn report_erased_type_check( &self, primary: TextRange, @@ -16192,6 +16109,95 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { whose `__orig_bases__` the probe can unwind" )); } + ErasedTargetReason::Dynamic => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: `{target}` admits every value, \ + so there is nothing for the test to look for" + )); + diagnostic.info(format_args!( + "a test that can only answer `True` narrows nothing; drop it, or name the \ + type the value is expected to have" + )); + } + ErasedTargetReason::Callable => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: a callable's parameter and \ + return types are not recorded on the value" + )); + diagnostic.info(format_args!( + "the runtime can see that a value is callable and nothing more, so the \ + signature would be assumed rather than checked" + )); + diagnostic.info(format_args!( + "test against a bare `Callable`, which asks only what the runtime records, \ + or take the value's type from the call site" + )); + } + ErasedTargetReason::TypedDict => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: a `TypedDict`'s instances are \ + plain dicts" + )); + diagnostic.info(format_args!( + "the test could only ask whether the value is a `dict`, assuming every key \ + and value type `{target}` declares" + )); + diagnostic.info(format_args!( + "`isinstance` refuses a `TypedDict` for the same reason" + )); + } + ErasedTargetReason::NonRuntimeCheckableProtocol => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: `{target}` has a member with no \ + runtime spelling" + )); + diagnostic.info(format_args!( + "a member whose type the emitted python cannot name leaves nothing to check \ + the value's own annotation against" + )); + diagnostic.info(format_args!( + "decorate `{target}` with `@typing.runtime_checkable` for the check python \ + itself performs, which asks only that the members are present" + )); + } + ErasedTargetReason::Intersection => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: an intersection has no single \ + runtime form" + )); + diagnostic.info(format_args!( + "test each part separately and join the tests with `and`" + )); + } + ErasedTargetReason::Subclass => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: `issubclass` cannot take what \ + `{target}` is parameterized by" + )); + diagnostic.info(format_args!( + "the test can see that the value is a class and nothing more; name the class \ + it must be a subclass of" + )); + } + ErasedTargetReason::UncomparableLiteral => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: equality does not decide \ + membership of `{target}`" + )); + diagnostic.info(format_args!( + "`0.0 == -0.0` and `nan != nan`, so a `float` or `complex` literal names a \ + type the runtime cannot be asked about; `LiteralString` is a property of how \ + a value was written rather than of the value" + )); + } + ErasedTargetReason::Unspellable => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: `{target}` has no runtime form" + )); + diagnostic.info(format_args!( + "the emitted python has no expression to evaluate for this target" + )); + } } } @@ -17311,44 +17317,6 @@ fn is_collection_literal(expression: &ast::Expr) -> bool { ) } -/// the flat arms of a `|` union type expression (`A | B | C` → `[A, B, C]`), or -/// `None` when `expr` is not a union — used to test each arm of a parametric -/// `is`-target union independently -/// basedpython: how a keyword-form `is`/`is not` instance check is decided, which -/// is what the `non-overlapping-type-test` lint reads to know whether the test can -/// ever hold. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum IsTestDecision { - /// a bare class target (`x is int`): ordinary disjointness decides it - Instance, - /// a parametric target (`x is list[int]`) the parametric engine folded to - /// `False`. That engine is asked rather than disjointness directly because it - /// also honours a use-site variance projection (`a is A[out int]`) - ParametricNeverHolds, - /// a parametric target left to a runtime probe, or a union with such an arm — - /// nothing the lint can call constant - Undecided, -} - -fn union_target_arms(expr: &ast::Expr) -> Option> { - fn collect<'a>(expr: &'a ast::Expr, arms: &mut Vec<&'a ast::Expr>) { - if let ast::Expr::BinOp(binop) = expr - && binop.op == ast::Operator::BitOr - { - collect(&binop.left, arms); - collect(&binop.right, arms); - } else { - arms.push(expr); - } - } - if !matches!(expr, ast::Expr::BinOp(binop) if binop.op == ast::Operator::BitOr) { - return None; - } - let mut arms = Vec::new(); - collect(expr, &mut arms); - Some(arms) -} - /// Returns `true` if `expression` is a link of a basedpython optional chain: a `?.` access, or a /// trailer applied to one. /// diff --git a/crates/ty_python_semantic/src/types/infer/builder/class.rs b/crates/ty_python_semantic/src/types/infer/builder/class.rs index f038cc78ec..fb8e6c5c50 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/class.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/class.rs @@ -15,7 +15,6 @@ use crate::types::{ }, special_form::TypeQualifier, }; -use ruff_db::source::source_text; use ruff_python_ast::{self as ast, helpers::any_over_expr}; use ty_module_resolver::{ImportingFile, KnownModule, file_to_module}; use ty_python_core::{definition::Definition, scope::NodeWithScopeRef}; @@ -35,8 +34,7 @@ impl<'db> TypeInferenceBuilder<'db, '_> { if !source_type.is_basedpython() { return; } - let source = source_text(self.db(), self.file()); - let reads = reified_class_reads(source.as_str(), source_type, class); + let reads = reified_class_reads(source_type, class); // reification fixes the variance, so a declaration saying anything else // is a contradiction rather than a refinement — and the variance it diff --git a/crates/ty_python_semantic/src/types/infer/builder/function.rs b/crates/ty_python_semantic/src/types/infer/builder/function.rs index b5f73395da..08601f8f2c 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/function.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/function.rs @@ -1013,12 +1013,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { || is_implicit_classmethod(&name.id)) && let Some(type_params) = function.type_params.as_deref() { - let source = ruff_db::source::source_text(db, self.file()); - let reified = crate::reified::reified_type_param_names( - source.as_str(), - self.file().source_type(db), - function, - ); + let reified = + crate::reified::reified_type_param_names(self.file().source_type(db), function); if let Some(first) = reified.first() && let Some(builder) = self.context.report_lint(&REIFIED_CLASSMETHOD, type_params) { diff --git a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs index 7b03151fe6..ad82b4dcd6 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs @@ -208,7 +208,20 @@ impl<'db> TypeInferenceBuilder<'db, '_> { if self.is_basedpython_file() && ty.as_enum_literal().is_some() { return ty; } - report_missing_type_arguments(&self.context, ty, annotation); + // a type test's target is complete without type arguments: it asks + // whether the value is one of these, and the arguments are what the + // runtime cannot see either way. that holds for the target itself, not + // for a bare generic *inside* it — `x is A[list]` really does leave an + // argument the probe compares + let bare_target_is_complete = self + .inference_flags() + .contains(InferenceFlags::IN_TYPE_TEST_TARGET) + && !self + .inference_flags() + .contains(InferenceFlags::IN_NESTED_TYPE_EXPRESSION); + if !bare_target_is_complete { + report_missing_type_arguments(&self.context, ty, annotation); + } let result_ty = ty .default_specialize(db, env) .in_type_expression( diff --git a/crates/ty_python_semantic/src/types/narrow.rs b/crates/ty_python_semantic/src/types/narrow.rs index 450d32c1fd..d8816bdae1 100644 --- a/crates/ty_python_semantic/src/types/narrow.rs +++ b/crates/ty_python_semantic/src/types/narrow.rs @@ -21,8 +21,8 @@ use crate::types::{ CallableType, ClassBase, ClassLiteral, ClassPatternPositionalSource, ClassType, IntersectionBuilder, IntersectionType, KnownClass, KnownInstanceType, LiteralValueTypeKind, Parameter, Parameters, Signature, SpecialFormType, SubclassOfInner, SubclassOfType, Truthiness, - Type, TypeContext, TypeVarBoundOrConstraints, UnionBuilder, basedpython_is_keeps_identity, - binding_type, callable_pattern_type, class_pattern_positional_sources, + Type, TypeContext, TypeVarBoundOrConstraints, UnionBuilder, binding_type, + callable_pattern_type, class_pattern_positional_sources, definite_match_pattern_type_for_subject, exact_sequence_pattern_type, infer_expression_types, mapping_pattern_type, pattern_binding_fallthrough_type, sequence_pattern_type_builder, singleton_pattern_type, starred_sequence_pattern_type, typed_dict_matches_class_pattern, @@ -4230,6 +4230,7 @@ impl<'db> NarrowingConstraintsBuilder<'db, '_> { let ast::ExprCompare { range: _, node_index: _, + identity_ops: _, left, ops, comparators, @@ -4543,19 +4544,18 @@ impl<'db> NarrowingConstraintsBuilder<'db, '_> { }; let mut last_rhs_ty: Option = None; - // basedpython: in `.by` files, the `is`/`is not` keyword form - // performs isinstance-style narrowing on the lhs (the `===`/`!==` - // operators retain Python's identity-narrowing semantics). The - // parser flattens both spellings to the same `CmpOp`, so we look - // at the original source text between the two operands to tell - // them apart - let file = expression.file(self.db); - let basedpython_keyword_form = file - .source_type(self.db) - .is_basedpython() - .then(|| ruff_db::source::source_text(self.db, file)); - - for (op, (left, right)) in std::iter::zip(&**ops, comparator_tuples) { + // basedpython: in `.by` files, the `is`/`is not` keyword form is a type + // test and narrows the way `isinstance` does; `===` / `!==` keep + // python's identity narrowing. the parser folds both spellings onto one + // `CmpOp` and records which it saw + let source_type = expression.file(self.db).source_type(self.db); + // a chain carrying a type test is a parse error, and the inference + // builder falls back to reading the whole chain as ordinary + // comparisons. narrowing has to read it the same way, or the body of a + // branch it guards disappears on top of the syntax error + let chained_type_test = ops.len() > 1 && expr_compare.has_type_test(source_type); + + for (index, (op, (left, right))) in std::iter::zip(&**ops, comparator_tuples).enumerate() { let lhs_ty = last_rhs_ty.unwrap_or_else(|| expression_type(left, &self.env)); let rhs_ty = expression_type(right, &self.env); let lhs_narrowing_rhs_ty = if matches!(op, ast::CmpOp::In | ast::CmpOp::NotIn) { @@ -4565,23 +4565,8 @@ impl<'db> NarrowingConstraintsBuilder<'db, '_> { rhs_ty }; - // literal rhs (None, True/False, numbers, strings, bytes, `...`) keeps - // Python identity semantics — `isinstance(x, None)` would be invalid. - // the same holds for any rhs that resolves to a plain value (an enum - // member like `Color.RED`, a based unit variant, an instance of a - // non-type class): it is not a class, so the transpiler keeps `is` - // and narrowing must mirror that - let basedpython_is_keyword = basedpython_keyword_form.as_ref().is_some_and(|src| { - use ruff_text_size::Ranged; - matches!(op, ast::CmpOp::Is | ast::CmpOp::IsNot) && !right.is_literal_expr() && { - let between = - &src[usize::from(left.range().end())..usize::from(right.range().start())]; - let trimmed = between.trim(); - !trimmed.starts_with("===") && !trimmed.starts_with("!==") - } - }) && !basedpython_is_keeps_identity( - self.db, &env, rhs_ty, - ); + let basedpython_is_keyword = + !chained_type_test && expr_compare.is_type_test(index, source_type); // Narrowing for: // - `if type(x) is Y` @@ -4644,15 +4629,25 @@ impl<'db> NarrowingConstraintsBuilder<'db, '_> { && let Some(narrowable) = PlaceExpr::try_from_expr(left) { let positive = is_positive == matches!(op, ast::CmpOp::Is); - // a parametric test (`x is list[int]`) verifies the exact - // specialization, so the positive branch narrows to it. no - // negative narrowing: an unreified or witness-less (empty) - // value answers `False` even when it *is* one statically - let constraint = if let Type::GenericAlias(alias) = rhs_ty { - positive.then(|| Type::instance(self.db, &env, ClassType::Generic(alias))) + // the right-hand side of a type test is a type expression, so + // its inferred type *is* the type being tested for — there is + // no class object to unwrap. the negative branch narrows only + // where the runtime check is exact: a parametric probe reads the + // arguments a value happens to record, and answers `False` for + // one that records none even though it is a match + let constraint = if positive { + Some(rhs_ty) } else { - ClassInfoConstraintFunction::IsInstance - .generate_constraint(self.db, &env, rhs_ty, positive, false) + crate::types::reified_infer::type_test_plan( + self.db, + &env, + expression.file(self.db), + lhs_ty, + rhs_ty, + Some(right), + ) + .narrows_negatively() + .then_some(rhs_ty) }; if let Some(constraint_ty) = constraint { let place = self.expect_place(&narrowable); diff --git a/crates/ty_python_semantic/src/types/reified_infer.rs b/crates/ty_python_semantic/src/types/reified_infer.rs index 8189cab160..577693b7b5 100644 --- a/crates/ty_python_semantic/src/types/reified_infer.rs +++ b/crates/ty_python_semantic/src/types/reified_infer.rs @@ -16,6 +16,8 @@ //! classes, exotic type forms — has no spelling and the bare call stays an //! error +use std::fmt::Write as _; + use itertools::Itertools; use ruff_db::files::File; use ruff_db::parsed::parsed_module; @@ -696,7 +698,16 @@ pub enum ParametricIsPlan { /// generic whose instances carry `__orig_class__` — probe it at runtime, /// matching each argument by the target's effective variance (one entry /// per type parameter). a legitimate, unwarned runtime test - Probe(Box<[ArgVariance]>), + Probe { + /// the target's runtime spelling (`A[int]`), or `None` when the source + /// already spells it and should be passed through — which is the more + /// robust of the two, since a name the source wrote is in scope by + /// construction while a rebuilt spelling needs the origin class to be + /// nameable where the test is written. `Some` where nothing in the + /// source spells this target on its own: one arm of a union + target: TargetSpelling, + variances: Box<[ArgVariance]>, + }, /// basedpython: not decidable from static types, and the target is a /// protocol — but every data member's specialized type has a runtime /// spelling, so the value's reified annotations can be checked structurally @@ -706,6 +717,114 @@ pub enum ParametricIsPlan { /// a usable `__orig_class__`, so no sound runtime probe exists — the test /// is an error. the reason picks the diagnostic wording ErasedTarget(ErasedTargetReason), + /// `isinstance(value, )` — the target is a plain class, which is + /// exactly what `isinstance` was built to answer. `None` where the source + /// spells the target itself and should be passed through, which names even + /// a class no module global does (an enum's `Shape.Circle`) + Isinstance(TargetSpelling), + /// `callable(value)` — a `Callable` with no signature, which asks only what + /// the runtime records + IsCallable, + /// `value is None`. `None` is a value, not a class, so `isinstance` cannot + /// take it, and identity is the whole test — there is only one `None` + IsNone, + /// `type(value) is and value == ` — a literal target such as + /// `Literal[3]`, whose type holds exactly one value. the class guard is not + /// redundant: python's `1 == True` would otherwise let a `bool` satisfy + /// `Literal[1]` + Equality { + class: String, + /// the value to compare against. the source's own literal where it + /// wrote one — which is already spelled correctly for wherever it sits, + /// including inside an f-string on a python that forbids reusing the + /// outer quote + value: TargetSpelling, + }, + /// `value is ` — an enum member, which is a singleton, so + /// identity is both exact and what the runtime compares anyway. `None` + /// passes the source through, as for [`Self::Isinstance`] + Identity(TargetSpelling), + /// `isinstance(value, type) and issubclass(value, )` — a + /// `type[C]` target, which the runtime can check in full. `None` passes the + /// source through, as for [`Self::Isinstance`] + Subclass(TargetSpelling), + /// a template literal type: the value must be a `str` matching this + /// regular expression, which spells the same language `matches_str` decides + Pattern(String), + /// the target is an interface something in scope visibly *conforms* to, so + /// the conformance registry answers the test. a conforming type is not a + /// subclass, so `isinstance` could never see the relationship + Conformance { + /// how the interface is written into the emitted python + target: TargetSpelling, + /// the interface's required member names, for the runtime check to look + /// for on a value nothing registered + members: Vec, + }, + /// nothing is known about the target — an error elsewhere left `Unknown` + /// behind. the test is lowered as the plain `isinstance` the source spells + /// and reported by whatever produced the `Unknown`, which is the sharper + /// report and the only one + Unresolved, + /// the disjunction of these plans — the target is a union, and a value + /// satisfies it as soon as one arm holds. the arms are carried as plans of + /// their own because a union's arms need not be spelled in the source: a + /// PEP 695 alias names one with a single identifier + Union(Box<[ParametricIsPlan]>), +} + +impl ParametricIsPlan { + /// whether a `False` from this test proves the value does *not* have the + /// type, so the negative branch may narrow. + /// + /// An exact check answers the question the type asks. The parametric ones + /// do not: a runtime probe reads the arguments a value happens to record, + /// and a value that records none answers `False` even where the static + /// types say it is a match — narrowing on that would remove a type the + /// value really has. + pub(crate) fn narrows_negatively(&self) -> bool { + match self { + Self::Fold(_) + | Self::Isinstance(_) + | Self::IsCallable + | Self::IsNone + | Self::Equality { .. } + | Self::Identity(_) + | Self::Subclass(_) + | Self::Pattern(_) => true, + Self::Union(arms) => arms.iter().all(Self::narrows_negatively), + Self::TokenEq(_) + | Self::Probe { .. } + | Self::ProtocolStructural(_) + | Self::Conformance { .. } + | Self::Unresolved + | Self::ErasedTarget(_) => false, + } + } + + /// whether this plan proves the test can never hold, so the guarded branch + /// is dead. only a static fold proves that; every runtime residue leaves + /// the answer to the value + pub(crate) fn never_holds(&self) -> bool { + match self { + Self::Fold(holds) => !holds, + Self::Union(arms) => arms.iter().all(Self::never_holds), + _ => false, + } + } +} + +/// basedpython: how a type test's target is written into the emitted python +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TargetSpelling { + /// the source spells the target itself, so its own text is passed through. + /// this is the more robust of the two: a name the source wrote is in scope + /// by construction, while a rebuilt spelling needs the class to be nameable + /// where the test is written + Written, + /// nothing in the source spells this target on its own — one arm of a union + /// the source named with a single word — so it is rebuilt + Rebuilt(String), } /// basedpython: one protocol member a parametric `is`-test checks structurally @@ -736,9 +855,36 @@ pub enum ProtocolMemberCheck { }, } -/// why a parametric test's target cannot be probed at runtime +/// why a type test's target cannot be checked at runtime #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ErasedTargetReason { + /// the target says nothing a runtime check could look for — `Any` and + /// `Unknown` admit every value, so the test has no content + Dynamic, + /// a callable type: the runtime sees that a value is callable and nothing + /// more, so its parameter and return types would be assumed rather than + /// checked + Callable, + /// a `TypedDict`: its instances are plain dicts, so a runtime check can + /// only ask whether the value is a `dict` and would assume every key + TypedDict, + /// an intersection of types has no single runtime form to test against + Intersection, + /// a protocol that is not `@runtime_checkable` and has a member with no + /// runtime spelling, so neither python's presence check nor a structural + /// one against the value's reified annotations can answer it + NonRuntimeCheckableProtocol, + /// a `type[…]` whose argument `issubclass` cannot take — `Any`, or a + /// protocol with a data member + Subclass, + /// a literal whose value the runtime cannot be asked to compare: a `float` + /// or `complex`, whose equality does not decide the type (`0.0 == -0.0`), + /// or `LiteralString`, which is a property of how a value was written and + /// not of the value + UncomparableLiteral, + /// the target has no runtime spelling at all — a type the checker can name + /// but the emitted python cannot evaluate + Unspellable, /// a builtin collection (`list` / `dict` / `set` / `frozenset` / `tuple`) /// erases its type arguments — its C-level instances reject /// `__orig_class__` entirely @@ -906,7 +1052,7 @@ fn erased_target_reason<'db>( /// (`X = list[int]`); a PEP 695 `type` alias is unwrapped to the same. `None` /// when the rhs is not a specialization — a bare class or value — so the caller /// keeps the ordinary `isinstance` lowering. -pub(crate) fn parametric_is_target<'db>( +fn parametric_is_target<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, rhs_ty: Type<'db>, @@ -969,11 +1115,11 @@ pub(crate) fn classify_parametric_is<'db>( file: File, lhs_ty: Type<'db>, rhs_alias: crate::types::class::GenericAlias<'db>, - rhs_node: &ast::Expr, + rhs_node: Option<&ast::Expr>, ) -> ParametricIsPlan { let target_origin = ClassLiteral::Static(rhs_alias.origin(db)); let target_args_ast: Vec<&ast::Expr> = match rhs_node { - ast::Expr::Subscript(subscript) => match subscript.slice.as_ref() { + Some(ast::Expr::Subscript(subscript)) => match subscript.slice.as_ref() { ast::Expr::Tuple(tuple) => tuple.elts.iter().collect(), single => vec![single], }, @@ -982,8 +1128,8 @@ pub(crate) fn classify_parametric_is<'db>( let plan = classify_value( db, env, + file, lhs_ty.promote(db, env), - target_origin, rhs_alias, &target_args_ast, rhs_node, @@ -997,7 +1143,7 @@ pub(crate) fn classify_parametric_is<'db>( // spellable data members can still be checked structurally against those // annotations. only a protocol that also has a method member (unrecoverable // from an annotation) stays an error - if let ParametricIsPlan::Probe(_) = plan + if let ParametricIsPlan::Probe { .. } = plan && let Some(ErasedTargetReason::Protocol) = erased_target_reason(db, target_origin) { return protocol_structural_members(db, env, file, ClassType::Generic(rhs_alias)) @@ -1015,7 +1161,7 @@ pub(crate) fn classify_parametric_is<'db>( // `x is Sequence[int]` — which runs perfectly well — into an error if matches!( plan, - ParametricIsPlan::Probe(_) | ParametricIsPlan::TokenEq(_) + ParametricIsPlan::Probe { .. } | ParametricIsPlan::TokenEq(_) ) && runtime_subscript(db, env, target_origin) == RuntimeSubscript::Unsupported { return ParametricIsPlan::ErasedTarget(ErasedTargetReason::NotSubscriptable); @@ -1023,6 +1169,517 @@ pub(crate) fn classify_parametric_is<'db>( plan } +/// basedpython: how a type test — `value is Target` — resolves, for *any* +/// target type. +/// +/// The right-hand side of a type test is a type expression, so an unusable +/// target has already been reported as `invalid-type-form` by the time this +/// runs. What is left is a narrower question: does the type it named have a +/// runtime form? A test must *earn* its `True` — it narrows — so a target the +/// runtime can only partly check is rejected rather than approximated. That +/// rules out a `TypedDict` (its instances are plain dicts), a callable type +/// (the runtime sees only that a value is callable), and `Any`. +/// +/// `target_node` is the source the target was written as, which the +/// specialization plans use to spell type arguments back out. +/// +/// `value_ty` is taken exactly as the value has it — a construction's `final A` +/// is disjoint from an unrelated class in a way a plain `A` is not, and that is +/// the whole point of tracking it. Only the specialization path widens, where a +/// literal argument would otherwise decide a test the runtime cannot. +pub(crate) fn type_test_plan<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: File, + value_ty: Type<'db>, + target: Type<'db>, + target_node: Option<&ast::Expr>, +) -> ParametricIsPlan { + type_test_plan_seen( + db, + env, + file, + value_ty, + target, + target_node, + &mut Vec::new(), + ) +} + +/// [`type_test_plan`] carrying the aliases already opened on the way here. +/// +/// A `type` alias may name itself — `type A = int | B` with `type B = str | A` +/// — and the plan for one is the plan for its value, so following that without +/// a record would not terminate. +fn type_test_plan_seen<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: File, + value_ty: Type<'db>, + target: Type<'db>, + target_node: Option<&ast::Expr>, + open: &mut Vec>, +) -> ParametricIsPlan { + // an `Unknown` target is not evidence of anything: every value is a subtype + // of it, and folding on that would answer `True` for a test the source got + // wrong somewhere else + if target.is_dynamic() { + return runtime_test_plan(db, env, file, value_ty, target, target_node, open); + } + // a target that resolves statically needs no runtime residue at all, and + // answering it here keeps every unspellable-but-decidable target working: + // `x is Never` is `False` without the runtime ever seeing `Never` + if value_ty.is_subtype_of(db, env, target) { + return ParametricIsPlan::Fold(true); + } + if value_ty.is_disjoint_from(db, env, target) { + return ParametricIsPlan::Fold(false); + } + runtime_test_plan(db, env, file, value_ty, target, target_node, open) +} + +/// The runtime residue of a type test whose answer the static types do not +/// already give. Split from [`type_test_plan`] so a union arm can be planned +/// without re-asking the static question the whole test already answered. +fn runtime_test_plan<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: File, + value_ty: Type<'db>, + target: Type<'db>, + target_node: Option<&ast::Expr>, + open: &mut Vec>, +) -> ParametricIsPlan { + if target.is_none(db) { + return ParametricIsPlan::IsNone; + } + match target { + // an alias stands for its value, and a use-site modifier (`final T`) + // constrains how the value may be used rather than what it is at + // runtime — neither adds anything a runtime check could look for + Type::TypeAlias(alias) => { + // an alias that names itself has no value to resolve to, and the + // definition is reported where it is written + if open.contains(&target) { + return ParametricIsPlan::Unresolved; + } + open.push(target); + let plan = runtime_test_plan(db, env, file, value_ty, alias.value_type(db), None, open); + open.pop(); + plan + } + // a use-site modifier is not python, so the source no longer spells + // what is left once it is dropped + Type::Restricted(restricted) => runtime_test_plan( + db, + env, + file, + value_ty, + restricted.value_type(db), + None, + open, + ), + + // a value satisfies a union as soon as it satisfies one arm. the arms + // are planned separately because a union need not be spelled as one: + // `type AU = int | str` names it with a single identifier, and + // `isinstance` cannot take the alias object that identifier evaluates to + Type::Union(union) => { + let mut arms = Vec::with_capacity(union.elements(db).len()); + for element in union.elements(db) { + // the source spells the union, not this arm, so the arm's own + // plan may not read type arguments back out of it + let arm = type_test_plan_seen(db, env, file, value_ty, *element, None, open); + // one unusable arm makes the whole disjunction unusable: it may + // not quietly fold to `False`, since that would answer `False` + // for a value the arm would have accepted + if let ParametricIsPlan::ErasedTarget(reason) = arm { + return ParametricIsPlan::ErasedTarget(reason); + } + // an arm the checker could not read has no spelling of its own, + // and the source spells the union rather than the arm — so the + // whole test falls back to what the source wrote + if matches!(arm, ParametricIsPlan::Unresolved) { + return ParametricIsPlan::Unresolved; + } + arms.push(arm); + } + ParametricIsPlan::Union(arms.into_boxed_slice()) + } + + // a literal type holds exactly the values equal to it, which is what + // the runtime compares. an enum member is a literal too, and its + // equality is identity + Type::LiteralValue(literal) => literal_target_plan(db, env, file, literal, target_node), + + // `type[C]`: the runtime can check this one in full — the value must be + // a class, and a subclass of `C` + Type::SubclassOf(subclass) => match subclass.subclass_of().into_class(db, env) { + // `type[C]` is written as a subscript, so the source spells the + // subscript rather than `C` — the spelling has to be rebuilt + Some(class) => match spell_class(db, env, file, class) { + Some(spelling) => ParametricIsPlan::Subclass(TargetSpelling::Rebuilt(spelling)), + None => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable), + }, + // `type[Any]`, or `type[]` — the value must be a class, + // and `issubclass` has nothing it can ask beyond that + None => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Subclass), + }, + + Type::NominalInstance(_) | Type::ProtocolInstance(_) => { + class_target_plan(db, env, file, value_ty, target, target_node) + } + + // the value's type is carried by a reified type parameter on the + // *target* side — `x is T` where `T` is reified spells `T` itself + Type::TypeVar(bound_typevar) if is_reified_function_typevar(db, bound_typevar) => { + ParametricIsPlan::Isinstance(TargetSpelling::Rebuilt( + bound_typevar.name(db).to_string(), + )) + } + + // `Any` really does admit every value and is worth rejecting. `Unknown` + // is what an already-reported error leaves behind, and a second report + // on the same target would only repeat it + Type::Dynamic(crate::types::DynamicType::Any) => { + ParametricIsPlan::ErasedTarget(ErasedTargetReason::Dynamic) + } + Type::Dynamic(_) => ParametricIsPlan::Unresolved, + // a bare `Callable` is exactly what `callable()` answers; only a + // *signature* asks for something the value does not record + Type::Callable(callable) + if callable.signatures(db).iter().all(is_unannotated_signature) => + { + ParametricIsPlan::IsCallable + } + Type::Callable(_) => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Callable), + Type::Intersection(_) => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Intersection), + // a `TypedDict`'s inhabitants are plain dicts: nothing at runtime + // records which one a dict was built as + Type::TypedDict(_) => ParametricIsPlan::ErasedTarget(ErasedTargetReason::TypedDict), + _ => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable), + } +} + +/// The runtime test for a literal target. A `TypedDict`-like erasure is not +/// good enough here: the type holds exactly the values equal to the literal, so +/// the check is that equality. +fn literal_target_plan<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: File, + literal: crate::types::LiteralValueType<'db>, + target_node: Option<&ast::Expr>, +) -> ParametricIsPlan { + let unspellable = ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable); + // the guard names a builtin, whose spelling is fixed and always in scope + let written = target_node + .is_some_and(|node| node.is_literal_expr() || matches!(node, ast::Expr::UnaryOp(_))); + let equality = |class: &str, value: String| ParametricIsPlan::Equality { + class: class.to_owned(), + value: if written { + TargetSpelling::Written + } else { + TargetSpelling::Rebuilt(value) + }, + }; + match literal.kind() { + // a pattern is a set of strings, and the regular expression that spells + // it decides exactly the language `matches_str` decides + LiteralValueTypeKind::Template(template) => match template_pattern(db, env, template) { + Some(pattern) => ParametricIsPlan::Pattern(pattern), + None => unspellable, + }, + LiteralValueTypeKind::Bool(value) => { + equality("bool", (if value { "True" } else { "False" }).to_owned()) + } + LiteralValueTypeKind::Int(value) => equality("int", value.as_i64().to_string()), + LiteralValueTypeKind::String(value) => equality("str", python_str_literal(value.value(db))), + LiteralValueTypeKind::Bytes(value) => { + equality("bytes", python_bytes_literal(value.value(db))) + } + // an enum member is a singleton, so identity is exact — and it is the + // comparison `Enum.__eq__` performs anyway + LiteralValueTypeKind::Enum(member) => { + match written_or_spelled(target_node, false, || { + spell_class_literal(db, env, file, member.enum_class(db)) + .map(|class| format!("{class}.{}", member.name(db))) + }) { + Some(spelling) => ParametricIsPlan::Identity(spelling), + None => unspellable, + } + } + // `LiteralString` is the *property* of having been written as a literal, + // which nothing about a value at runtime records; `float` and `complex` + // literals are values the checker tracks but does not promise are + // distinguishable from equal ones + LiteralValueTypeKind::LiteralString + | LiteralValueTypeKind::Float(_) + | LiteralValueTypeKind::Complex(_) => { + ParametricIsPlan::ErasedTarget(ErasedTargetReason::UncomparableLiteral) + } + } +} + +/// how a target is written into the emitted python: the spelling rebuilt from +/// the type, or the source's own text where that is what the runtime evaluates. +/// +/// The rebuilt spelling is preferred, because the source names a *type* and the +/// emitted check needs a *value*, and the two part company far more often than +/// they look like they do: a PEP 695 alias evaluates to a `TypeAliasType`, +/// `Literal[…]` and `Annotated[…]` to special forms, `list[Any]` to a +/// subscripted generic — none of which `isinstance` will take. Falling back to +/// the source is for the one case rebuilding cannot express: a class the +/// emitting module cannot name as a global, such as an enum's attached variant +/// or a class imported under another name. Only a plain dotted name qualifies, +/// and only a [`Probe`](ParametricIsPlan::Probe) also accepts a subscript, +/// whose value is the specialization it unwinds. +fn written_or_spelled( + target_node: Option<&ast::Expr>, + subscript_evaluates: bool, + spell: impl FnOnce() -> Option, +) -> Option { + if let Some(spelling) = spell() { + return Some(TargetSpelling::Rebuilt(spelling)); + } + target_node + .filter(|node| evaluates_to_its_target(node, subscript_evaluates)) + .map(|_| TargetSpelling::Written) +} + +/// whether the runtime value of `node` is the thing the type it names denotes +fn evaluates_to_its_target(node: &ast::Expr, subscript_evaluates: bool) -> bool { + match node { + ast::Expr::Name(_) => true, + ast::Expr::Attribute(attribute) => evaluates_to_its_target(&attribute.value, false), + ast::Expr::Subscript(subscript) if subscript_evaluates => { + evaluates_to_its_target(&subscript.value, false) + } + _ => false, + } +} + +/// a python string literal for `value`, escaped so the emitted source reads it +/// back character for character. +fn python_str_literal(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('"'); + for ch in value.chars() { + match ch { + '\\' => out.push_str("\\\\"), + ch if ch.is_control() || ch == '"' => push_escaped_char(ch, &mut out), + ch => out.push(ch), + } + } + out.push('"'); + out +} + +/// a python bytes literal for `value`, written one `\xNN` escape per byte so +/// every byte round-trips whatever it is +fn python_bytes_literal(value: &[u8]) -> String { + let mut out = String::with_capacity(value.len() * 4 + 3); + out.push_str("b\""); + for byte in value { + let _ = write!(out, "\\x{byte:02x}"); + } + out.push('"'); + out +} + +/// The runtime test for an instance target — the common case, and the one the +/// parametric engine already answered for a specialization. +fn class_target_plan<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: File, + value_ty: Type<'db>, + target: Type<'db>, + target_node: Option<&ast::Expr>, +) -> ParametricIsPlan { + let Some(class) = target_class(db, env, target) else { + return ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable); + }; + let literal = class.class_literal(db); + // a `TypedDict`'s instances are plain dicts, so the only runtime question + // is whether the value is a `dict` — every key and value type would be + // assumed. a test that must earn its `True` cannot assume them + if literal.is_typed_dict(db) { + return ParametricIsPlan::ErasedTarget(ErasedTargetReason::TypedDict); + } + match class { + // a bare generic class in a type expression means every specialization + // of it, which is the class itself — and `isinstance` answers exactly + // that. only *written* arguments give the parametric engine something + // to check + ClassType::Generic(alias) + if alias + .specialization(db) + .types(db) + .iter() + .all(Type::is_dynamic) => + { + match written_or_spelled(target_node, false, || { + spell_class_literal(db, env, file, ClassLiteral::Static(alias.origin(db))) + }) { + Some(spelling) => ParametricIsPlan::Isinstance(spelling), + None => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable), + } + } + // a specialization keeps the engine it always had: reified cells, + // an `__orig_class__` probe, or a structural protocol check + ClassType::Generic(alias) => { + classify_parametric_is(db, env, file, value_ty, alias, target_node) + } + ClassType::NonGeneric(_) => { + // an interface something visibly conforms to is answered by the + // registry rather than by the class hierarchy, so it is checkable + // even though a conforming type is not a subclass + if let Some(members) = conformance_members(db, file, class) { + return match written_or_spelled(target_node, false, || { + spell_class(db, env, file, class) + }) { + Some(target) => ParametricIsPlan::Conformance { target, members }, + None => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable), + }; + } + if let Some(protocol) = class.into_protocol_class(db) { + // `@runtime_checkable` is the author's own statement that + // `isinstance` may take the class, and it is what python then + // checks — that the members are present + if !protocol.is_runtime_checkable(db) { + // basedpython reifies class annotations, so a protocol whose + // members all have a runtime spelling can be checked against + // them member by member — a stricter answer than presence, + // and the only one available without the decorator + return match protocol_structural_members(db, env, file, class) { + Some(checks) => { + ParametricIsPlan::ProtocolStructural(checks.into_boxed_slice()) + } + None => ParametricIsPlan::ErasedTarget( + ErasedTargetReason::NonRuntimeCheckableProtocol, + ), + }; + } + } + match written_or_spelled(target_node, false, || spell_class(db, env, file, class)) { + Some(spelling) => ParametricIsPlan::Isinstance(spelling), + None => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable), + } + } + } +} + +/// The required member names of `class` when something in `file`'s scope +/// visibly conforms to it, or `None` when it is not a conformance interface. +fn conformance_members<'db>( + db: &'db dyn Db, + file: File, + class: ClassType<'db>, +) -> Option> { + let conformed = crate::types::conformance::visible_conformances(db, file) + .iter() + .any(|(_, declared)| declared.class_literal(db) == class.class_literal(db)); + conformed.then(|| { + crate::types::conformance::interface_requirements(db, class) + .iter() + .map(ToString::to_string) + .collect() + }) +} + +/// whether a callable signature says nothing beyond "this is callable" — the +/// gradual form `Callable[..., Any]` the bare `Callable` denotes +fn is_unannotated_signature(signature: &crate::types::signatures::Signature<'_>) -> bool { + signature.parameters().is_gradual() && signature.return_ty.is_dynamic() +} + +/// The class an instance target names, for both the nominal and the protocol +/// spelling of one. +fn target_class<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + target: Type<'db>, +) -> Option> { + match target { + Type::NominalInstance(instance) => Some(instance.class(db, env)), + Type::ProtocolInstance(instance) => match instance.inner { + crate::types::instance::Protocol::FromClass(class) => Some(*class), + crate::types::instance::Protocol::Materialized(_) + | crate::types::instance::Protocol::Synthesized(_) => None, + }, + _ => None, + } +} + +/// The regular expression that matches exactly the strings a template literal +/// type produces, for `re.fullmatch`, or `None` when one of its holes has no +/// regular-expression spelling. +/// +/// The holes are read through the same [`HoleShape`](crate::types::template::HoleShape) +/// classification +/// `matches_str` uses, so the runtime test and the static one decide the same +/// language rather than two that happen to agree on the cases anyone tried. +fn template_pattern<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + template: crate::types::template::TemplateLiteralType<'db>, +) -> Option { + let mut pattern = String::new(); + for part in template.parts(db) { + match part { + crate::types::template::TemplatePart::Text(text) => { + escape_regex(text.as_str(), &mut pattern); + } + crate::types::template::TemplatePart::Hole(hole) => { + pattern.push_str(crate::types::template::HoleShape::of(db, env, *hole).regex()?); + } + } + } + Some(pattern) +} + +/// Append `text` to `pattern` with every character python's `re` gives a +/// meaning escaped, so the text matches itself. +/// +/// A character the emitted source cannot carry — anything the `str` escaping +/// below would have to spell — is written as its own `\\xNN` / `\\uNNNN` +/// escape, which `re` reads as that character. Passing one through raw would +/// put a literal control byte in the python, and CPython refuses to compile a +/// source containing a NUL at all. +fn escape_regex(text: &str, pattern: &mut String) { + for ch in text.chars() { + if "\\.^$*+?()[]{}|-#&~".contains(ch) { + pattern.push('\\'); + pattern.push(ch); + } else if ch.is_control() || ch == '"' { + push_escaped_char(ch, pattern); + } else { + pattern.push(ch); + } + } +} + +/// Write `ch` as the escape a python string literal reads back as that +/// character. Used for anything the emitted source cannot carry raw. +fn push_escaped_char(ch: char, out: &mut String) { + match ch { + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + '"' => out.push_str("\\\""), + ch if (ch as u32) < 0x100 => { + let _ = write!(out, "\\x{:02x}", ch as u32); + } + ch if (ch as u32) < 0x1_0000 => { + let _ = write!(out, "\\u{:04x}", ch as u32); + } + ch => { + let _ = write!(out, "\\U{:08x}", ch as u32); + } + } +} + /// basedpython: the structural runtime check for a protocol target whose data /// members can all be verified against a value's reified class annotations, or /// `None` when the protocol has a member that can't be — a method (its shape @@ -1163,12 +1820,13 @@ fn is_object_instance<'db>(db: &'db dyn Db, env: &ProgramEnvironment<'db>, ty: T fn classify_value<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, + file: File, value_ty: Type<'db>, - target_origin: ClassLiteral<'db>, rhs_alias: crate::types::class::GenericAlias<'db>, target_args_ast: &[&ast::Expr], - rhs_node: &ast::Expr, + rhs_node: Option<&ast::Expr>, ) -> ParametricIsPlan { + let target_origin = ClassLiteral::Static(rhs_alias.origin(db)); // when the value's type is carried by a reified type parameter, the answer // lives in a runtime cell rather than the static type — extract the cell // comparisons before falling back to static subtyping @@ -1196,7 +1854,17 @@ fn classify_value<'db>( } else { // undecidable statically; `classify_parametric_is` turns this into a // runtime probe (user generic) or an erased-target error (builtin) - ParametricIsPlan::Probe(target_variances(db, rhs_alias)) + // the source's own spelling is preferred: a name it wrote is in scope, + // and the runtime probe unwraps a `TypeAliasType` for itself + let Some(target) = written_or_spelled(rhs_node, true, || { + spell_class(db, env, file, ClassType::Generic(rhs_alias)) + }) else { + return ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable); + }; + ParametricIsPlan::Probe { + target, + variances: target_variances(db, rhs_alias), + } } } @@ -1211,7 +1879,7 @@ fn try_token_eq<'db>( target_origin: ClassLiteral<'db>, rhs_alias: crate::types::class::GenericAlias<'db>, target_args_ast: &[&ast::Expr], - rhs_node: &ast::Expr, + rhs_node: Option<&ast::Expr>, ) -> Option { match value_ty { // `x: T is ` compares the reified `T` cell against the target @@ -1221,11 +1889,11 @@ fn try_token_eq<'db>( // `TypeAliasType` wrapper), so it falls through to the static resolution Type::TypeVar(bound_typevar) if is_reified_function_typevar(db, bound_typevar) - && matches!(rhs_node, ast::Expr::Subscript(_)) => + && matches!(rhs_node, Some(ast::Expr::Subscript(_))) => { Some(ParametricIsPlan::TokenEq(vec![( bound_typevar.name(db).clone(), - rhs_node.range(), + rhs_node.expect("guarded above").range(), )])) } Type::NominalInstance(instance) => { @@ -1453,8 +2121,7 @@ fn is_reified_function_typevar<'db>( return false; }; let node = function.node(&module); - let source = ruff_db::source::source_text(db, def_file); - crate::reified::reified_type_param_names(source.as_str(), def_file.source_type(db), node) + crate::reified::reified_type_param_names(def_file.source_type(db), node) .iter() .any(|name| name == bound_typevar.name(db)) } diff --git a/crates/ty_python_semantic/src/types/template.rs b/crates/ty_python_semantic/src/types/template.rs index 9211971fc9..6fb16675a9 100644 --- a/crates/ty_python_semantic/src/types/template.rs +++ b/crates/ty_python_semantic/src/types/template.rs @@ -92,7 +92,7 @@ pub(crate) enum HoleShape { } impl HoleShape { - fn of<'db>(db: &'db dyn Db, env: &ProgramEnvironment<'db>, hole: Type<'db>) -> Self { + pub(crate) fn of<'db>(db: &'db dyn Db, env: &ProgramEnvironment<'db>, hole: Type<'db>) -> Self { let Some(class) = hole.nominal_class(db, env) else { return Self::Anything; }; @@ -123,6 +123,22 @@ impl HoleShape { fn admits_empty(self) -> bool { self == Self::Anything } + + /// the regular expression matching exactly this shape's strings, or `None` + /// when it has no spelling python's `re` can evaluate. + /// + /// A grapheme is a cluster of code points rather than one, and `re` has no + /// way to say that, so a pattern with a grapheme hole has no runtime test — + /// which is why the type test rejects one rather than approximating it + pub(crate) fn regex(self) -> Option<&'static str> { + match self { + Self::Anything => Some("(?s:.*)"), + // exactly the renderings `is_int_rendering` accepts — `str(-0)` is + // `"0"`, so the sign belongs to the non-zero alternative alone + Self::Int => Some("(?:0|-?[1-9][0-9]*)"), + Self::Grapheme => None, + } + } } /// whether `value` is what `str()` produces for some `int` diff --git a/docs/basedpython/features/differences-from-python.md b/docs/basedpython/features/differences-from-python.md index 05e0c5c507..0a2b060927 100644 --- a/docs/basedpython/features/differences-from-python.md +++ b/docs/basedpython/features/differences-from-python.md @@ -32,7 +32,7 @@ the same source, running, does something else the compiler doesn't always do `isinstance`, for example `x is None` will become `x is None` in python, this is because "type of x is None" and "value of x is None" have identical meanings -see [identity and isinstance](identity-swap.md) +see [type tests and identity](identity-swap.md) ### a mutable default is re-evaluated per call diff --git a/docs/basedpython/features/enums.md b/docs/basedpython/features/enums.md index cd575279d4..c11421e27e 100644 --- a/docs/basedpython/features/enums.md +++ b/docs/basedpython/features/enums.md @@ -94,9 +94,10 @@ variants lower to **subclasses** of the enum attached as class attributes, so they are reached qualified through the enum name — `Shape.Circle(2.0)`, `Shape.Point` — everywhere: inside the enum body, in pattern contexts (`case Shape.Circle(r):`), and at module scope. variant constructors are real -classes at runtime, so `x is Shape.Circle` works (recall `is` is basedpython's -[`isinstance`](identity-swap.md); use `type(x) === Shape.Circle` for an -exact-class check). because variants are qualified, the same variant name may +classes at runtime, so `x is Shape.Circle` [tests](identity-swap.md) whether `x` +is one; use `type(x) === Shape.Circle` for an exact-class check. a payload-less +variant names the type holding exactly that one object, so `x is Shape.Point` is +the identity it comes down to. because variants are qualified, the same variant name may appear in two different enums (`A.Same` vs `B.Same`) without collision where the expected type is the enum, a variant may also be written diff --git a/docs/basedpython/features/identity-swap.md b/docs/basedpython/features/identity-swap.md index 712acae1bb..f04b625307 100644 --- a/docs/basedpython/features/identity-swap.md +++ b/docs/basedpython/features/identity-swap.md @@ -1,7 +1,7 @@ -# identity and isinstance +# type tests and identity -basedpython swaps the surface syntax for identity comparison and `isinstance` -checks: `===` is identity and `is` is an instance check +basedpython swaps the surface syntax for identity comparison and type tests: +`===` is identity, and `is` asks whether a value has a type ```by if x === y: @@ -31,8 +31,12 @@ if not isinstance(x, str): | ------------ | ---------------------- | | `x === y` | `x is y` | | `x !== y` | `x is not y` | -| `x is y` | `isinstance(x, y)` | -| `x is not y` | `not isinstance(x, y)` | +| `x is T` | `isinstance(x, T)` | +| `x is not T` | `not isinstance(x, T)` | + +`T` is a *type*, and the check that comes out is whatever decides membership of +it — `isinstance` for a class, but an equality for a literal and an identity for +`None`. the sections below say which ## why @@ -40,26 +44,62 @@ if not isinstance(x, str): rare outside of `is None`. basedpython promotes the common case to a keyword and demotes identity to a triple-equals operator borrowed from JavaScript -## checking against `None` +## the right-hand side is a type + +`is` takes a *type expression*, the same thing an annotation takes. anything an +annotation can name, a test can test against: + +```by +from typing import Literal + +def f(v: object): + if v is int | str: ... # a union + if v is list: ... # a class, arguments and all + if v is Literal[1, 2]: ... # a set of values + if v is type[int]: ... # a class object + if v is f"item-{int}": ... # a string pattern +``` + +and anything an annotation rejects, a test rejects with the same message — +`v is os` names a module, which is not a type in either place + +this is why `is` does not mean `isinstance`'s tuple: `isinstance(v, (int, str))` +spells "any of these", while `(int, str)` in a type expression is the *tuple +type*. write `v is int | str` for the first + +## checking against `None` and other values -`a is None` and `a is not None` stay as python identity checks. `a === None` -spells the same thing +`None`, `True`, a number, a string and an enum member are all type expressions — +each names the type holding exactly that one value. so the check that comes out +is the identity or equality that decides membership of that type: + +```by +if a is None: ... # → a is None +if flag is True: ... # → type(flag) is bool and flag == True +if g is Genre.A: ... # → g is Genre.A +``` -## checking against values +the class guard on a literal is not redundant: python's `1 == True` would +otherwise let a `bool` satisfy `Literal[1]` -`isinstance` requires a class as its second argument, so a rhs that resolves -to a plain *value* keeps python identity semantics. this covers literal -singletons (`None`, `True`/`False`, numbers, strings, `...`), enum members, -and any other rhs whose static type is an instance rather than a class: +## a target with no runtime form -```py -enum class Genre: - case A, B +a test narrows, so it has to *earn* its `True`. a target the runtime could only +partly check is rejected rather than approximated, by `erased-type-check` — and +the emitted python answers `False`, since a test that cannot be made is one +nothing satisfies: -Genre.A is not Genre.B # stays `is not` — members are singleton instances +```by +if v is Any: ... # error: admits every value +if v is Callable[[], int]: ... # error: the signature is not recorded on the value +if v is Movie: ... # error: a TypedDict's instances are plain dicts ``` -a payload-bearing variant (`Shape.Circle`) *is* a class, so `x is Shape.Circle` lowers to `isinstance(x, Shape.Circle)` as usual +a protocol is checkable when basedpython can see enough to check it: a +`@runtime_checkable` one gets the presence check python itself performs, and any +other is checked member by member against the value's +[reified annotations](reified-generics.md). one with a member the emitted python +cannot name is rejected ## a test that can never hold @@ -86,8 +126,30 @@ subclass of `A` either. a [parametric target](parametric-type-tests.md) is judged by the same fold that decides the test, so a use-site variance projection (`a is A[out int]`) that -makes the test possible keeps it quiet. a union target is never reported: any arm -matching makes the whole test hold +makes the test possible keeps it quiet. a union target is reported only when no +arm can hold: any arm matching makes the whole test hold + +## a settled test is its answer + +where the value's type decides the question, the test *is* that answer, and the +branch it guards is decided with it: + +```by +def f(x: int): + reveal_type(x is int) # revealed: True +``` + +an undecidable test is `bool`. the identity folds python applies to the same +operator have no place here: the right-hand side names a type rather than the +class object the same source spells as a value + +## chaining + +a type test may not join a chained comparison. python chains `a is int is str` +into `a is int and int is str`, whose second half asks whether the *class* `int` +has the type `str` — never what the writer meant, so it is a syntax error. split +it into separate tests joined with `and`. a chain of `===` / `!==` is ordinary +python and stays legal ## interaction with `==` @@ -96,7 +158,6 @@ and `===` are remapped ## scope -the swap applies to every comparison in source, with the value-rhs exemption -above decided from static types. there is no opt-out at the statement level — -write `===` / `!==` whenever you mean identity. ty understands both forms -when type-checking `.by` files +the swap applies to every comparison in source. there is no opt-out at the +statement level — write `===` / `!==` whenever you mean identity. ty understands +both forms when type-checking `.by` files diff --git a/docs/basedpython/features/index.md b/docs/basedpython/features/index.md index aacec24383..806039e161 100644 --- a/docs/basedpython/features/index.md +++ b/docs/basedpython/features/index.md @@ -172,7 +172,7 @@ syntax inside a function body
    - [context-sensitive resolution](context-sensitive-resolution.md) — `a: Color = Red` -- [identity and isinstance (`===` / `!==` / `is`)](identity-swap.md) +- [type tests and identity (`is` / `===` / `!==`)](identity-swap.md) - [optional chaining (`?.`)](optional-chaining.md) - [none-coalesce operator (`??`)](none-coalesce.md) - [postfix await (`.await`)](await-attribute.md) diff --git a/zensical.toml b/zensical.toml index ab4acdcda4..807c233347 100644 --- a/zensical.toml +++ b/zensical.toml @@ -174,7 +174,7 @@ features = [ ] }, { "expressions and statements" = [ { "context-sensitive resolution" = "features/context-sensitive-resolution.md" }, - { "identity and isinstance" = "features/identity-swap.md" }, + { "type tests and identity" = "features/identity-swap.md" }, { "optional chaining (?.)" = "features/optional-chaining.md" }, { "none-coalesce operator (??)" = "features/none-coalesce.md" }, { "postfix await (.await)" = "features/await-attribute.md" }, From 4c10e4de159aa2c5cf2bb1779cc879ecde8474c6 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:56:32 +1000 Subject: [PATCH 6/7] update docs --- docs/basedpython/getting-started.md | 12 ++++-------- docs/basedpython/index.md | 10 +++++----- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/docs/basedpython/getting-started.md b/docs/basedpython/getting-started.md index 981ec036f5..0b19c34a85 100644 --- a/docs/basedpython/getting-started.md +++ b/docs/basedpython/getting-started.md @@ -8,11 +8,9 @@ minutes basedpython ships as the `basedpython` package, which installs two executables: `by`, the type checker and transpiler, and `buff`, the linter and formatter -=== "uv" - - ```sh - uv add --dev basedpython - ``` +```sh +uv add --dev basedpython +``` verify it works: @@ -36,8 +34,7 @@ by run main ``` `by run main` finds `main.by` in the current directory, transpiles it (and all -other `.by` files in the project) to a temporary directory, then executes -`python -m main` from there +other `.by` files in the project), then executes it with your python interpreter !!! note "`by run` takes a module, not a path" @@ -86,7 +83,6 @@ the generated `.py` files are ordinary python. run them with any python tool: python build/main.py pytest build/ mypy build/ -ruff check build/ ``` to ship the project rather than run it, build a wheel — see diff --git a/docs/basedpython/index.md b/docs/basedpython/index.md index 2296f5010f..a4d62e3d94 100644 --- a/docs/basedpython/index.md +++ b/docs/basedpython/index.md @@ -9,9 +9,9 @@ files any python tool can read - **a python type checker with [framework support](frameworks/index.md)** — pydantic, sqlalchemy, pytest and django are modelled directly, so the magic they do at runtime checks like ordinary code -- **a build system** — write code against the latest version of python, and ship wheels that are compatible with old ones, no more waiting for 5 years to use something -- **basedpython, a python-like language that builds into python wheels** — `uv build`, and see [packaging](packaging.md) -- **compiles into high performance python extension modules** +- **polyfill and transpilation** — write code against the latest version of python, and ship wheels that are compatible with old ones, no more waiting for 5 years to use new features +- **basedpython-language** — based on Python, powerful and modern. fully backwards compatible +- **compiles into high performance python extension modules** — or ordinary Python - **a language server, formatter and linter** — high performance and feature rich tooling
    @@ -60,7 +60,7 @@ def main(): > > \- Guido van Rossum -Python and it's type system are held back due to an inability to make breaking changes and a +Python and its type system are held back due to an inability to make breaking changes and a hesitation to introduce new syntax other languages have indulged in modern features, powerful type systems, and integrated tooling. @@ -77,7 +77,7 @@ we want to close that gap `by build` writes ordinary `.py` files. pytest, mypy, ruff and everything else in your stack keep working, because what they see is python - [:octicons-arrow-right-24: how transpilation works](development/how-transpilation-works.md) + [:octicons-arrow-right-24: how to build](cli-reference.md) - :lucide-shapes:{ .lg .middle } **syntax python doesn't have** From 83b8f07c0983ef859a8c305ee2a423657f878840 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 8 Sep 2026 00:02:22 +1000 Subject: [PATCH 7/7] fixup! add cargo-doc prek task MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the entry is a folded scalar, which only folds the lines indented to match its first one — the deeper-indented continuations keep their newlines, so bash was handed `&& RUSTDOCFLAGS=...` at the start of a line and refused the whole command. every commit failed the hook with a syntax error rather than a doc error. the continuations line up with the first line again. --- .github/workflows/ci.yaml | 2 +- .pre-commit-config.yaml | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 302d025764..347bbc29bd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1227,7 +1227,7 @@ jobs: echo '```console' > "$GITHUB_STEP_SUMMARY" # Enable color output for prek and remove it for the summary # Use --hook-stage=manual to enable slower hooks that are skipped by default - SKIP=rustfmt uv run --only-dev --locked prek run --all-files --show-diff-on-failure --color always --hook-stage manual | \ + SKIP=rustfmt,cargo-doc uv run --only-dev --locked prek run --all-files --show-diff-on-failure --color always --hook-stage manual | \ tee >(sed -E 's/\x1B\[([0-9]{1,2}(;[0-9]{1,2})*)?[mGK]//g' >> "$GITHUB_STEP_SUMMARY") >&1 exit_code="${PIPESTATUS[0]}" echo '```' >> "$GITHUB_STEP_SUMMARY" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 32d926f708..d33a431e1f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -79,14 +79,15 @@ repos: - id: cargo-doc name: check `cargo doc` and `cargo test --doc` entry: > - bash -c 'RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --document-private-items -p ty_python_semantic - -p ty_python_core -p ty_module_resolver -p ty_site_packages -p ty_combine -p ty_project -p ty_ide -p ty_wasm - -p ty_vendored -p ty_static -p ty -p ty_test -p ruff_db -p ruff_python_formatter - && cargo test --all-features --doc' + bash -c 'RUSTDOCFLAGS="-D warnings" cargo doc --all --no-deps + && RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --document-private-items -p ty_python_semantic + -p ty_python_core -p ty_module_resolver -p ty_site_packages -p ty_combine -p ty_project -p ty_ide + -p ty_wasm -p ty_vendored -p ty_static -p ty -p ty_test -p ruff_db -p ruff_python_formatter + && cargo test --all-features --doc' types: [rust] language: system pass_filenames: false - priority: 0 + stages: [pre-push] # Prettier - repo: https://github.com/rbubley/mirrors-prettier