diff --git a/crates/basedpython/Cargo.lock b/crates/basedpython/Cargo.lock index 9bd539e6bf..bf75299539 100644 --- a/crates/basedpython/Cargo.lock +++ b/crates/basedpython/Cargo.lock @@ -1047,6 +1047,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -2235,6 +2236,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20", + "getrandom 0.4.2", "rand_core 0.10.1", ] @@ -3890,6 +3892,7 @@ dependencies = [ "jod-thread", "libc", "lsp-server", + "rand 0.10.1", "ruff_db", "ruff_diagnostics", "ruff_macros", @@ -3914,6 +3917,7 @@ dependencies = [ "ty_project", "ty_python_core", "ty_python_semantic", + "ty_static", ] [[package]] diff --git a/crates/by_stage/src/emit.rs b/crates/by_stage/src/emit.rs index 501929a4cf..f4f3e3c52c 100644 --- a/crates/by_stage/src/emit.rs +++ b/crates/by_stage/src/emit.rs @@ -20,6 +20,8 @@ use ty_project::ProjectDatabase; use ty_project::parallel::ParallelIteratorExt; use crate::project::Rebuilder; +use crate::runtime::RuntimeLayout; +use crate::staging::transpiled_destination; /// How much of the check outcome blocks emitting output. #[derive(Clone, Copy, PartialEq, Eq)] @@ -68,17 +70,41 @@ pub struct Emitted { pub blocked: bool, } +/// how and where a build turns its sources into python: one value, because it +/// is one decision taken once per command and carried unchanged through the run +pub struct Emit<'a> { + /// what the transpiler lowers for. its `runtime_module` is settled per file + pub config: &'a Config, + pub gate: CheckGate, + pub rebuilder: &'a Rebuilder, + /// what the emitted python needs that the standard library does not provide + pub requirements: &'a mut by_transforms::RuntimeRequirements, + /// where the runtime helpers go, and which copy each module imports. `None` + /// pastes the definitions into every module instead + pub runtime: Option<&'a mut RuntimeLayout>, + /// the project's module roots, longest first, and the project root + pub roots: &'a [PathBuf], + pub root: &'a Path, +} + /// Check every file, then for each non-blocked file call `consume` with the /// transpiled Python. pub fn check_and_transpile( db: &ProjectDatabase, handles: &[(PathBuf, ruff_db::files::File)], - config: &Config, - gate: CheckGate, - rebuilder: &Rebuilder, - requirements: &mut by_transforms::RuntimeRequirements, + emit: &mut Emit<'_>, mut consume: impl FnMut(&Transpiled<'_>) -> anyhow::Result<()>, ) -> anyhow::Result { + let Emit { + config, + gate, + rebuilder, + requirements, + runtime, + roots, + root, + } = emit; + let gate = *gate; let mut all_diagnostics: Vec = Vec::new(); let mut unusable: Vec = Vec::new(); @@ -148,7 +174,17 @@ pub fn check_and_transpile( if unusable.contains(file) { continue; } - match by_transforms::transpile_typed_with_report(db, *file, config, Some(&rebuild)) { + // the helpers this module calls come from a copy written beside it, and + // which copy depends on where it lands — so the config is settled per + // file rather than once for the build + let relative = transpiled_destination(roots, root, bpy); + let config = Config { + runtime_module: runtime + .as_deref_mut() + .and_then(|layout| layout.claim(&relative, bpy, ruff_db::Db::system(db))), + ..config.clone() + }; + match by_transforms::transpile_typed_with_report(db, *file, &config, Some(&rebuild)) { Ok((out, line_map, needed)) => { requirements.merge(needed); let by_source = source_text(db, *file); diff --git a/crates/by_stage/src/lib.rs b/crates/by_stage/src/lib.rs index 622bc11221..2442af103e 100644 --- a/crates/by_stage/src/lib.rs +++ b/crates/by_stage/src/lib.rs @@ -49,6 +49,7 @@ pub mod emit; pub mod project; pub mod record; pub mod restage; +pub mod runtime; pub mod sourcemap; pub mod staging; pub mod verbatim; diff --git a/crates/by_stage/src/restage.rs b/crates/by_stage/src/restage.rs index 28bfd8301e..23674a2448 100644 --- a/crates/by_stage/src/restage.rs +++ b/crates/by_stage/src/restage.rs @@ -33,9 +33,10 @@ use ruff_db::diagnostic::{ }; use ty_project::ProjectDatabase; -use crate::emit::{CheckGate, Transpiled, check_and_transpile}; +use crate::emit::{CheckGate, Emit, Transpiled, check_and_transpile}; use crate::project::{BY_SOURCES, Rebuilder, project_sources}; use crate::record::{BuildRecord, build_identity}; +use crate::runtime::RuntimeLayout; use crate::sourcemap::{ BY_SOURCEMAP_FILENAME, content_digest, describe_module, rewrite_sourcemap_entry, sourcemap_key_for, @@ -226,12 +227,22 @@ fn restage_transpiled( let outcome = check_and_transpile( db, std::slice::from_ref(&handle), - config, - // the gate `by run` uses: a program that fails `by check` must not run, and - // a module reloaded into a running one is that program continuing - CheckGate::AllErrors, - &Rebuilder::for_project(db), - &mut by_transforms::RuntimeRequirements::default(), + &mut Emit { + config, + // the gate `by run` uses: a program that fails `by check` must not + // run, and a module reloaded into a running one is that program + // continuing + gate: CheckGate::AllErrors, + rebuilder: &Rebuilder::for_project(db), + requirements: &mut by_transforms::RuntimeRequirements::default(), + // a re-stage writes one module back into a tree the build already + // laid out, so what it claims is thrown away: whatever copy of the + // runtime that module imports is already sitting where the build + // put it + runtime: Some(&mut RuntimeLayout::default()), + roots: &record.module_roots, + root: &record.project_root, + }, |emitted: &Transpiled<'_>| { produced = Some((describe_module(emitted), emitted.python.to_owned())); Ok(()) diff --git a/crates/by_stage/src/runtime.rs b/crates/by_stage/src/runtime.rs new file mode 100644 index 0000000000..6bfc67ae2f --- /dev/null +++ b/crates/by_stage/src/runtime.rs @@ -0,0 +1,210 @@ +//! where a build puts the runtime helpers, and which copy each module imports +//! +//! # one copy per package, not one at the root +//! +//! a distribution ships packages — directories with an `__init__` — so each +//! package carries a copy of its own. a copy at the tree root would be a +//! top-level module: a distribution that ships root modules would ship it as one, +//! and a second basedpython wheel, built by another version whose helpers differ, +//! overwrites it on install +//! +//! # a module in no package +//! +//! so a module at a module root gets the definitions pasted in. so does a module +//! in a directory that is not a package — a `scripts/` folder, a namespace +//! package — which has no import that works both when it is run as a script and +//! when it is imported +//! +//! # the import is absolute +//! +//! `from pkg._by_runtime import …` rather than `from ._by_runtime import …`: a +//! relative import fails in a module run as `__main__`, and a project's entry +//! point is exactly that + +use std::collections::BTreeSet; +use std::path::{Component, Path, PathBuf}; + +use ruff_db::system::{System, SystemPath}; + +/// the copies a build writes +/// +/// claimed for every module rather than only those that turned out to call a +/// helper: `by restage` recomputes one module against a tree an earlier build +/// wrote, and an edit that newly reaches for a helper has to find the copy there +#[derive(Debug, Default)] +pub struct RuntimeLayout { + /// each package holding a copy + packages: BTreeSet, +} + +impl RuntimeLayout { + /// record that `source`, landing at `relative` in the module tree, needs the + /// runtime, and answer the module it imports it from — or `None` for a module + /// in no package, which gets the definitions pasted in + pub(crate) fn claim( + &mut self, + relative: &Path, + source: &Path, + system: &dyn System, + ) -> Option { + let mut components = relative.components(); + let first = components.next()?; + // a `relative` of one component is the module's own file name: it sits at + // a module root, in no package + components.next()?; + let Component::Normal(package) = first else { + return None; + }; + let package = package.to_str()?; + if !is_package(&module_tree_base(relative, source).join(package), system) { + return None; + } + self.packages.insert(package.to_owned()); + Some(format!("{package}.{}", by_transforms::runtime::MODULE_NAME)) + } + + /// where each copy goes within the module tree + pub fn files(&self) -> impl Iterator + '_ { + self.packages + .iter() + .map(|package| Path::new(package).join(by_transforms::runtime::FILE_NAME)) + } +} + +/// the directory `relative` is relative to: `source` with as many trailing +/// components dropped as `relative` has. `/p/src/pkg/a.by` laid out at +/// `pkg/a.py` is rooted at `/p/src` +fn module_tree_base(relative: &Path, source: &Path) -> PathBuf { + source + .ancestors() + .nth(relative.components().count()) + .unwrap_or(Path::new("")) + .to_path_buf() +} + +/// whether `directory` is a regular package, the kind a distribution ships +fn is_package(directory: &Path, system: &dyn System) -> bool { + let Some(directory) = SystemPath::from_std_path(directory) else { + return false; + }; + ["__init__.py", "__init__.pyi", "__init__.by", "__init__.byi"] + .iter() + .any(|init| system.is_file(&directory.join(init))) +} + +#[cfg(test)] +mod tests { + use ruff_db::system::OsSystem; + + use super::*; + + struct Project { + root: tempfile::TempDir, + system: OsSystem, + } + + impl Project { + fn with(files: &[&str]) -> Self { + let root = tempfile::tempdir().expect("tempdir"); + for file in files { + let path = root.path().join(file); + std::fs::create_dir_all(path.parent().expect("a file has a parent")) + .expect("create"); + std::fs::write(&path, "").expect("write"); + } + let system = + OsSystem::new(SystemPath::from_std_path(root.path()).expect("utf-8 tempdir")); + Self { root, system } + } + + fn claim(&self, layout: &mut RuntimeLayout, relative: &str) -> Option { + let source = self + .root + .path() + .join("src") + .join(relative) + .with_extension("by"); + layout.claim(Path::new(relative), &source, &self.system) + } + } + + /// a copy at the root would be a top-level module of its own + #[test] + fn a_root_module_gets_the_definitions_pasted_in() { + let project = Project::with(&["src/main.by"]); + let mut layout = RuntimeLayout::default(); + assert_eq!(project.claim(&mut layout, "main.py"), None); + assert_eq!(layout.files().count(), 0); + } + + #[test] + fn a_module_in_a_package_imports_that_packages_copy() { + let project = Project::with(&["src/pkg/__init__.py", "src/pkg/a.by"]); + let mut layout = RuntimeLayout::default(); + assert_eq!( + project.claim(&mut layout, "pkg/a.py").as_deref(), + Some("pkg._by_runtime") + ); + assert_eq!( + layout.files().collect::>(), + vec![PathBuf::from("pkg/_by_runtime.py")] + ); + } + + /// a package whose `__init__` is itself transpiled is a package all the same + #[test] + fn a_by_init_makes_a_package() { + let project = Project::with(&["src/pkg/__init__.by", "src/pkg/a.by"]); + let mut layout = RuntimeLayout::default(); + assert_eq!( + project.claim(&mut layout, "pkg/a.py").as_deref(), + Some("pkg._by_runtime") + ); + } + + /// `scripts/tool.py` run as a script has `scripts/` on its path and not the + /// directory above, so `scripts._by_runtime` would not import + #[test] + fn a_module_in_no_package_gets_the_definitions_pasted_in() { + let project = Project::with(&["src/scripts/tool.by"]); + let mut layout = RuntimeLayout::default(); + assert_eq!(project.claim(&mut layout, "scripts/tool.py"), None); + assert_eq!(layout.files().count(), 0); + } + + /// a subpackage ships as part of the package above it + #[test] + fn a_subpackage_shares_the_packages_copy() { + let project = Project::with(&["src/pkg/__init__.py", "src/pkg/sub/deep/a.by"]); + let mut layout = RuntimeLayout::default(); + assert_eq!( + project.claim(&mut layout, "pkg/sub/deep/a.py").as_deref(), + Some("pkg._by_runtime") + ); + assert_eq!( + layout.files().collect::>(), + vec![PathBuf::from("pkg/_by_runtime.py")] + ); + } + + #[test] + fn each_package_carries_its_own() { + let project = Project::with(&["src/one/__init__.py", "src/two/__init__.py"]); + let mut layout = RuntimeLayout::default(); + project.claim(&mut layout, "one/a.py"); + project.claim(&mut layout, "two/b.py"); + project.claim(&mut layout, "one/c.py"); + assert_eq!( + layout.files().collect::>(), + vec![ + PathBuf::from("one/_by_runtime.py"), + PathBuf::from("two/_by_runtime.py") + ] + ); + } + + #[test] + fn an_unclaimed_layout_writes_nothing() { + assert_eq!(RuntimeLayout::default().files().count(), 0); + } +} diff --git a/crates/by_transforms/src/config.rs b/crates/by_transforms/src/config.rs index 6afbf34589..c3fa6db20e 100644 --- a/crates/by_transforms/src/config.rs +++ b/crates/by_transforms/src/config.rs @@ -71,6 +71,11 @@ pub struct Config { /// how a float or complex literal written in a type position is spelled in /// the emitted python. see [`FloatLiteralLowering`] pub float_literals: FloatLiteralLowering, + /// the module the emitted code imports its runtime helpers from, or `None` + /// to paste the definitions in. a build names the package-qualified copy it + /// writes into its tree; a transpile with nowhere to write one leaves it + /// `None` + pub runtime_module: Option, } /// What a float or complex literal type becomes in the emitted python. @@ -192,6 +197,7 @@ impl Default for Config { runtime_raises_checks: false, unique_loop_bindings: true, float_literals: FloatLiteralLowering::Nominal, + runtime_module: None, stamps: BTreeMap::new(), } } diff --git a/crates/by_transforms/src/lib.rs b/crates/by_transforms/src/lib.rs index a6e815d9a9..62b2f4bac4 100644 --- a/crates/by_transforms/src/lib.rs +++ b/crates/by_transforms/src/lib.rs @@ -1,5 +1,6 @@ pub mod config; mod reverse_transforms; +pub mod runtime; pub(crate) mod source_map; mod transforms; pub(crate) mod type_info; @@ -109,6 +110,11 @@ fn run_erased_union_phase<'a>( file: File, config: &Config, ) -> std::borrow::Cow<'a, str> { + // the reified parameter carries a call's specialization into the body, and a + // stub has no body to carry it to. its declaration keeps the union written + if config.is_stub { + return std::borrow::Cow::Borrowed(source); + } let parsed = ruff_db::parsed::parsed_module(db, db.program_file(file).python_file(db)).load(db); if !parsed.errors().is_empty() { return std::borrow::Cow::Borrowed(source); @@ -138,6 +144,16 @@ fn transpile_with_report( // and — as long as nothing rewrites the source — by phase 0's type-aware // passes, which would otherwise build an identical one of their own let (local_db, local_file) = make_in_memory_db(source); + // what the author wrote, before any rewrite, so phase 3 can tell a helper + // call the transpiler emitted from a name the program reads itself + let written = names_written( + ruff_db::parsed::parsed_module( + &local_db, + ty_python_semantic::Db::program_file(&local_db, local_file).python_file(&local_db), + ) + .load(&local_db) + .suite(), + ); // --- Erased-union reification: give a `list[int] | list[str]` parameter a // reified type parameter, while the source is still the one ty checks --- @@ -162,7 +178,7 @@ fn transpile_with_report( // --- Enum lowering: rewrite `enum` sum types to Python before the main // pipeline, so member bodies (copied verbatim) are lowered downstream --- - let enum_lowered = transforms::enums::lower(source, config.min_version); + let enum_lowered = transforms::enums::lower(source, config); if let Some(first) = enum_lowered.errors.first() { return Err(first.clone()); } @@ -215,7 +231,7 @@ fn transpile_with_report( let final_output = run_version_polyfill_phase(final_output, config); // --- Phase 3: syntax verification --- - verify_syntax(&final_output).map_err(|e| e.message)?; + verify_syntax(&final_output, &written).map_err(|e| e.message)?; verify_target_syntax(&final_output, config).map_err(|e| e.message)?; Ok((final_output, requirements)) @@ -309,6 +325,13 @@ pub fn transpile_typed_with_report( config: &Config, rebuild: Option>, ) -> Result<(String, Vec>, RuntimeRequirements), TranspileError> { + // whether the output is a stub is a fact about the file rather than about + // the command that asked for it. `by build` hands every source it stages + // the one config, so a flag each caller had to set would go unset there + let config = &Config { + is_stub: file.source_type(db).is_stub(), + ..config.clone() + }; let source_ref = ruff_db::source::source_text(db, file); let original_source = source_ref.as_str(); @@ -321,6 +344,15 @@ pub fn transpile_typed_with_report( )); } + let written = names_written( + ruff_db::parsed::parsed_module( + db, + ty_python_semantic::Db::program_file(db, file).python_file(db), + ) + .load(db) + .suite(), + ); + // erased-union reification: give a `list[int] | list[str]` parameter a // reified type parameter, against the source ty checks. edits stay inside // the def header and the annotations, so line correspondence is unaffected @@ -347,7 +379,7 @@ pub fn transpile_typed_with_report( // enum lowering: rewrite `enum` sum types to Python first. when it fires, // the working source differs from the project file, so type-aware passes // and the final lowering run against a single-file db built from it - let enum_lowered = transforms::enums::lower(qualified.as_ref(), config.min_version); + let enum_lowered = transforms::enums::lower(qualified.as_ref(), config); if let Some(first) = enum_lowered.errors.first() { return Err(first.clone().into()); } @@ -466,8 +498,8 @@ pub fn transpile_typed_with_report( line_map.extend(composed[kept..].iter().copied()); // verify last: on failure, map the generated span back to a `.by` range - let verified = - verify_syntax(&final_output).and_then(|()| verify_target_syntax(&final_output, config)); + let verified = verify_syntax(&final_output, &written) + .and_then(|()| verify_target_syntax(&final_output, config)); if let Err(mut err) = verified { err.by_range = err.output_range.and_then(|r| { output_offset_to_by_range(&line_map, &final_output, original_source, r.start()) @@ -544,8 +576,9 @@ fn run_anon_named_tuple_cleanup(mut source: String, config: &Config) -> Result (String, Runtim ( output, RuntimeRequirements { - typing_extensions: true, + // a stub is never imported, and the checker that reads it brings + // its own `typing_extensions` + typing_extensions: !config.is_stub, }, ) } @@ -617,6 +652,9 @@ fn run_import_redirect_phase(source: String, config: &Config) -> (String, Runtim /// `eager_names` names the *bindings* that must be bound to the real object: a /// lazy proxy cannot stand where cpython checks for a real class, which is what /// `except` does +/// +/// A stub defers nothing: it is never executed, and deferring its imports would +/// hand a checker a call result where the stub declares a module or a class fn run_lazy_import_phase( source: String, config: &Config, @@ -636,9 +674,18 @@ fn run_lazy_import_phase( ) .load(&db); - let keyword_supported = config.min_version >= ruff_python_ast::PythonVersion::from((3, 15)); - let mut lazy = - transforms::lazy_import::LazyImport::new(src, keyword_supported, eager, eager_names); + let deferral = if config.is_stub { + transforms::lazy_import::Deferral::Never + } else if config.min_version >= ruff_python_ast::PythonVersion::from((3, 15)) { + transforms::lazy_import::Deferral::Keyword + } else { + transforms::lazy_import::Deferral::Polyfill + }; + // the runtime is what the polyfill itself runs on, and a helper reached + // through a proxy would be a proxy call on every use + let mut eager = eager.to_vec(); + eager.extend(config.runtime_module.clone()); + let mut lazy = transforms::lazy_import::LazyImport::new(src, deferral, &eager, eager_names); for stmt in module.suite() { lazy.visit_stmt(stmt); } @@ -647,16 +694,17 @@ fn run_lazy_import_phase( let needs_ty_ext = lazy.needs_ty_ext_marker; let needs_character_class = lazy.needs_character_class; - let preamble = transforms::lazy_import::polyfill_preamble( + let helpers = transforms::lazy_import::polyfill_helpers( needs_module, needs_attr, needs_ty_ext, needs_character_class, ); - if lazy.edits.is_empty() && preamble.is_empty() { + if lazy.edits.is_empty() && helpers.is_empty() { return source; } + let preamble = runtime_preamble(config, &helpers); let (body, _) = apply_transforms_once(src, lazy.edits); if preamble.is_empty() { body @@ -665,6 +713,31 @@ fn run_lazy_import_phase( } } +/// The lines that give a module the runtime helpers it calls: an import of the +/// module a build wrote them to, or the definitions themselves when this +/// transpile has nowhere to write one. +/// +/// Both come out of `_by_runtime.py`, so the pasted-in form and the imported one +/// are the same code either way. +fn runtime_preamble(config: &Config, helpers: &[runtime::Helper]) -> String { + runtime_entries(config, helpers).concat() +} + +/// The same lines, one per entry and each newline-terminated, for a caller that +/// prepends them one at a time. +fn runtime_entries(config: &Config, helpers: &[runtime::Helper]) -> Vec { + if helpers.is_empty() { + return Vec::new(); + } + match config.runtime_module.as_deref() { + Some(module) => vec![format!( + "{}\n", + runtime::import_line(module, helpers.iter().copied()) + )], + None => runtime::inline(helpers.iter().copied()), + } +} + /// Version polyfill phase: rewrite syntax the target python cannot parse into /// syntax it can. Runs over the finished python rather than over `.by`, so a /// `match` an earlier lowering *generated* — for a `let` destructuring, an @@ -676,7 +749,7 @@ fn run_lazy_import_phase( /// lines it adds are the runtime preamble's, at the top, where the line map /// already accounts for generated leading lines. fn run_version_polyfill_phase(source: String, config: &Config) -> String { - transforms::match_polyfill::lower(source, config.min_version) + transforms::match_polyfill::lower(source, config) } /// Re-parse the transpiled output *as the target python version* and report any @@ -764,7 +837,7 @@ impl From for TranspileError { } } -fn verify_syntax(source: &str) -> Result<(), TranspileError> { +fn verify_syntax(source: &str, written: &HashSet) -> Result<(), TranspileError> { use ruff_python_ast::{PySourceType, visitor::Visitor}; let parsed = ruff_python_parser::parse_unchecked_source(source, PySourceType::Python); @@ -852,7 +925,84 @@ fn verify_syntax(source: &str) -> Result<(), TranspileError> { }); } - Ok(()) + verify_runtime_helpers(parsed.suite(), written) +} + +/// every name the author's own source reads +fn names_written(suite: &[Stmt]) -> HashSet { + use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr, walk_stmt}; + + struct Reads(HashSet); + impl SourceOrderVisitor<'_> for Reads { + fn visit_expr(&mut self, expr: &ruff_python_ast::Expr) { + if let ruff_python_ast::Expr::Name(name) = expr + && name.ctx.is_load() + { + self.0.insert(name.id.to_string()); + } + walk_expr(self, expr); + } + } + let mut reads = Reads(HashSet::new()); + for stmt in suite { + walk_stmt(&mut reads, stmt); + } + reads.0 +} + +/// reject output that calls a runtime helper the module was never given +/// +/// a transform emits a call and records the helper it needs in two different +/// places. forgetting the second half produces python that parses, checks, and +/// raises `NameError` the first time the lowered line runs +/// +/// only a name the author's source never reads is put down to the transpiler: +/// one the author wrote is theirs to have bound, however they bound it. the +/// runtime is only ever provided at module scope, as an import or a pasted +/// definition, so that is where the name has to be bound +fn verify_runtime_helpers(suite: &[Stmt], written: &HashSet) -> Result<(), TranspileError> { + use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr, walk_stmt}; + + struct Emitted<'a> { + written: &'a HashSet, + reads: Vec<(String, TextRange)>, + } + impl SourceOrderVisitor<'_> for Emitted<'_> { + fn visit_expr(&mut self, expr: &ruff_python_ast::Expr) { + if let ruff_python_ast::Expr::Name(name) = expr + && name.ctx.is_load() + && runtime::defines(name.id.as_str()) + && !self.written.contains(name.id.as_str()) + { + self.reads.push((name.id.to_string(), Ranged::range(name))); + } + walk_expr(self, expr); + } + } + + let provided: HashSet = suite.iter().flat_map(runtime::bindings).collect(); + let mut emitted = Emitted { + written, + reads: Vec::new(), + }; + for stmt in suite { + walk_stmt(&mut emitted, stmt); + } + match emitted + .reads + .into_iter() + .find(|(name, _)| !provided.contains(name)) + { + Some((name, range)) => Err(TranspileError { + message: format!( + "transpiler emitted a call to the runtime helper `{name}` without asking for \ + it, so the module it produced does not define it" + ), + output_range: Some(range), + by_range: None, + }), + None => Ok(()), + } } /// The first `match` in `suite` that python's own parse-time checks reject. @@ -999,7 +1149,11 @@ pub fn optional_marker_edits(source: &str) -> Vec<(TextRange, String)> { /// round-trip testing — `transpile(reverse_transpile(py))` should produce /// AST-equivalent code to `transpile(py)`. pub fn reverse_transpile(source: &str, config: &Config) -> Result { - let (db, file) = make_in_memory_db(source); + // every transform below keys its edits on one parse, so the annotation + // strings are unquoted first: the transforms then see `A | None` where + // the source wrote `"A | None"` + let source = reverse_transforms::forward_references::unquote_forward_references(source); + let (db, file) = make_in_memory_db(&source); let source_ref = ruff_db::source::source_text(&db, file); let src = source_ref.as_str(); let module = ruff_db::parsed::parsed_module( @@ -1557,7 +1711,7 @@ mod transpile_error { // nested in a function, which the scan has to reach "def f(x):\n match x:\n case a | b:\n pass\n", ] { - let err = verify_syntax(source).unwrap_err(); + let err = verify_syntax(source, &std::collections::HashSet::new()).unwrap_err(); assert!( err.message .starts_with("transpiler produced invalid Python:"), @@ -1573,13 +1727,15 @@ mod transpile_error { fn verify_syntax_accepts_a_qualified_match() { verify_syntax( "match x:\n case Color.Red | Color.Green:\n pass\n case Color.Blue:\n pass\n", + &std::collections::HashSet::new(), ) .unwrap(); } #[test] fn verify_syntax_message_has_no_byte_range() { - let err = verify_syntax("def f(:\n pass\n").unwrap_err(); + let err = + verify_syntax("def f(:\n pass\n", &std::collections::HashSet::new()).unwrap_err(); assert!( !err.message.contains("byte range"), "message must not leak internal byte ranges: {}", @@ -2326,3 +2482,61 @@ mod cross_file { } } } + +#[cfg(test)] +mod runtime_helper_check { + use std::collections::HashSet; + + use super::{Config, transpile, verify_syntax}; + + fn verify(output: &str, written: &[&str]) -> Result<(), String> { + let written: HashSet = written.iter().map(|name| (*name).to_owned()).collect(); + verify_syntax(output, &written).map_err(|error| error.message) + } + + #[test] + fn a_helper_call_nothing_provides_is_rejected() { + let error = verify("x = _lazy_module(\"os\")\n", &[]).unwrap_err(); + assert!(error.contains("`_lazy_module`"), "{error}"); + } + + /// the runtime is only ever provided at module scope, so a binding inside a + /// function provides nothing to a call outside it + #[test] + fn a_binding_inside_a_function_provides_nothing() { + let error = verify( + "def f():\n _lazy_module = 1\nx = _lazy_module(\"os\")\n", + &[], + ) + .unwrap_err(); + assert!(error.contains("`_lazy_module`"), "{error}"); + } + + #[test] + fn an_import_provides_the_helper() { + verify( + "from app._by_runtime import _lazy_module\nx = _lazy_module(\"os\")\n", + &[], + ) + .unwrap(); + } + + /// a helper whose name the program could share is checked all the same when + /// the author never wrote it + #[test] + fn a_public_helper_is_checked() { + assert!(verify("x = Optional(1)\n", &[]).is_err()); + verify("x = Optional(1)\n", &["Optional"]).unwrap(); + } + + /// a name the author wrote is theirs, however they bound it — here through a + /// star import nothing can see into + #[test] + fn a_helper_name_the_author_reads_is_theirs() { + transpile( + "from helpers import *\n\nprint(_by_alias(1))\n", + &Config::test_default(), + ) + .unwrap(); + } +} diff --git a/crates/by_transforms/src/reverse_transforms/auto_quote.rs b/crates/by_transforms/src/reverse_transforms/auto_quote.rs index f109775dea..a7056c46f8 100644 --- a/crates/by_transforms/src/reverse_transforms/auto_quote.rs +++ b/crates/by_transforms/src/reverse_transforms/auto_quote.rs @@ -1,8 +1,12 @@ -//! reverse of `crate::transforms::auto_quote`: -//! `"ClassName"` string in subscript slice → bare name within class definition +//! reverse of `crate::transforms::auto_quote` for the positions that are not +//! annotations: +//! `"ClassName"` string in a base's subscript slice, or in a value-position +//! subscript in the class body (`list["A"]()`) → bare name //! -//! mirrors the forward transform's traversal exactly: only fires inside class -//! base-class subscripts and class body annotation positions +//! a base and a value subscript evaluate while the class is being built, so the +//! forward transpile quotes only the class's own name there, and only that is +//! unquoted: any other string stays one. annotation strings are unquoted before +//! any reverse transform runs, by [`super::forward_references`] use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::visitor::{Visitor, walk_stmt}; @@ -89,31 +93,10 @@ impl AutoQuoteReverse { Stmt::Expr(e) => self.walk_expr_for_subscripts(&e.value, class_name), Stmt::Assign(a) => self.walk_expr_for_subscripts(&a.value, class_name), Stmt::AnnAssign(a) => { - self.find_quoted_refs_in_annotation(&a.annotation, class_name); if let Some(value) = &a.value { self.walk_expr_for_subscripts(value, class_name); } } - Stmt::FunctionDef(f) => { - for param in f.parameters.iter_non_variadic_params() { - if let Some(ann) = ¶m.parameter.annotation { - self.find_quoted_refs_in_annotation(ann, class_name); - } - } - if let Some(var) = &f.parameters.vararg { - if let Some(ann) = &var.annotation { - self.find_quoted_refs_in_annotation(ann, class_name); - } - } - if let Some(kwarg) = &f.parameters.kwarg { - if let Some(ann) = &kwarg.annotation { - self.find_quoted_refs_in_annotation(ann, class_name); - } - } - if let Some(ret) = &f.returns { - self.find_quoted_refs_in_annotation(ret, class_name); - } - } _ => {} } } @@ -182,7 +165,7 @@ mod tests { "}, indoc! {" class A(list[A]): - def method(self, x: list[A]) -> list[A] + def method(self, x: list[A]) -> list[A]: ... "}, ); } diff --git a/crates/by_transforms/src/reverse_transforms/decorated_type.rs b/crates/by_transforms/src/reverse_transforms/decorated_type.rs index db64a179d3..6ab593db7e 100644 --- a/crates/by_transforms/src/reverse_transforms/decorated_type.rs +++ b/crates/by_transforms/src/reverse_transforms/decorated_type.rs @@ -212,7 +212,7 @@ mod tests { "}, indoc! {" from typing import Annotated - def field(gt: int) -> int + def field(gt: int) -> int: ... x: @field(gt=0) int "}, ); diff --git a/crates/by_transforms/src/reverse_transforms/dynamic_keyword.rs b/crates/by_transforms/src/reverse_transforms/dynamic_keyword.rs index bf51e965b3..65fb46da44 100644 --- a/crates/by_transforms/src/reverse_transforms/dynamic_keyword.rs +++ b/crates/by_transforms/src/reverse_transforms/dynamic_keyword.rs @@ -99,7 +99,7 @@ mod tests { "}, indoc! {" from typing import Any - def f() -> dynamic + def f() -> dynamic: ... "}, ); } @@ -137,7 +137,7 @@ mod tests { "}, indoc! {" from typing import Any - def f(x: dynamic) -> None + def f(x: dynamic) -> None: ... "}, ); } diff --git a/crates/by_transforms/src/reverse_transforms/empty_declarations.rs b/crates/by_transforms/src/reverse_transforms/empty_declarations.rs index 71d9e48f13..0429e447f4 100644 --- a/crates/by_transforms/src/reverse_transforms/empty_declarations.rs +++ b/crates/by_transforms/src/reverse_transforms/empty_declarations.rs @@ -2,7 +2,7 @@ //! handling in `crate::transforms::overload`: //! //! `class Foo: ...` → `class Foo` -//! `def f(x: int) -> int: ...` → `def f(x: int) -> int` +//! `def f(x: int) -> int: ...` → `def f(x: int) -> int` (in a stub) //! //! Only fires when the body is exactly a single ellipsis expression statement, //! which is what the forward transforms emit. `class Foo: pass` / @@ -10,9 +10,14 @@ //! alone — they're not what the forward produces and rewriting them would lose //! author intent. //! -//! Function defs are skipped if any decorator is attached, since stripping -//! `: ...` from e.g. `@overload def f(...): ...` would leave the decorator -//! orphaned (the overload-reverse pass handles those groups itself). +//! A `def` only loses its body in a stub. A class with no body is a whole class +//! wherever it is written, but a `def` with no body is a declaration, and +//! `missing-function-body` reports one written where the position asks for an +//! implementation. This pass cannot tell those positions apart, so outside a +//! stub the `: ...` the author wrote stays — valid basedpython either way. +//! +//! An `@overload`-decorated `def` is left to the overload-reverse pass, which +//! strips the decorator and the body together wherever the group is written. use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::visitor::{Visitor, walk_body, walk_stmt}; @@ -20,11 +25,8 @@ use ruff_python_ast::{Expr, Stmt, StmtClassDef, StmtFunctionDef}; use ruff_text_size::{Ranged, TextRange, TextSize}; pub(crate) struct EmptyDeclarations { - /// when reversing a non-stub `.py`, an abstract method keeps its `: ...` - /// body: the forward pass maps a bodyless `abstract def` to `: raise - /// NotImplementedError`, so stripping the body would not round-trip. in a - /// stub the body is dropped — bodyless is the stub idiom and the forward - /// pass re-emits `: ...` there + /// whether a `def` may be left bodyless: only in a stub, where declaring a + /// signature is the point and the forward pass re-emits `: ...` is_stub: bool, pub(crate) edits: Vec, } @@ -37,14 +39,6 @@ impl EmptyDeclarations { } } - fn is_abstract(func: &StmtFunctionDef) -> bool { - func.decorator_list.iter().any(|d| match &d.expression { - Expr::Name(n) => n.id.as_str() == "abstractmethod", - Expr::Attribute(a) => a.attr.id.as_str() == "abstractmethod", - _ => false, - }) - } - fn is_ellipsis_body(body: &[Stmt]) -> bool { matches!( body, @@ -75,6 +69,9 @@ impl EmptyDeclarations { } fn process_function(&mut self, func: &StmtFunctionDef) { + if !self.is_stub { + return; + } // `@overload`-decorated functions belong to the overload reverse pass, // which strips the decorator and the `: ...` body together. other // decorators (`@property`, `@deprecated`, modifier-backed ones like @@ -87,12 +84,6 @@ impl EmptyDeclarations { { return; } - // outside a stub, an abstract method's `: ...` body must survive: the - // forward pass turns a bodyless `abstract def` into `: raise - // NotImplementedError`, so dropping it here would not round-trip - if !self.is_stub && Self::is_abstract(func) { - return; - } if !Self::is_ellipsis_body(&func.body) { return; } @@ -159,6 +150,14 @@ mod tests { ); } + fn check_stub(input: &str, expected: &str) { + let config = Config { + is_stub: true, + ..Config::test_default() + }; + assert_eq!(reverse_transpile(input, &config).unwrap(), expected); + } + #[test] fn single_line_ellipsis() { check("class Foo: ...\n", "class Foo\n"); @@ -247,18 +246,18 @@ mod tests { } #[test] - fn single_empty_function() { - check("def f(a: int) -> int: ...\n", "def f(a: int) -> int\n"); + fn single_empty_function_in_stub() { + check_stub("def f(a: int) -> int: ...\n", "def f(a: int) -> int\n"); } #[test] - fn empty_function_no_return_type() { - check("def f(): ...\n", "def f()\n"); + fn empty_function_no_return_type_in_stub() { + check_stub("def f(): ...\n", "def f()\n"); } #[test] - fn empty_function_multiline_ellipsis() { - check( + fn empty_function_multiline_ellipsis_in_stub() { + check_stub( indoc! {" def f(a: int) -> int: ... @@ -267,6 +266,21 @@ mod tests { ); } + #[test] + fn empty_function_keeps_its_body_outside_a_stub() { + // a `def` with no body declares a signature, which is what a stub is for. outside one + // the position may well need an implementation, and nothing here can tell — so what + // the author wrote stays + check("def f(a: int) -> int: ...\n", "def f(a: int) -> int: ...\n"); + check("def f(): ...\n", "def f(): ...\n"); + } + + #[test] + fn empty_class_loses_its_body_outside_a_stub() { + // a class with no members is a whole class, so this one is not held back + check("class Foo: ...\n", "class Foo\n"); + } + #[test] fn function_with_pass_unchanged() { check( @@ -282,10 +296,10 @@ mod tests { } #[test] - fn property_decorated_function_stripped() { + fn property_decorated_function_stripped_in_stub() { // non-`@overload` decorators don't defer to the overload pass; the // `: ...` body is stripped and the decorator survives in front - check( + check_stub( indoc! {" class A: @property @@ -300,10 +314,10 @@ mod tests { } #[test] - fn abstract_function_keeps_body_in_non_stub() { - // non-stub: `@abstractmethod` reverses to `abstract` but the `: ...` - // body is kept — a bodyless `abstract def` forward-maps to `: raise - // NotImplementedError`, so stripping would not round-trip + fn abstract_function_keeps_body_outside_a_stub() { + // `@abstractmethod` reverses to the `abstract` modifier, and the `: ...` body stays + // with it: a bodyless `abstract def` forward-maps to `: raise NotImplementedError`, + // so stripping it would not round-trip even where a declaration is allowed check( indoc! {" from abc import abstractmethod @@ -323,21 +337,13 @@ mod tests { fn abstract_function_stripped_in_stub() { // stub: bodyless is the idiom and the forward pass re-emits `: ...` // for an abstract method in a stub, so the body is dropped here - let config = Config { - is_stub: true, - ..Config::test_default() - }; - assert_eq!( - reverse_transpile( - indoc! {" - from abc import abstractmethod - class A: - @abstractmethod - def f(self) -> None: ... - "}, - &config, - ) - .unwrap(), + check_stub( + indoc! {" + from abc import abstractmethod + class A: + @abstractmethod + def f(self) -> None: ... + "}, indoc! {" from abc import abstractmethod class A: diff --git a/crates/by_transforms/src/reverse_transforms/forward_references.rs b/crates/by_transforms/src/reverse_transforms/forward_references.rs new file mode 100644 index 0000000000..0c10f7cdd6 --- /dev/null +++ b/crates/by_transforms/src/reverse_transforms/forward_references.rs @@ -0,0 +1,253 @@ +//! reverse of the forward-reference quoting in [`crate::transforms::auto_quote`] +//! +//! a string annotation is a forward reference in python, and in basedpython it +//! would be a string-literal *type*: `def f() -> "Plain"` reads as returning the +//! string `"Plain"`, and transpiles back as `Literal["Plain"]`. basedpython +//! resolves every annotation as deferred and the forward transpile quotes each +//! reference that needs it, so the reverse writes out the expression the string +//! spells: `-> Plain` +//! +//! this runs over the source *before* the other reverse transforms, whose edits +//! all key on one parse of it. unquoted first, `"Plain | None"` is a real +//! union by the time the optional transform looks for one to write `Plain?`. a string +//! inside the unquoted text is a forward reference too, and is unquoted the same +//! way +//! +//! the positions are the annotation positions: parameters, returns, variables +//! (local ones included — the reader of a `.by` file reads them all the same), +//! `TypeAlias` values and pep 695 bounds and defaults. inside one, the +//! arguments of `Literal[…]` and the metadata of `Annotated[…]` are values, and +//! stay strings. a string whose text is not a single expression, or is a bare +//! tuple — which means something else once it is not a string — is left as it +//! is + +use ruff_python_ast::Expr; +use ruff_python_parser::{parse_expression, parse_module}; +use ruff_text_size::{Ranged, TextRange}; + +use crate::transforms::source_util::for_each_annotation_in_stmt; +use crate::transforms::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_one_type_expr}; + +/// `source` with each string annotation replaced by the expression it spells +pub(crate) fn unquote_forward_references(source: &str) -> String { + let Ok(parsed) = parse_module(source) else { + return source.to_owned(); + }; + let mut collector = Collector { edits: Vec::new() }; + for stmt in parsed.suite() { + for_each_annotation_in_stmt(stmt, |annotation| { + walk_one_type_expr(annotation, &mut collector); + }); + } + splice(source, collector.edits) +} + +struct Collector { + edits: Vec<(TextRange, String)>, +} + +impl TypeExprVisitor for Collector { + fn visit(&mut self, expr: &Expr, _pos: TypePos) -> Recurse { + let Expr::StringLiteral(string) = expr else { + return Recurse::Descend; + }; + // an implicitly concatenated string is one value spread over several + // literals, and is left as the author wrote it + if !string.value.is_implicit_concatenated() + && let Some(expression) = spelled_expression(string.value.to_str()) + { + self.edits.push((expr.range(), expression)); + } + Recurse::Stop + } +} + +/// the expression a forward reference's text spells, with the forward +/// references inside it unquoted too, or `None` when it is not one the +/// annotation can hold in its place +fn spelled_expression(text: &str) -> Option { + let text = text.trim(); + let parsed = parse_expression(text).ok()?; + let expression = parsed.expr(); + if let Expr::Tuple(tuple) = expression + && !tuple.parenthesized + { + return None; + } + let mut collector = Collector { edits: Vec::new() }; + walk_one_type_expr(expression, &mut collector); + let unquoted = splice(text, collector.edits); + // the string stood as one operand wherever it was written. a name, a + // generic, an attribute or a union keeps that shape without help; anything + // looser, or spread over lines, is parenthesized to hold together + let holds_together = matches!( + expression, + Expr::Name(_) + | Expr::Attribute(_) + | Expr::Subscript(_) + | Expr::NoneLiteral(_) + | Expr::EllipsisLiteral(_) + | Expr::List(_) + | Expr::Tuple(_) + ) || matches!(expression, Expr::BinOp(binop) if binop.op.is_bit_or()); + Some(if holds_together && !unquoted.contains('\n') { + unquoted + } else { + format!("({unquoted})") + }) +} + +fn splice(source: &str, mut edits: Vec<(TextRange, String)>) -> String { + edits.sort_by_key(|(range, _)| range.start()); + let mut out = String::with_capacity(source.len()); + let mut cursor = 0usize; + for (range, text) in edits { + let (start, end) = (usize::from(range.start()), usize::from(range.end())); + if start < cursor { + continue; + } + out.push_str(&source[cursor..start]); + out.push_str(&text); + cursor = end; + } + out.push_str(&source[cursor..]); + out +} + +#[cfg(test)] +mod tests { + use crate::{Config, reverse_transpile, transpile}; + use indoc::indoc; + + fn check(input: &str, expected: &str) { + assert_eq!( + reverse_transpile(input, &Config::test_default()).unwrap(), + expected + ); + } + + #[test] + fn a_string_annotation_is_the_expression_it_spells() { + check( + indoc! {" + def later() -> \"Later\": + return Later() + + + class Later: ... + "}, + indoc! {" + def later() -> Later: + return Later() + + + class Later + "}, + ); + } + + #[test] + fn a_self_reference_in_a_method_signature() { + check( + indoc! {" + class Plain: + def m(self, other: \"Plain\") -> \"Plain\": + return self + "}, + indoc! {" + class Plain: + def m(self, other: Plain) -> Plain: + return self + "}, + ); + } + + #[test] + fn a_string_inside_a_generic() { + check("x: list[\"Later\"] = []\n", "x: list[Later] = []\n"); + } + + /// the text of a string annotation can hold forward references of its own + #[test] + fn a_string_inside_the_unquoted_text() { + check( + "x: \"dict[str, 'Later']\" = {}\n", + "x: dict[str, Later] = {}\n", + ); + } + + /// unquoted before the other transforms run, the text is a real annotation + /// by the time they look at it + #[test] + fn the_unquoted_text_is_reversed_as_well() { + check( + indoc! {" + def f(x: \"Later | None\") -> None: ... + + + class Later: ... + "}, + indoc! {" + def f(x: Later?) -> None: ... + + + class Later + "}, + ); + } + + /// `Literal`'s arguments and `Annotated`'s metadata are values, not types + #[test] + fn literal_arguments_and_annotated_metadata_stay_strings() { + check( + indoc! {" + from typing import Annotated + + + x: Annotated[int, \"doc\"] = 1 + "}, + indoc! {" + from typing import Annotated + + + x: Annotated[int, \"doc\"] = 1 + "}, + ); + } + + #[test] + fn a_local_variable_annotation() { + check( + indoc! {" + def f() -> None: + x: \"Later\" = Later() + "}, + indoc! {" + def f() -> None: + x: Later = Later() + "}, + ); + } + + /// a bare tuple means something else once it is not a string + #[test] + fn a_bare_tuple_stays_a_string() { + check("x: \"int, str\" = 1\n", "x: \"int, str\" = 1\n"); + } + + /// the forward transpile quotes whatever the reverse unquoted wherever + /// python evaluates annotations as the definition runs + #[test] + fn the_reference_is_quoted_again_on_the_way_back() { + let python = indoc! {" + def later() -> \"Later\": + return Later() + + + class Later: ... + "}; + let by = reverse_transpile(python, &Config::test_default()).unwrap(); + let back = transpile(&by, &Config::test_default()).unwrap(); + assert!(back.contains("def later() -> \"Later\":"), "{back}"); + } +} diff --git a/crates/by_transforms/src/reverse_transforms/identity_swap.rs b/crates/by_transforms/src/reverse_transforms/identity_swap.rs index 23b9177827..172e00a453 100644 --- a/crates/by_transforms/src/reverse_transforms/identity_swap.rs +++ b/crates/by_transforms/src/reverse_transforms/identity_swap.rs @@ -1,40 +1,29 @@ //! reverse of `crate::transforms::identity_swap`: -//! `x is y` → `x === y` -//! `x is not y` → `x !== y` -//! `isinstance(x, y)` → `x is y` -//! `not isinstance(x, y)` → `x is not y` +//! `x is y` → `x === y` +//! `x is not y` → `x !== y` //! -//! basedpython's `is` is the instance check, so a python identity comparison -//! round-trips to `===` / `!==` and an `isinstance` call round-trips to `is` +//! basedpython gives the `is` keyword to the type test and spells identity +//! `===`, so a python identity comparison round-trips to `===` / `!==`. that +//! includes `x is None`: basedpython reads `x is None` as a test for the type +//! `None`, and a type test the value's static type settles is emitted as its +//! answer. python's comparison runs whatever the annotations say — `x is None` +//! on a parameter annotated `int` is how python code defends against a caller +//! the annotation does not bind — so only the identity spelling keeps it a check +//! that runs. leaving a python `is not` in place would also re-read it as +//! `not isinstance(...)` on the way back out //! -//! `x is None` is left alone: basedpython reads it as a type test against the -//! type `None`, which is the same runtime check, so rewriting it to -//! `x === None` would churn idiomatic source for no change in meaning. Every -//! *other* literal must still be rewritten — `x is 3` is identity in python but -//! a test for the type `Literal[3]` in basedpython, and those differ for an int -//! outside the interned range. -//! -//! This is also why the operator has to be rewritten rather than skipped: -//! leaving a python `is not` in place re-reads it as a type test on the way -//! back out. -//! -//! An `isinstance` call is only rewritten when its second argument is -//! something a type expression can say. A tuple of classes is `isinstance`'s -//! own spelling for a union and reads as a *tuple type* in a type expression, -//! so it is written back out as `A | B`; anything else — `type(y)`, a variable -//! holding classinfo — keeps the call it was, which basedpython runs just as -//! python does +//! an `isinstance` call stays a call for the same reason. `x is int` is the +//! idiomatic basedpython test, and it folds wherever `x`'s type already decides +//! it, which would erase the validation a python `isinstance` guard exists to +//! perform. basedpython runs the call just as python does use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{CmpOp, Expr, Stmt, UnaryOp}; +use ruff_python_ast::{CmpOp, Expr, Stmt}; use ruff_text_size::{Ranged, TextRange, TextSize}; pub(crate) struct IdentitySwapReverse<'src> { source: &'src str, - /// `isinstance` calls already folded into an enclosing `not`, so the call - /// itself must not also be rewritten into an overlapping edit - folded_into_not: Vec, pub(crate) edits: Vec, } @@ -42,31 +31,22 @@ impl<'src> IdentitySwapReverse<'src> { pub(crate) fn new(source: &'src str) -> Self { Self { source, - folded_into_not: Vec::new(), edits: Vec::new(), } } - fn src(&self, range: TextRange) -> &str { - &self.source[usize::from(range.start())..usize::from(range.end())] - } - fn process_compare(&mut self, c: &ruff_python_ast::ExprCompare) { let mut lhs_end = c.left.range().end(); for (op, rhs) in c.ops.iter().zip(c.comparators.iter()) { let rhs_start = rhs.range().start(); let between = &self.source[usize::from(lhs_end)..usize::from(rhs_start)]; - // `is None` reads the same in both languages; every other literal - // rhs changes meaning, so it takes the identity spelling - if !rhs.is_none_literal_expr() { - let words: &[&str] = match op { - CmpOp::Is => &["is"], - CmpOp::IsNot => &["is", "not"], - _ => &[], - }; - if let Some(tokens) = operator_tokens(between, lhs_end, words) { - self.rewrite_operator(&tokens, between, lhs_end); - } + let words: &[&str] = match op { + CmpOp::Is => &["is"], + CmpOp::IsNot => &["is", "not"], + _ => &[], + }; + if let Some(tokens) = operator_tokens(between, lhs_end, words) { + self.rewrite_operator(&tokens, between, lhs_end); } lhs_end = rhs.range().end(); } @@ -120,92 +100,6 @@ impl<'src> IdentitySwapReverse<'src> { last.end() + TextSize::from(u32::try_from(trailing).unwrap_or(0)), )))); } - - /// `not isinstance(x, y)` → `x is not y`, the exact inverse of the forward - /// transform. rewriting only the call would leave the correct but clumsier - /// `not x is y` - fn process_unary(&mut self, unary: &ruff_python_ast::ExprUnaryOp) { - if unary.op != UnaryOp::Not { - return; - } - let Expr::Call(call) = unary.operand.as_ref() else { - return; - }; - let Some((x, y)) = isinstance_operands(call) else { - return; - }; - let Some(target) = self.type_expression_target(y) else { - return; - }; - let x_src = self.src(x.range()).to_owned(); - self.folded_into_not.push(call.range()); - self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("{x_src} is not {target}"), - unary.range(), - ))); - } - - fn process_call(&mut self, call: &ruff_python_ast::ExprCall) { - if self.folded_into_not.contains(&call.range()) { - return; - } - let Some((x, y)) = isinstance_operands(call) else { - return; - }; - let Some(target) = self.type_expression_target(y) else { - return; - }; - let x_src = self.src(x.range()).to_owned(); - self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("{x_src} is {target}"), - call.range(), - ))); - } - - /// how an `isinstance` classinfo argument is written as the type expression - /// a type test takes, or `None` when its shape is not one a type expression - /// has and the call has to stay a call. - /// - /// A tuple is `isinstance`'s spelling for "any of these", which a type - /// expression spells `A | B` — writing the tuple back out would name the - /// *tuple type* instead, a different test entirely. - fn type_expression_target(&self, classinfo: &Expr) -> Option { - match classinfo { - Expr::Name(_) | Expr::Attribute(_) | Expr::Subscript(_) => { - Some(self.src(classinfo.range()).to_owned()) - } - Expr::BinOp(binop) if binop.op == ruff_python_ast::Operator::BitOr => { - Some(self.src(classinfo.range()).to_owned()) - } - Expr::Tuple(tuple) => { - let arms: Option> = tuple - .elts - .iter() - .map(|element| self.type_expression_target(element)) - .collect(); - let arms = arms?; - // an empty `isinstance(x, ())` is always `False`; there is no - // union to write for it - (!arms.is_empty()).then(|| arms.join(" | ")) - } - _ => None, - } - } -} - -/// the two operands of an `isinstance(x, y)` call. anything else — a keyword -/// argument, a different arity — stays as-is rather than lose semantics -fn isinstance_operands(call: &ruff_python_ast::ExprCall) -> Option<(&Expr, &Expr)> { - if !matches!(call.func.as_ref(), Expr::Name(n) if n.id.as_str() == "isinstance") { - return None; - } - if !call.arguments.keywords.is_empty() { - return None; - } - let [x, y] = &*call.arguments.args else { - return None; - }; - Some((x, y)) } /// the range of each word of the operator written between two comparison @@ -255,12 +149,8 @@ fn operator_tokens(gap: &str, gap_start: TextSize, words: &[&str]) -> Option Visitor<'ast> for IdentitySwapReverse<'_> { fn visit_expr(&mut self, expr: &'ast Expr) { - match expr { - Expr::Compare(c) => self.process_compare(c), - // before the walk reaches the call inside it - Expr::UnaryOp(unary) => self.process_unary(unary), - Expr::Call(call) => self.process_call(call), - _ => {} + if let Expr::Compare(c) = expr { + self.process_compare(c); } walk_expr(self, expr); } @@ -295,34 +185,6 @@ mod tests { ); } - #[test] - fn isinstance_to_is() { - check( - indoc! {" - if isinstance(x, int): - pass - "}, - indoc! {" - if x is int: - pass - "}, - ); - } - - #[test] - fn not_isinstance_to_is_not() { - check( - indoc! {" - if not isinstance(x, str): - pass - "}, - indoc! {" - if x is not str: - pass - "}, - ); - } - #[test] fn identity_to_triple_equals() { check("y = a is b\n", "y = a === b\n"); @@ -369,34 +231,13 @@ mod tests { ); } + /// `is None` is a type test in basedpython, and one the value's static type + /// settles is emitted as its answer, so it takes the identity operator like + /// every other literal #[test] - fn none_comparisons_left_alone() { - // basedpython reads `is None` as a test for the type `None`, which is - // the same runtime check python's identity performs - check("y = a is None\n", "y = a is None\n"); - check("y = a is not None\n", "y = a is not None\n"); - } - - #[test] - fn a_classinfo_tuple_becomes_a_union() { - // `isinstance`'s tuple means "any of these"; a type expression spells - // that `A | B`, and a tuple there is the tuple *type* - check("y = isinstance(a, (int, str))\n", "y = a is int | str\n"); - check( - "y = not isinstance(a, (int, (str, bytes)))\n", - "y = a is not int | str | bytes\n", - ); - } - - #[test] - fn a_classinfo_no_type_expression_can_say_keeps_the_call() { - // a call is not a type expression, and neither is the empty tuple — - // whose test is always `False` and has no union to write - check( - "y = isinstance(a, type(a))\n", - "y = isinstance(a, type(a))\n", - ); - check("y = isinstance(a, ())\n", "y = isinstance(a, ())\n"); + fn none_comparisons_take_the_identity_operator() { + check("y = a is None\n", "y = a === None\n"); + check("y = a is not None\n", "y = a !== None\n"); } #[test] @@ -408,6 +249,17 @@ mod tests { check("y = a is not 1\n", "y = a !== 1\n"); } + /// `x is int` folds wherever `x`'s type decides it, so a call keeps the + /// check python wrote + #[test] + fn isinstance_stays_a_call() { + check("y = isinstance(a, int)\n", "y = isinstance(a, int)\n"); + check( + "y = not isinstance(a, (int, str))\n", + "y = not isinstance(a, (int, str))\n", + ); + } + /// the comment case cannot round-trip byte for byte — the operator's layout /// normalises around the comment. what must survive is the *meaning*: it /// has to come back as identity, not as the `not isinstance(...)` a @@ -421,19 +273,6 @@ mod tests { assert!(!back.contains("isinstance"), "{back:?}"); } - #[test] - fn unrelated_call_left_alone() { - check("y = some(x, int)\n", "y = some(x, int)\n"); - } - - #[test] - fn isinstance_with_keyword_left_alone() { - check( - "y = isinstance(x, class_or_tuple=int)\n", - "y = isinstance(x, class_or_tuple=int)\n", - ); - } - #[test] fn round_trips() { check_round_trip("y = a is b\n"); @@ -443,4 +282,21 @@ mod tests { check_round_trip("y = isinstance(a, int)\n"); check_round_trip("y = not isinstance(a, int)\n"); } + + /// the annotation says what a check can only confirm at runtime. a type test + /// basedpython settles statically is emitted as its answer, so a python + /// check the annotations already decide must come back as the check + #[test] + fn a_check_the_annotations_settle_survives() { + check_round_trip(indoc! {" + def validate(x: int) -> int: + if not isinstance(x, int): + raise TypeError + return x + "}); + check_round_trip(indoc! {" + def f(x: int) -> None: + assert x is not None + "}); + } } diff --git a/crates/by_transforms/src/reverse_transforms/literal_string.rs b/crates/by_transforms/src/reverse_transforms/literal_string.rs index 6e61169afa..3ac8574b11 100644 --- a/crates/by_transforms/src/reverse_transforms/literal_string.rs +++ b/crates/by_transforms/src/reverse_transforms/literal_string.rs @@ -122,7 +122,7 @@ mod tests { "}, indoc! {" from typing import LiteralString - def f(x: literal str) -> literal str + def f(x: literal str) -> literal str: ... "}, ); } diff --git a/crates/by_transforms/src/reverse_transforms/mod.rs b/crates/by_transforms/src/reverse_transforms/mod.rs index 0bafcef735..c5928b8663 100644 --- a/crates/by_transforms/src/reverse_transforms/mod.rs +++ b/crates/by_transforms/src/reverse_transforms/mod.rs @@ -23,6 +23,7 @@ pub(crate) mod enums; pub(crate) mod export_import; pub(crate) mod extension; pub(crate) mod flexible_keyword; +pub(crate) mod forward_references; pub(crate) mod generics; pub(crate) mod identity_swap; pub(crate) mod intersection; diff --git a/crates/by_transforms/src/reverse_transforms/modifiers.rs b/crates/by_transforms/src/reverse_transforms/modifiers.rs index c4793f6dad..470bfa61b5 100644 --- a/crates/by_transforms/src/reverse_transforms/modifiers.rs +++ b/crates/by_transforms/src/reverse_transforms/modifiers.rs @@ -197,7 +197,7 @@ mod tests { fn final_method() { check( "class A:\n @final\n def f(self): ...\n", - "class A:\n final def f(self)\n", + "class A:\n final def f(self): ...\n", ); } @@ -205,7 +205,7 @@ mod tests { fn override_method() { check( "class A:\n @override\n def f(self): ...\n", - "class A:\n override def f(self)\n", + "class A:\n override def f(self): ...\n", ); } @@ -213,7 +213,7 @@ mod tests { fn static_method() { check( "class A:\n @staticmethod\n def f(): ...\n", - "class A:\n static def f()\n", + "class A:\n static def f(): ...\n", ); } @@ -221,7 +221,7 @@ mod tests { fn class_method() { check( "class A:\n @classmethod\n def f(cls): ...\n", - "class A:\n class def f(cls)\n", + "class A:\n class def f(cls): ...\n", ); } @@ -268,7 +268,7 @@ mod tests { // modifier decorator. check( "class A:\n @property\n @final\n def f(self) -> int: ...\n", - "class A:\n @property\n final def f(self) -> int\n", + "class A:\n @property\n final def f(self) -> int: ...\n", ); } diff --git a/crates/by_transforms/src/reverse_transforms/not_type.rs b/crates/by_transforms/src/reverse_transforms/not_type.rs index 233515a989..0a0bb8a92e 100644 --- a/crates/by_transforms/src/reverse_transforms/not_type.rs +++ b/crates/by_transforms/src/reverse_transforms/not_type.rs @@ -130,7 +130,7 @@ mod tests { "}, indoc! {" from ty_extensions import Not - def f() -> not str + def f() -> not str: ... "}, ); } diff --git a/crates/by_transforms/src/reverse_transforms/optional_type.rs b/crates/by_transforms/src/reverse_transforms/optional_type.rs index 0eda53c885..0e1169eeb0 100644 --- a/crates/by_transforms/src/reverse_transforms/optional_type.rs +++ b/crates/by_transforms/src/reverse_transforms/optional_type.rs @@ -232,7 +232,7 @@ mod tests { fn parameter_and_return_annotations() { check( "def f(x: str | None = None) -> bytes | None: ...\n", - "def f(x: str? = None) -> bytes?\n", + "def f(x: str? = None) -> bytes?: ...\n", ); } @@ -272,7 +272,7 @@ mod tests { "}, indoc! {" from typing import Callable - def f(cb: (int?) -> str) -> None + def f(cb: (int?) -> str) -> None: ... "}, ); } @@ -294,7 +294,7 @@ mod tests { "}, indoc! {" class Box[T]: - def get(self) -> T | None + def get(self) -> T | None: ... "}, ); } @@ -316,7 +316,7 @@ mod tests { _T = TypeVar("_T") class Box(Generic[_T]): - def get(self) -> _T | None + def get(self) -> _T | None: ... "#}, ); } @@ -336,7 +336,7 @@ mod tests { from typing import Self class Box: - def peek(self) -> Self? + def peek(self) -> Self?: ... "}, ); } @@ -352,7 +352,7 @@ mod tests { "}, indoc! {" class Box[T]: - def get(self) -> list[T]? + def get(self) -> list[T]?: ... "}, ); } @@ -408,10 +408,7 @@ mod tests { /// a value-position union is not an annotation and is never touched #[test] fn a_runtime_union_is_left_alone() { - check( - "def f(v: object):\n return isinstance(v, int | None)\n", - "def f(v: object):\n return v is int | None\n", - ); + unchanged("def f(v: object):\n return isinstance(v, int | None)\n"); } /// `raises` is a plain name, so `-> int? raises E` would read as the result diff --git a/crates/by_transforms/src/reverse_transforms/overload.rs b/crates/by_transforms/src/reverse_transforms/overload.rs index d0c3b0a8ad..34537fed1a 100644 --- a/crates/by_transforms/src/reverse_transforms/overload.rs +++ b/crates/by_transforms/src/reverse_transforms/overload.rs @@ -229,9 +229,9 @@ mod tests { #[test] fn single_def_overload_pass_unchanged() { - // overload reverse leaves a lone def alone; the empty-declarations - // reverse pass independently strips the `: ...` body - check("def f(a: int) -> int: ...\n", "def f(a: int) -> int\n"); + // overload reverse leaves a lone def alone, and outside a stub the + // empty-declarations reverse pass leaves its `: ...` body alone too + check("def f(a: int) -> int: ...\n", "def f(a: int) -> int: ...\n"); } #[test] diff --git a/crates/by_transforms/src/reverse_transforms/type_is.rs b/crates/by_transforms/src/reverse_transforms/type_is.rs index 6eccbc1829..f3484bda8f 100644 --- a/crates/by_transforms/src/reverse_transforms/type_is.rs +++ b/crates/by_transforms/src/reverse_transforms/type_is.rs @@ -116,7 +116,7 @@ mod tests { "}, indoc! {" from typing import TypeIs - def f(x) -> x is int + def f(x) -> x is int: ... "}, ); } @@ -132,7 +132,7 @@ mod tests { indoc! {" from typing import TypeIs def is_str(x: object) -> x is str: - return x is str + return isinstance(x, str) "}, ); } @@ -167,7 +167,7 @@ mod tests { int | str | bytes - ) + ): ... "}, ); } @@ -183,7 +183,7 @@ mod tests { "}, indoc! {" from typing import Callable, TypeIs - def f(x) -> x is (int) -> str + def f(x) -> x is (int) -> str: ... "}, ); } diff --git a/crates/by_transforms/src/reverse_transforms/unpack.rs b/crates/by_transforms/src/reverse_transforms/unpack.rs index a45b79247d..a27eea1569 100644 --- a/crates/by_transforms/src/reverse_transforms/unpack.rs +++ b/crates/by_transforms/src/reverse_transforms/unpack.rs @@ -145,7 +145,7 @@ mod tests { "}, indoc! {" from typing import Unpack - def f(*args: *tuple[int, ...]) + def f(*args: *tuple[int, ...]): ... "}, ); } @@ -161,7 +161,7 @@ mod tests { indoc! {" from typing import Unpack class A: - def method(self, *args: *tuple[str, ...]) + def method(self, *args: *tuple[str, ...]): ... "}, ); } @@ -171,7 +171,7 @@ mod tests { fn paramspec_pair_reversed() { check( "def f(*args: P.args, **kwargs: P.kwargs): ...\n", - "def f(*args: *P, **kwargs: **P)\n", + "def f(*args: *P, **kwargs: **P): ...\n", ); } @@ -184,7 +184,7 @@ mod tests { &Config::test_default(), ) .expect("reverse failed"); - assert_eq!(reversed, "def f(*args: *P, **kwargs: **P)\n"); + assert_eq!(reversed, "def f(*args: *P, **kwargs: **P): ...\n"); let forward = transpile(&reversed, &Config::test_default()).expect("forward failed"); assert!( forward.contains("*args: P.args, **kwargs: P.kwargs"), @@ -195,7 +195,7 @@ mod tests { /// only the paired form identifies a `ParamSpec`; a lone `.args` is left alone #[test] fn lone_args_component_unchanged() { - check("def f(*args: P.args): ...\n", "def f(*args: P.args)\n"); + check("def f(*args: P.args): ...\n", "def f(*args: P.args): ...\n"); } /// two different receivers are not a pair @@ -203,14 +203,14 @@ mod tests { fn mismatched_receivers_unchanged() { check( "def f(*args: P.args, **kwargs: Q.kwargs): ...\n", - "def f(*args: P.args, **kwargs: Q.kwargs)\n", + "def f(*args: P.args, **kwargs: Q.kwargs): ...\n", ); } #[test] fn regular_arg_unchanged_by_unpack() { - // unpack reverse leaves it alone; empty-declarations strips `: ...` - check("def f(x: int): ...\n", "def f(x: int)\n"); + // nothing here to reverse, and outside a stub the body stays + check("def f(x: int): ...\n", "def f(x: int): ...\n"); } /// the inner type keeps an edit another reverse transform made inside it — @@ -224,7 +224,7 @@ mod tests { "}, indoc! {" from typing import Unpack - def f(*args: *(int, (str, bytes))) + def f(*args: *(int, (str, bytes))): ... "}, ); } @@ -238,7 +238,7 @@ mod tests { "}, indoc! {" Unpack = object() - def f(*args: Unpack[tuple[int, ...]]) + def f(*args: Unpack[tuple[int, ...]]): ... "}, ); } diff --git a/crates/by_transforms/src/runtime.rs b/crates/by_transforms/src/runtime.rs new file mode 100644 index 0000000000..9679d65eca --- /dev/null +++ b/crates/by_transforms/src/runtime.rs @@ -0,0 +1,408 @@ +//! the runtime helpers the emitted python calls, and the two ways a module gets +//! at them +//! +//! the helpers live in [`SOURCE`], which is `_by_runtime.py`. a build writes that +//! file beside the modules it emits and each module imports the names it calls. a +//! transpile with nowhere to write it (`by transpile `, the language +//! server's `by/transpile`) pastes the definitions in instead. both are slices of +//! the one text, so the two renderings cannot drift apart +//! +//! # naming a helper +//! +//! a transform names a helper through one of the `Helper` constants, never a +//! string, so a misspelled helper does not compile. `every_helper_is_defined` +//! holds each constant to the file +//! +//! # slicing +//! +//! a helper resolves to the top-level statement that binds its name, as our own +//! parser reads the file. a comment written between two definitions falls outside +//! both and never reaches the output; one inside a body is part of it. a +//! top-level statement that binds nothing, such as +//! `_by_forward_operators(_LazyAttr)`, travels with the definition above it +//! +//! # dependencies +//! +//! helpers call each other: `_lazy_attr` resolves through `_lazy_module`, +//! `_soundness_parametric` through `_parametric_is`. what a helper needs is read +//! out of the calls in its body, so asking for one yields a set that runs + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::OnceLock; + +use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr}; +use ruff_python_ast::{self as ast, ExceptHandler, Expr, Stmt}; +use ruff_python_parser::parse_module; +use ruff_text_size::Ranged; + +/// the runtime, as the file a build writes out +pub const SOURCE: &str = include_str!("runtime/_by_runtime.py"); + +/// the module a build writes [`SOURCE`] to, without a package qualifier +pub const MODULE_NAME: &str = "_by_runtime"; + +/// the file a build writes [`SOURCE`] to +pub const FILE_NAME: &str = "_by_runtime.py"; + +/// a runtime helper the emitted python calls, by the name it calls it under +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) struct Helper(&'static str); + +impl Helper { + pub(crate) const fn name(self) -> &'static str { + self.0 + } +} + +macro_rules! helpers { + ($($konst:ident = $name:literal,)*) => { + $(pub(crate) const $konst: Helper = Helper($name);)* + + /// every helper a transform can name + #[cfg(test)] + const ALL: &[Helper] = &[$($konst),*]; + }; +} + +helpers! { + SOUNDNESS_CHECK = "_soundness_check", + SOUNDNESS_ITER = "_soundness_iter", + SOUNDNESS_AITER = "_soundness_aiter", + SOUNDNESS_PARAMETRIC = "_soundness_parametric", + SOUNDNESS_ITER_P = "_soundness_iter_p", + SOUNDNESS_AITER_P = "_soundness_aiter_p", + CHECKED_CAST = "_checked_cast", + TRY_CAST = "_try_cast", + CHECKED_CAST_PRED = "_checked_cast_pred", + TRY_CAST_PRED = "_try_cast_pred", + PARAMETRIC_IS = "_parametric_is", + PARAMETRIC_IS_LENIENT = "_parametric_is_lenient", + PROTOCOL_IS = "_by_protocol_is", + LITERAL = "_by_lit", + PATTERN_IS = "_by_pattern_is", + CONFORM = "_by_conform", + CONFORMS = "_by_conforms", + WITNESS = "_by_witness", + WITNESS_CLASS = "_by_witness_class", + WITNESS_GET = "_by_witness_get", + TEMPLATE = "_Template", + INTERPOLATION = "_Interpolation", + GRAPHEMES = "_by_graphemes", + PREFIX = "_by_prefix", + SUFFIX = "_by_suffix", + GENERIC = "generic", + GENERIC_CLASS = "generic_class", + TYPE_ARGUMENT = "_type_argument", + DISCARD = "_by_discard", + LOOP_BIND = "_by_loop_bind", + STATIC_PROPERTY = "_by_static_property", + OPTIONAL = "Optional", + FORCE_UNWRAP = "_force_unwrap", + MAIN_ARGS = "_by_main_args", + RAISES = "_by_raises", + LAZY_MODULE = "_lazy_module", + LAZY_ATTR = "_lazy_attr", + TY_EXT_MARKER = "_TyExtMarker", + CHARACTER = "Character", + MATCH_MISS = "_by_match_miss", + MATCH_SEQ = "_by_match_seq", + MATCH_MAP = "_by_match_map", + MATCH_KEY = "_by_match_key", + MATCH_REST = "_by_match_rest", + MATCH_ARGS = "_by_match_args", + MATCH_ATTR = "_by_match_attr", +} + +/// one definition of the runtime, with whatever finishes setting it up +struct Definition { + /// its source as the file spells it, newline-terminated like any other + /// preamble entry + source: String, + /// position in the file. a set of definitions renders in this order, which + /// is the order they can be executed in + order: usize, + /// the other definitions its body reads + needs: Vec, +} + +/// every definition, indexed by each name it binds +fn index() -> &'static BTreeMap { + static INDEX: OnceLock> = OnceLock::new(); + INDEX.get_or_init(build_index) +} + +fn build_index() -> BTreeMap { + // `the_runtime_parses` pins the file as valid python. were it not, every + // helper would come back undefined and phase 3 would report the calls + let Ok(parsed) = parse_module(SOURCE) else { + return BTreeMap::new(); + }; + let suite = parsed.suite(); + // two passes: the first learns every name the file defines, so the second can + // tell a call to a sibling from a call to a builtin + let defined: BTreeSet = suite.iter().flat_map(bindings).collect(); + + let mut definitions = BTreeMap::new(); + let mut order = 0usize; + let mut at = 0usize; + while at < suite.len() { + let names = bindings(&suite[at]); + if names.is_empty() { + // the module docstring: every later statement that binds nothing is + // swept up by the definition before it + at += 1; + continue; + } + let mut last = at; + while last + 1 < suite.len() && bindings(&suite[last + 1]).is_empty() { + last += 1; + } + let span = suite[at].range().cover(suite[last].range()); + let mut source = SOURCE[span].to_owned(); + if !source.ends_with('\n') { + source.push('\n'); + } + let mut needs: Vec = suite[at..=last] + .iter() + .flat_map(reads) + .filter(|read| defined.contains(read) && !names.contains(read)) + .collect(); + needs.sort(); + needs.dedup(); + // `every_name_has_one_definition` keeps a name from being bound twice, + // so first-wins here never decides anything + for name in names { + definitions.entry(name).or_insert_with(|| Definition { + source: source.clone(), + order, + needs: needs.clone(), + }); + } + order += 1; + at = last + 1; + } + definitions +} + +/// the names a top-level statement binds at module scope +pub(crate) fn bindings(stmt: &Stmt) -> BTreeSet { + let mut names = BTreeSet::new(); + bind(stmt, &mut names); + names +} + +/// a block that runs at module scope (an `if`, a `try`, a loop) binds what the +/// statements inside it bind. a function or class body is a scope of its own +fn bind(stmt: &Stmt, names: &mut BTreeSet) { + let block = |body: &[Stmt], names: &mut BTreeSet| { + for stmt in body { + bind(stmt, names); + } + }; + match stmt { + Stmt::FunctionDef(def) => { + names.insert(def.name.to_string()); + } + Stmt::ClassDef(def) => { + names.insert(def.name.to_string()); + } + Stmt::Assign(assign) => { + for target in &assign.targets { + if let Expr::Name(name) = target { + names.insert(name.id.to_string()); + } + } + } + Stmt::AnnAssign(assign) => { + if let Expr::Name(name) = assign.target.as_ref() { + names.insert(name.id.to_string()); + } + } + Stmt::Import(import) => { + for alias in &import.names { + let bound = match &alias.asname { + Some(asname) => asname.to_string(), + // `import a.b` binds `a` + None => alias.name.split('.').next().unwrap_or_default().to_string(), + }; + names.insert(bound); + } + } + Stmt::ImportFrom(import) => { + for alias in &import.names { + names.insert(match &alias.asname { + Some(asname) => asname.to_string(), + None => alias.name.to_string(), + }); + } + } + Stmt::If(node) => { + block(&node.body, names); + for clause in &node.elif_else_clauses { + block(&clause.body, names); + } + } + Stmt::Try(node) => { + block(&node.body, names); + for ExceptHandler::ExceptHandler(handler) in &node.handlers { + block(&handler.body, names); + } + block(&node.orelse, names); + block(&node.finalbody, names); + } + Stmt::With(node) => block(&node.body, names), + Stmt::For(node) => { + block(&node.body, names); + block(&node.orelse, names); + } + Stmt::While(node) => { + block(&node.body, names); + block(&node.orelse, names); + } + _ => {} + } +} + +/// every name a statement loads, at any depth +fn reads(stmt: &Stmt) -> BTreeSet { + struct Reads(BTreeSet); + impl SourceOrderVisitor<'_> for Reads { + fn visit_expr(&mut self, expr: &Expr) { + if let Expr::Name(ast::ExprName { id, ctx, .. }) = expr + && ctx.is_load() + { + self.0.insert(id.to_string()); + } + walk_expr(self, expr); + } + } + let mut visitor = Reads(BTreeSet::new()); + ruff_python_ast::visitor::source_order::walk_stmt(&mut visitor, stmt); + visitor.0 +} + +/// the definitions `helpers` need, their own included, in the order the file +/// defines them +/// +/// a name the file does not define is left out rather than reported here: the +/// module then calls something it never got, and phase 3 says so with a span +fn closure(helpers: impl IntoIterator) -> Vec<&'static Definition> { + let index = index(); + let mut resolved: BTreeMap = BTreeMap::new(); + let mut seen: BTreeSet<&str> = BTreeSet::new(); + let mut queue: Vec<&str> = helpers.into_iter().map(Helper::name).collect(); + while let Some(name) = queue.pop() { + if !seen.insert(name) { + continue; + } + let Some(definition) = index.get(name) else { + continue; + }; + // a statement bound under several names is still one definition + resolved.insert(definition.order, definition); + queue.extend(definition.needs.iter().map(String::as_str)); + } + resolved.into_values().collect() +} + +/// whether `name` is one of the runtime's definitions +pub(crate) fn defines(name: &str) -> bool { + index().contains_key(name) +} + +/// the definitions `helpers` need, as preamble entries for a module that has +/// nowhere to import them from +pub(crate) fn inline(helpers: impl IntoIterator) -> Vec { + closure(helpers) + .into_iter() + .map(|definition| definition.source.clone()) + .collect() +} + +/// the import a module uses to reach helpers written out beside it +/// +/// it names only what the emitted code calls: a helper that exists because +/// another helper calls it is resolved inside `_by_runtime`. absolute rather than +/// relative, because a relative import fails in a module run as `__main__` +pub(crate) fn import_line(module: &str, helpers: impl IntoIterator) -> String { + let names: BTreeSet<&str> = helpers.into_iter().map(Helper::name).collect(); + format!( + "from {module} import {}", + names.into_iter().collect::>().join(", ") + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_runtime_parses() { + if let Err(error) = parse_module(SOURCE) { + panic!("{FILE_NAME} does not parse: {error}"); + } + } + + /// a constant naming something the file does not define would reach the + /// output as a call to nothing + #[test] + fn every_helper_is_defined() { + for helper in ALL { + assert!( + defines(helper.name()), + "`{}` is not defined in {FILE_NAME}", + helper.name() + ); + } + } + + /// two statements binding one name would leave it to file order which one a + /// module gets + #[test] + fn every_name_has_one_definition() { + let parsed = parse_module(SOURCE).expect("the runtime parses"); + let mut seen = BTreeSet::new(); + for stmt in parsed.suite() { + for name in bindings(stmt) { + assert!(seen.insert(name.clone()), "`{name}` is bound twice"); + } + } + } + + /// the preamble is one entry per line, so an entry that did not end its own + /// line would run into the next + #[test] + fn definitions_end_in_a_newline() { + for (name, definition) in index() { + assert!(definition.source.ends_with('\n'), "`{name}` does not"); + } + } + + #[test] + fn dependencies_resolve() { + for (name, definition) in index() { + for need in &definition.needs { + assert!(defines(need), "`{name}` reads unknown `{need}`"); + } + } + } + + /// the discard adapter is spelled by the checker as well as emitted here + #[test] + fn the_discard_adapter_is_the_one_ty_resolves() { + assert_eq!(DISCARD.name(), ty_python_semantic::DISCARD_ADAPTER); + } + + /// the proxy's operator forwarding is read by nothing; it has to travel with + /// the class it patches, or the proxy is left with no operators at all + #[test] + fn set_up_code_travels_with_its_definition() { + let pasted = inline([LAZY_ATTR]).concat(); + assert!(pasted.contains("class _LazyAttr:"), "{pasted}"); + assert!( + pasted.contains("_by_forward_operators(_LazyAttr)"), + "{pasted}" + ); + assert!(pasted.contains("def _lazy_module("), "{pasted}"); + } +} diff --git a/crates/by_transforms/src/runtime/_by_runtime.py b/crates/by_transforms/src/runtime/_by_runtime.py new file mode 100644 index 0000000000..1d0a00d024 --- /dev/null +++ b/crates/by_transforms/src/runtime/_by_runtime.py @@ -0,0 +1,1405 @@ +"""the helpers basedpython's emitted python calls at run time + +a build writes this file beside the modules it emits, and each module imports +the names it calls. a transpile with nowhere to write it pastes the definitions +it needs in instead. `runtime.rs` slices them out by name, and nothing here is +imported by hand +""" + + +# --- runtime type-soundness checks --------------------------------------- +# inserted where ty accepts a value on an annotation-level claim it cannot +# verify: a generic call's result, a projection out of a specialized container, +# a loop element, an annotated assignment, a return, an argument + + +def _soundness_check(_v, _t): + if not isinstance(_v, _t): + raise TypeError( + f"type soundness violation: expected {getattr(_t, '__name__', _t)}, " + f"got {type(_v).__name__}" + ) + return _v + + +def _soundness_iter(_it, _t): + for _x in _it: + yield _soundness_check(_x, _t) + + +async def _soundness_aiter(_it, _t): + async for _x in _it: + yield _soundness_check(_x, _t) + + +# --- checked casts --------------------------------------------------------- +# `value cast! T` verifies at run time and raises on a mismatch; `value cast? T` +# yields None instead. the predicate forms serve any target the parametric +# engine below can decide — a reified-cell comparison (`T == int`), an +# `__orig_class__` probe, a structural protocol check, or a disjunction of those +# across a union's arms. their predicate is a lambda, so the value is evaluated +# exactly once (as `_v`) and referenced from inside the test + + +def _checked_cast(_v, _t): + if not isinstance(_v, _t): + raise TypeError( + f"cast to {getattr(_t, '__name__', _t)} failed: value is {type(_v).__name__}" + ) + return _v + + +def _try_cast(_v, _t): + return _v if isinstance(_v, _t) else None + + +def _checked_cast_pred(_v, _pred): + if not _pred(_v): + raise TypeError(f"cast failed: value is {type(_v).__name__}") + return _v + + +def _try_cast_pred(_v, _pred): + return _v if _pred(_v) else None + + +# --- optional values ------------------------------------------------------- +# `T?` is a type, not a wrapper, so this class exists for the few places the +# emitted code needs one at run time: `Some(x)`, a force-unwrap, a spelled-out +# cast target + + +class Optional: + def __init__(self, value): + self.value = value + + def __class_getitem__(cls, item): + return cls + + def __repr__(self): + return f"Some({self.value!r})" + + +def _force_unwrap(_v): + if isinstance(_v, Optional): + return _v.value + if _v is None: + raise RuntimeError("force-unwrap of absent value") + if isinstance(_v, BaseException): + raise RuntimeError("force-unwrap of absent value") from _v + return _v + + +# --- effects, entry points and loop capture -------------------------------- + + +# a clause naming a reified type parameter is passed as `_resolve`, a lambda +# building the test from the parameters' runtime values, beside the constant +# ceiling. `_own` names the guarded function's own parameters, read off the +# specialization its `generic` was called through; `_receiver` names its class's, +# read off the instance the method was called on — the one the specialization +# was bound to, or else the first argument. a parameter of an enclosing reified +# function is neither: the lambda closes over it where the guard is evaluated. +# whenever a value cannot be read, the guard tests the ceiling +def _by_raises(_allowed, _name, _resolve=None, _own=(), _receiver=()): + import functools + import inspect + + def _target(_generic, _args): + if _resolve is None: + return _allowed + try: + _values = [] + if _own: + _bound = _bind_type_params( + _generic.fn.__type_params__, + _generic.args or (), + _generic.fields or {}, + _generic.fn.__name__, + ) + _values.extend(_bound[_param] for _param in _own) + if _receiver: + _instance = None if _generic is None else _generic.instance + _owner = _args[0] if _instance is None else _instance + _values.extend(_type_argument(_owner, _param) for _param in _receiver) + return _resolve(*_values) + except Exception: + # a parameter nothing answered for: what the declaration says without + # it is the ceiling + return _allowed + + def _check(_exc, _generic, _args): + if not _by_isinstance(_exc, _target(_generic, _args), _allowed): + raise AssertionError( + f"{_name} raised {type(_exc).__name__}, which its `raises` clause does not include" + ) from _exc + + def _wrap(_shape, _call, _generic): + if inspect.isasyncgenfunction(_shape): + @functools.wraps(_shape) + async def _wrapper(*_args, **_kwargs): + try: + async for _item in _call(*_args, **_kwargs): + yield _item + except BaseException as _exc: + _check(_exc, _generic, _args) + raise + elif inspect.iscoroutinefunction(_shape): + @functools.wraps(_shape) + async def _wrapper(*_args, **_kwargs): + try: + return await _call(*_args, **_kwargs) + except BaseException as _exc: + _check(_exc, _generic, _args) + raise + elif inspect.isgeneratorfunction(_shape): + @functools.wraps(_shape) + def _wrapper(*_args, **_kwargs): + try: + yield from _call(*_args, **_kwargs) + except BaseException as _exc: + _check(_exc, _generic, _args) + raise + else: + @functools.wraps(_shape) + def _wrapper(*_args, **_kwargs): + try: + return _call(*_args, **_kwargs) + except BaseException as _exc: + _check(_exc, _generic, _args) + raise + return _wrapper + + def _decorate(_fn): + # a reified generic is specialized after it is decorated (`f[int](…)`), + # so the guard has to keep answering the subscript, and it reads the + # type arguments off the specialization that subscript produces + if getattr(_fn, "__by_generic__", False): + return _by_guarded_generic(_fn, _wrap) + return _wrap(_fn, _fn, None) + + return _decorate + + +def _by_isinstance(_value, _target, _fallback): + # a type argument is whatever the caller wrote, and `isinstance` refuses a + # subscripted generic. the shallow test is its origin, the way `list[str]` + # is tested as `list`, and past that the ceiling + try: + return isinstance(_value, _target) + except TypeError: + pass + try: + return isinstance(_value, _by_runtime_classes(_target)) + except TypeError: + return isinstance(_value, _fallback) + + +def _by_runtime_classes(_target): + import typing + + if isinstance(_target, tuple): + return tuple(_by_runtime_classes(_member) for _member in _target) + return typing.get_origin(_target) or _target + + +class _by_guarded_generic: + # a guarded reified generic: still subscriptable, still a descriptor, and + # documented by the function it wraps + __doc__ = property(lambda self: self._by_inner.__doc__) + + def __init__(self, _inner, _wrap): + self._by_inner = _inner + self._by_wrap = _wrap + self._by_call = None + + def __repr__(self): + return repr(self._by_inner) + + def __getattr__(self, _name): + if _name in ("_by_inner", "_by_wrap", "_by_call"): + raise AttributeError(_name) + return getattr(self._by_inner, _name) + + def __get__(self, _obj, _objtype=None): + return _by_guarded_generic(self._by_inner.__get__(_obj, _objtype), self._by_wrap) + + def __getitem__(self, *_items, **_fields): + return _by_guarded_generic( + self._by_inner.__getitem__(*_items, **_fields), self._by_wrap + ) + + def __call__(self, *_args, **_kwargs): + # the wrapper's shape comes from the function this specialization holds, + # so it is built once for it rather than once per call + if self._by_call is None: + _inner = self._by_inner + self._by_call = self._by_wrap(_inner.fn, _inner, _inner) + return self._by_call(*_args, **_kwargs) + + +def _by_main_args(_fn, _params, _extra=None): + import argparse + + _parser = argparse.ArgumentParser(description=_fn.__doc__) + for _i, (_name, _type, _kind, _required, _choices) in enumerate(_params): + _flags = [f"--{_name.replace('_', '-')}"] + if "_" in _name: + _flags.append(f"--{_name}") + if _type is None: + _parser.add_argument(*_flags, dest=f"o{_i}", action="store_true", default=None) + _parser.add_argument( + *[f"--no-{_flag[2:]}" for _flag in _flags], + dest=f"o{_i}", + action="store_false", + default=None, + ) + continue + if _kind != "keyword": + _parser.add_argument( + f"p{_i}", + metavar=_name, + nargs="?", + type=_type, + default=None, + choices=_choices, + ) + _parser.add_argument( + *_flags, + dest=f"o{_i}", + metavar=_name.upper(), + type=_type, + default=None, + choices=_choices, + ) + if _extra is None: + _parsed = vars(_parser.parse_args()) + _rest = [] + else: + _namespace, _rest = _parser.parse_known_args() + _parsed = vars(_namespace) + # `parse_known_args` hands back what it did not recognise as it was + # written, so the vararg's own annotation is what converts it + _rest = [_extra(_value) for _value in _rest] + _args = [] + _kwargs = {} + _omitted = None + for _i, (_name, _type, _kind, _required, _choices) in enumerate(_params): + _value = _parsed.get(f"o{_i}") + _positional = _parsed.get(f"p{_i}") + if _value is not None and _positional is not None: + _parser.error(f"argument {_name}: given both positionally and as an option") + if _value is None: + _value = _positional + if _value is None: + if _required: + _parser.error(f"the following arguments are required: {_name}") + if _kind == "positional": + _omitted = _name + continue + if _kind == "positional": + if _omitted is not None: + _parser.error(f"argument {_name}: cannot be given without {_omitted}") + _args.append(_value) + else: + _kwargs[_name] = _value + for _value in _rest: + if _omitted is not None: + _parser.error(f"argument {_value}: cannot be given without {_omitted}") + _args.append(_value) + return _args, _kwargs + + +def _by_loop_bind(**_by_values): + from types import CellType, FunctionType + + def _by_rebind(_by_fn): + _by_code = _by_fn.__code__ + _by_bound = FunctionType( + _by_code, + _by_fn.__globals__, + _by_fn.__name__, + _by_fn.__defaults__, + tuple( + CellType(_by_values[_by_name]) if _by_name in _by_values else _by_cell + for _by_name, _by_cell in zip(_by_code.co_freevars, _by_fn.__closure__ or ()) + ), + ) + _by_bound.__kwdefaults__ = _by_fn.__kwdefaults__ + _by_bound.__qualname__ = _by_fn.__qualname__ + _by_bound.__doc__ = _by_fn.__doc__ + _by_bound.__dict__.update(_by_fn.__dict__) + if hasattr(_by_fn, "__annotate__"): + _by_bound.__annotate__ = _by_fn.__annotate__ + else: + _by_bound.__annotations__ = _by_fn.__annotations__ + if hasattr(_by_fn, "__type_params__"): + _by_bound.__type_params__ = _by_fn.__type_params__ + return _by_bound + return _by_rebind + + +# --- string templates and graphemes ---------------------------------------- + + +# PEP 750 `Template` / `Interpolation` polyfill for runtimes before 3.14 +# +# matches the `string.templatelib` shape a tag relies on: `Template.strings` +# is the literal segments (always one more than the interpolations), +# `Template.interpolations` is the replacement fields, and `Template.values` +# is their evaluated values. iterating a `Template` yields the segments and +# interpolations interleaved in source order, the same as the stdlib type +class _Interpolation: + def __init__(self, value, expression, conversion=None, format_spec=""): + self.value = value + self.expression = expression + self.conversion = conversion + self.format_spec = format_spec + + +class _Template: + def __init__(self, *args): + strings = [] + interpolations = [] + if not args or isinstance(args[-1], _Interpolation): + args = (*args, "") + pending = "" + for arg in args: + if isinstance(arg, _Interpolation): + strings.append(pending) + pending = "" + interpolations.append(arg) + else: + pending += arg + strings.append(pending) + self.strings = tuple(strings) + self.interpolations = tuple(interpolations) + + @property + def values(self): + return tuple(i.value for i in self.interpolations) + + def __iter__(self): + for index, string in enumerate(self.strings): + if string: + yield string + if index < len(self.interpolations): + yield self.interpolations[index] + + +def _by_graphemes(_text): + try: + import regex as _regex + except ImportError as _err: + raise ImportError( + "basedpython's grapheme string surface (character_count / first / last / " + "characters / character_at / ...) needs the 'regex' package: uv add regex" + ) from _err + return _regex.findall(r"\X", _text) + + +def _by_prefix(_text, _n): + return "".join(_by_graphemes(_text)[:max(0, _n)]) + + +def _by_suffix(_text, _n): + _g = _by_graphemes(_text) + return "".join(_g[max(0, len(_g) - _n):]) + + +# --- class-level properties and reified generics --------------------------- + + +# binds supplied type arguments onto a type-parameter list, shared by the +# function wrapper and the class specializer +# +# a `TypeVarTuple` takes, as a tuple, the whole run of positional arguments +# the fixed parameters around it don't claim, so `[int, str, bool]` on +# `[T, *Args]` binds `T = int` and `Args = (str, bool)`; a keyword-variadic +# `**Kwargs` sits outside the positional slots entirely and binds the mapping +# of the keyword fields (`f[foo=int]` → `Kwargs = {'foo': int}`, spelled +# `f.__getitem__(foo=int)` in the lowered python, since subscripts take no +# keywords). an omitted slot is filled from its pep 696 default, read off the +# parameter list itself; an unfilled `TypeVarTuple` or `**Kwargs` binds empty, +# and any other slot is simply left out for the caller to answer for. +# over-specializing a parameter list with no variadic raises +def _bind_type_params(params, supplied, fields, owner): + from typing import ParamSpec, TypeVarTuple + pack = next((p for p in params if isinstance(p, ParamSpec)), None) + if pack is None and fields: + raise TypeError( + f"{owner} has no keyword-variadic type parameter for " + f"{', '.join(fields)}" + ) + slots = [p for p in params if p is not pack] + variadic = next( + (i for i, p in enumerate(slots) if isinstance(p, TypeVarTuple)), None + ) + if variadic is None: + if len(supplied) > len(slots): + raise TypeError( + f"too many type arguments for {owner}: " + f"expected {len(slots)}, got {len(supplied)}" + ) + bound = dict(zip((p.__name__ for p in slots), supplied)) + else: + trailing = slots[variadic + 1:] + packed = tuple(supplied[variadic:len(supplied) - len(trailing)]) + bound = dict(zip((p.__name__ for p in slots[:variadic]), supplied)) + if packed: + bound[slots[variadic].__name__] = packed + bound.update( + zip( + (p.__name__ for p in trailing), + supplied[variadic + len(packed):], + ) + ) + if fields: + bound[pack.__name__] = dict(fields) + for param in params: + name = param.__name__ + if name in bound: + continue + has_default = getattr(param, "has_default", None) + if has_default is not None and has_default(): + bound[name] = param.__default__ + elif isinstance(param, TypeVarTuple): + bound[name] = () + elif param is pack: + bound[name] = {} + return bound + + + +class _by_static_property: + def __init__(self, fget): + self._fget = fget + def __get__(self, instance, owner=None): + return self._fget(owner if owner is not None else type(instance)) + + +# the `generic` wrapper, for a function that reifies its type parameters +# +# `f[int]` produces a specialized `generic` carrying `args=(int,)`; calling it +# rebuilds the function with a closure whose type-parameter cells hold the +# type arguments, keyed by `co_freevars` name so unrelated cells (captured +# locals, `__class__`) survive. parameter defaults, kwonly defaults and the +# qualname carry over to the rebuilt function +# +# the supplied arguments are mapped onto the parameters by +# `_bind_type_params`, so `f()` works when every reified parameter +# carries a pep 696 default; a slot that binding leaves empty and the body +# reads raises `TypeError` at the call. the wrapper is also a descriptor: +# `__get__` captures the receiver so a reified *method* (`obj.m[int]()`) binds +# `self` like an ordinary method. attribute access falls through to the +# wrapped function, keeping introspection (`f.__name__`, `f.__doc__`) working +class generic: + # what tells another lowering's wrapper that this is a reified generic and + # not the plain function it forwards to — the `raises` guard has to keep the + # specialization subscript working + __by_generic__ = True + __doc__ = property(lambda self: self.fn.__doc__) + + def __init__(self, fn, args=None, instance=None, fields=None): + self.fn = fn + self.args = args + self.instance = instance + self.fields = fields + + def __repr__(self): + return f"" + + def __getattr__(self, name): + if name == "fn": + raise AttributeError(name) + return getattr(self.fn, name) + + def __get__(self, obj, objtype=None): + if obj is None: + return self + return generic(self.fn, self.args, obj, self.fields) + + def __getitem__(self, *items, **fields): + if self.args is not None or self.fields is not None: + raise TypeError("type arguments already specified") + if len(items) == 1 and isinstance(items[0], tuple): + items = items[0] + # reject a bad arity here, not at the call + _bind_type_params(self.fn.__type_params__, items, fields, self.fn.__name__) + return generic(self.fn, items, self.instance, fields) + + def __call__(self, *args, **kwargs): + from types import CellType, FunctionType + + fn = self.fn + code = fn.__code__ + values = _bind_type_params( + fn.__type_params__, self.args or (), self.fields or {}, fn.__name__ + ) + for param in fn.__type_params__: + name = param.__name__ + if name not in values and name in code.co_freevars: + # a synthesized parameter stands for an erased union the user + # never spelled, so naming it would leak the lowering + if name.startswith("__by_erased"): + raise TypeError( + f"{fn.__name__}() cannot tell which specialization it was " + f"given: the argument's type arguments are erased at " + f"runtime, and the call site did not record them" + ) + raise TypeError(f"{fn.__name__}() missing a type argument for {name!r}") + closure = tuple( + CellType(values[name]) if name in values else cell + for name, cell in zip(code.co_freevars, fn.__closure__ or ()) + ) + temp_fn = FunctionType(code, fn.__globals__, fn.__name__, fn.__defaults__, closure) + temp_fn.__kwdefaults__ = fn.__kwdefaults__ + temp_fn.__qualname__ = fn.__qualname__ + if self.instance is not None: + return temp_fn(self.instance, *args, **kwargs) + return temp_fn(*args, **kwargs) + + +# --- conformance registry -------------------------------------------------- +# a module declaring `extension str(Show)` registers the conformance when it is +# imported, and every module that tests one reads the same registry + + +# the runtime a conformance needs: the registry, the per-member lookup, the +# `is`-test, and the two dispatchers (a method is fetched and called by the +# parentheses that already follow the access; a data member is read) +# +# three things here are load-bearing and were each a bug before: +# +# - **the registry is per *process*, not per module.** a module-level `{}` would +# be private to whichever copy of these helpers ran — and a transpile with +# nowhere to write this file pastes them into each module that needs them, so +# there can be several. it is parked in `sys.modules` instead, which is the one +# namespace every module already shares +# - **the lookup is per *member*.** walking the MRO for the first class with +# *any* table would let a base's conformance beat a subclass's own method — +# the same object answering two ways depending on its static type. whichever +# comes first in the MRO wins: a table entry for this member, or a class that +# defines it +# - **a conformance registers under every interface it implies.** conforming to +# `Loud(Show)` conforms to `Show`, and a receiver typed as `Show` looks up +# under `Show` +def _by_registry(): + # one registry per process: a module that pasted these helpers in has a copy + # of its own, and a conformance registered through any copy has to be + # visible to all of them. `sys.modules` is the namespace they already share. + # imported inside the function so the lazy-import pass has no statement to + # rewrite + import sys + import types + module = sys.modules.get("_by_conformance_registry") + if module is None: + module = types.ModuleType("_by_conformance_registry") + module.table = {} + sys.modules["_by_conformance_registry"] = module + return module.table + +_by_conformances = _by_registry() + +def _by_conform(interface, cls, witness): + # conforming to an interface conforms to everything it derives, so a + # receiver typed as a supertype finds the same witness + for base in getattr(interface, "__mro__", (interface,)): + if base is object or getattr(base, "__module__", None) == "typing": + continue + _by_conformances.setdefault(base, {}).setdefault(cls, {}).update(witness) + +def _by_witness_entry(value, interface, name): + table = _by_conformances.get(interface) + if table is None: + return None + for cls in type(value).__mro__: + witness = table.get(cls) + if witness is not None and name in witness: + return witness[name] + # a class that defines the member itself answers it, and beats any + # conformance registered further up the mro + if name in cls.__dict__: + return None + return None + +def _by_conforms(value, interface, members=None): + table = _by_conformances.get(interface) + if table is not None: + for cls in type(value).__mro__: + if cls in table: + return True + if members is None: + return isinstance(value, interface) + return all(hasattr(value, name) for name in members) + +def _by_witness(value, interface, name): + function = _by_witness_entry(value, interface, name) + if function is None: + return getattr(value, name) + return lambda *args, **kwargs: function(value, *args, **kwargs) + +def _by_witness_class(value, interface, name): + function = _by_witness_entry(value, interface, name) + if function is None: + return getattr(value, name) + owner = value if isinstance(value, type) else type(value) + return lambda *args, **kwargs: function(owner, *args, **kwargs) + +def _by_witness_get(value, interface, name): + function = _by_witness_entry(value, interface, name) + if function is None: + return getattr(value, name) + return function(value) + + +# --- parametric type tests ------------------------------------------------- +# the one engine behind `is`, `cast` and the deep soundness checks: given a +# value and an alias, decide whether the value really is that specialization + + +def _by_type_param_defaults(args): + # a class records its generic bases *unsubstituted* — `class L[T = Never] + # (list[T])` stores `list[T]`, never `list[Never]` — so a type parameter + # left at its pep 696 default resolves to that default rather than staying a + # bare TypeVar that matches nothing + resolved = [] + substituted = False + for arg in args: + has_default = getattr(arg, "has_default", None) + if has_default is not None and has_default(): + resolved.append(arg.__default__) + substituted = True + else: + resolved.append(arg) + return tuple(resolved) if substituted else args + +def _by_alias(value): + # a reified generic class specializes to a *subclass*, which records the + # alias it stands for; anything else already is what it says it is. read + # from the class's own dict, so an ordinary subclass of a specialization is + # not mistaken for one + if isinstance(value, type): + return value.__dict__.get("__orig_class__", value) + return value + +def _by_subst(annotation, mapping): + # replace type parameters with the arguments bound to them, rebuilding + # nested aliases (`list[dict[str, T]]` with `T = int` → `list[dict[str, int]]`) + annotation = _by_alias(annotation) + try: + if annotation in mapping: + return mapping[annotation] + except TypeError: + pass + args = getattr(annotation, "__args__", ()) + if not args: + return annotation + replaced = tuple(_by_subst(arg, mapping) for arg in args) + if replaced == args: + return annotation + origin = getattr(annotation, "__origin__", None) + if origin is None: + return annotation + try: + return origin[replaced] + except TypeError: + return annotation + +def _by_specialize(alias, origin, depth=0): + # the arguments with which `alias` satisfies `origin`, resolved *down the + # declared base chain* rather than assumed to line up positionally. a base + # that fixes or reorders its arguments is then followed faithfully: + # `class Odd[T](list[int])` is a `list[int]` whatever `T` is, and + # `class Swap[A, B](dict[B, A])` specializes `dict` in the other order + if depth > 16: + return None + alias = _by_alias(alias) + klass = getattr(alias, "__origin__", alias) + if not isinstance(klass, type): + return None + args = getattr(alias, "__args__", ()) + params = getattr(klass, "__type_params__", ()) + if not args: + defaulted = _by_type_param_defaults(params) + if defaulted is not params: + args = defaulted + if klass is origin: + return args or None + mapping = {} + for param, arg in zip(params, args): + try: + mapping[param] = arg + except TypeError: + pass + bases = klass.__dict__.get("__orig_bases__") + if bases is None: + # a class inheriting only plain classes records no `__orig_bases__` + bases = getattr(klass, "__bases__", ()) + for base in bases: + found = _by_specialize(_by_subst(base, mapping) if mapping else base, origin, depth + 1) + if found is not None: + return found + # the declared bases don't reach `origin`: a builtin registered as a *virtual* + # subclass of an abc (`list` for `Sequence`) has no base to walk. its + # arguments do line up positionally once membership is established. this runs + # only after resolution has failed, so it applies to the already-resolved base + # (`list[int]`), never to a subclass that fixes or reorders arguments + if args and isinstance(origin, type): + try: + if issubclass(klass, origin): + return args + except TypeError: + pass + return None + +def _by_generic_args(value, origin): + # an explicit `A[int]()` records its specialization on the instance; + # otherwise the class itself is the starting point and any pep 696 defaults + # stand in for the arguments it was constructed with + reified = getattr(value, "__orig_class__", None) + found = _by_specialize(reified if reified is not None else type(value), origin) + return [found] if found is not None else [] + +def _parametric_is(value, alias, variances): + alias = _by_alias(getattr(alias, "__value__", alias)) + origin = getattr(alias, "__origin__", alias) + if not isinstance(value, origin): + return False + target_args = getattr(alias, "__args__", ()) + if len(target_args) != len(variances): + return False + for reified_args in _by_generic_args(value, origin): + if len(reified_args) != len(target_args): + continue + for r, t, v in zip(reified_args, target_args, variances): + if v == 3 or r == t: + continue + if v == 1 and _parametric_is_sub(r, t): + continue + if v == 2 and _parametric_is_sub(t, r): + continue + break + else: + return True + return False + +def _parametric_is_sub(a, b): + if a is b or b is object: + return True + a_origin = getattr(a, "__origin__", a) + b_origin = getattr(b, "__origin__", b) + if isinstance(a_origin, type) and isinstance(b_origin, type) and not getattr(b, "__args__", ()): + try: + return issubclass(a_origin, b_origin) + except TypeError: + return False + return a == b + +def _parametric_is_lenient(value, alias, variances): + # the checked-cast form: a value that records no reification has no + # arguments to check, so the base class test is the whole guarantee. this is + # what keeps `[1, 2] cast list[int]` legal while still rejecting a value + # whose recorded arguments contradict the target + alias = _by_alias(getattr(alias, "__value__", alias)) + origin = getattr(alias, "__origin__", alias) + if not isinstance(value, origin): + return False + if not _by_generic_args(value, origin): + return True + return _parametric_is(value, alias, variances) + + +# --- structural protocol tests --------------------------------------------- + + +_by_proto_missing = object() + +def _by_member_annotation(klass, name): + try: + import typing + hints = typing.get_type_hints(klass) + except Exception: + hints = None + if hints is not None and name in hints: + return hints[name] + for base in klass.__mro__: + annotations = base.__dict__.get("__annotations__", {}) + if name in annotations: + return annotations[name] + return _by_proto_missing + +def _by_lit(*values): + # rebuild `typing.Literal[…]` for a literal type argument (`A[True]` + # specializes `T` to `Literal[True]`). spelled as a call so the member list + # needs no import of its own — this helper ships with the check + import typing + return typing.Literal[values] + +def _by_literal_args(t): + import typing + return typing.get_args(t) if typing.get_origin(t) is typing.Literal else None + +def _by_proto_sub(a, b): + if a is b or b is object: + return True + a_values = _by_literal_args(a) + b_values = _by_literal_args(b) + if a_values is not None: + # `Literal[True]` is a subtype of another literal that lists all its + # values, and of any class every value is an instance of + if b_values is not None: + return all(value in b_values for value in a_values) + return isinstance(b, type) and all(isinstance(value, b) for value in a_values) + if b_values is not None: + # a whole class is never a subtype of a narrower literal + return False + a_origin = getattr(a, "__origin__", a) + b_origin = getattr(b, "__origin__", b) + if isinstance(a_origin, type) and isinstance(b_origin, type) and not getattr(b, "__args__", ()): + try: + return issubclass(a_origin, b_origin) + except TypeError: + return False + return a == b + +def _by_variance_ok(actual, expected, variance): + # 0 invariant (equality), 1 covariant (actual <: expected), + # 2 contravariant (expected <: actual), 3 bivariant (any) + if variance == 3 or actual == expected: + return True + if variance == 1 and _by_proto_sub(actual, expected): + return True + if variance == 2 and _by_proto_sub(expected, actual): + return True + return False + +def _by_method_matches(klass, name, params, ret): + method = getattr(klass, name, None) + if not callable(method): + return False + import inspect, typing + try: + signature = inspect.signature(method) + hints = typing.get_type_hints(method) + except Exception: + return False + positional = [ + p for p in signature.parameters.values() + if p.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) + ] + # drop the receiver (`self` / `cls`) an unbound method still carries + positional = positional[1:] + if len(positional) < len(params): + return False + # extra positional parameters the protocol doesn't supply must be optional, + # else a caller matching the protocol would fail to provide them + for p in positional[len(params):]: + if p.default is inspect.Parameter.empty: + return False + # likewise any required keyword-only parameter would break a protocol call + for p in signature.parameters.values(): + if p.kind == inspect.Parameter.KEYWORD_ONLY and p.default is inspect.Parameter.empty: + return False + for (expected, variance), p in zip(params, positional): + if p.name in hints: + actual = hints[p.name] + elif p.default is not inspect.Parameter.empty: + # a reified default gives the parameter's inferred type at runtime + actual = type(p.default) + else: + return False + if not _by_variance_ok(actual, expected, variance): + return False + if ret is not None: + expected, variance = ret + if "return" not in hints or not _by_variance_ok(hints["return"], expected, variance): + return False + return True + +# a parametric test against a *protocol* target +# (`value is A[int]`). a protocol's instances never record which +# specialization they satisfy, so `__orig_class__` can't answer it — but +# basedpython reifies annotations, so the value's class is checked +# structurally: each protocol member's reified annotation must match the +# member's specialized type. `members` is a list of kind-tagged tuples: +# +# - `("attr", name, expected_type, variance)` — a data member, checked against +# the value class's annotation for `name` +# - `("method", name, [(type, variance), …], return_or_None)` — a method +# member, whose parameters (contravariant) and return (covariant) are checked +# against the value method's reified parameter/return annotations; a +# parameter with no annotation but a default falls back to `type(default)` +# +# `variance` is a code the transpiler picks per argument (0 invariant → equality, 1 +# covariant → subtype, 2 contravariant → supertype, 3 bivariant → any). +# annotations are read with `typing.get_type_hints` (resolving string +# annotations and inherited members), falling back to a raw `__mro__` walk +def _by_protocol_is(value, members): + klass = type(value) + for member in members: + kind = member[0] + if kind == "attr": + _, name, expected, variance = member + actual = _by_member_annotation(klass, name) + if actual is _by_proto_missing: + # the member is *there*, it just carries no annotation any + # runtime can read — python records nothing for a `self.a: int` + # written inside `__init__`. answering `False` would contradict + # the checker, which accepts that class as satisfying the + # protocol, so refuse to answer rather than answer wrongly + if hasattr(value, name): + raise TypeError( + "cannot check `" + klass.__qualname__ + "." + name + + "` against a parameterized protocol: its type is declared " + + "inside a method, and only a class-level annotation " + + "survives to runtime. declare it in the class body" + ) + return False + if not _by_variance_ok(actual, expected, variance): + return False + else: + _, name, params, ret = member + if not _by_method_matches(klass, name, params, ret): + return False + return True + + +# --- template literal types ------------------------------------------------ + + +import re as _by_re + +# matches a value against a template literal type — a pattern such as +# `f"a{int}b"`, whose type is the set of strings it can produce +# +# the regular expression comes from the checker, which builds it from the same +# reading of the pattern's holes that decides the static answer, so the test +# accepts exactly the strings the type contains. a non-`str` value is not one +# of them +def _by_pattern_is(value, pattern): + return isinstance(value, str) and _by_re.fullmatch(pattern, value) is not None + + +# --- deep soundness checks ------------------------------------------------- +# a specialized target validates its base class always, and its reified type +# arguments when the value carries them (`__orig_class__`, stamped by `A[int](…)`). +# a value with no reification passes the argument check — its parameters are not +# available to check, leaving the base `isinstance` as the guarantee + + +def _soundness_parametric(_v, _alias, _variances): + _alias = _by_alias(_alias) + _origin = getattr(_alias, "__origin__", _alias) + if not isinstance(_v, _origin): + raise TypeError( + f"type soundness violation: expected {getattr(_origin, '__name__', _origin)}, " + f"got {type(_v).__name__}" + ) + if getattr(_v, "__orig_class__", None) is not None and not _parametric_is(_v, _alias, _variances): + raise TypeError( + f"type soundness violation: expected {_alias}, got {_v.__orig_class__}" + ) + return _v + + +def _soundness_iter_p(_it, _alias, _variances): + for _x in _it: + yield _soundness_parametric(_x, _alias, _variances) + + +async def _soundness_aiter_p(_it, _alias, _variances): + async for _x in _it: + yield _soundness_parametric(_x, _alias, _variances) + + +# --- reified class generics ------------------------------------------------ +# `A[int]` is a memoized subclass that records its arguments, so an instance +# can be asked what it was specialized to + + +_by_absent = object() + + +# the `generic_class` decorator, for a class that reifies its type parameters +# +# it replaces the class's `__class_getitem__`, so `A[int]` no longer builds a +# `typing` alias but a memoized subclass of `A` carrying the type arguments — +# which is what makes them readable from `__new__` and `__init__` onwards, +# where an `__orig_class__` stamp applied after construction is not yet there. +# being a real subclass also keeps `isinstance(a, A)` and `class B(A[int])` +# working, neither of which survives an alias standing in for a class; the +# specialization declares an empty `__slots__` so a slotted class stays slotted, +# and `__init_subclass__` is held back for it, since it is the same class with +# its arguments fixed rather than a subclass the program wrote +# +# each specialization composes what it binds with what its bases already bound +# and resolves the chain, so `class B[U](A[U])` specialized as `B[int]` answers +# `T` with `int` and not with `U`. `__orig_class__` is carried as a class +# attribute, which is where the alias would have put it, so every reader of a +# runtime specialization — `_parametric_is` included — sees the same thing it +# saw before +# +# `_type_argument` answers one read. it takes the receiver rather than the +# class so a `classmethod` can pass `cls` and everything else `self`, and it +# raises rather than returning the `TypeVar` object the parameter would +# otherwise still name — whether because nothing specialized the class or +# because a base's argument was never filled in +def generic_class(cls): + cls.__class_getitem__ = classmethod(_specialize) + return cls + + +def _specialize(cls, item): + from types import GenericAlias + from typing import TypeVar, TypeVarTuple + + args = item if isinstance(item, tuple) else (item,) + if "__by_type_arguments__" in cls.__dict__: + raise TypeError(f"{cls.__name__} is already specialized") + cache = cls.__dict__.get("__by_specializations__") + if cache is None: + cache = {} + cls.__by_specializations__ = cache + try: + made = cache.get(args) + except TypeError: + raise TypeError( + f"a type argument to {cls.__name__} is not hashable, so the " + f"specialization it names cannot be built" + ) from None + if made is not None: + return made + params = cls.__type_params__ + bound = {} + for base in reversed(cls.__mro__): + bound.update(base.__dict__.get("__by_type_arguments__") or {}) + bound.update(_bind_type_params(params, args, {}, cls.__name__)) + for param in params: + if param.__name__ not in bound: + raise TypeError( + f"too few type arguments for {cls.__name__}: " + f"no argument for {param.__name__!r}" + ) + for name, value in bound.items(): + seen = {name} + while isinstance(value, (TypeVar, TypeVarTuple)) and value.__name__ in bound: + if value.__name__ in seen: + break + seen.add(value.__name__) + value = bound[value.__name__] + bound[name] = value + namespace = { + "__by_type_arguments__": bound, + "__orig_class__": GenericAlias(cls, args), + # the specialization declares nothing of its own, so a slotted class + # stays slotted instead of gaining a `__dict__` here + "__slots__": (), + } + # a specialization is the same class with its arguments fixed, not a + # subclass the program wrote, so the hook that greets a subclass must not + # run for it: it would be handed neither the class keywords the definition + # was given nor a class anybody declared + saved = cls.__dict__.get("__init_subclass__", _by_absent) + cls.__init_subclass__ = classmethod(lambda cls, **kwargs: None) + try: + made = type(cls)(cls.__name__, (cls,), namespace) + except TypeError as exc: + # a metaclass that takes class-creation keywords cannot be given them + # again: nothing records what the definition was written with + raise TypeError( + f"cannot build a specialization of {cls.__name__}: {exc}" + ) from exc + finally: + if saved is _by_absent: + del cls.__init_subclass__ + else: + cls.__init_subclass__ = saved + made.__module__ = cls.__module__ + made.__qualname__ = cls.__qualname__ + cache[args] = made + return made + + +def _type_argument(owner, name): + from typing import TypeVar, TypeVarTuple + + cls = owner if isinstance(owner, type) else type(owner) + bound = getattr(cls, "__by_type_arguments__", None) + value = _by_absent if bound is None else bound.get(name, _by_absent) + # a value still standing as a type parameter is a base's argument that + # nothing filled in, which means the instance came from the bare class + if value is _by_absent or isinstance(value, (TypeVar, TypeVarTuple)): + raise TypeError( + f"{cls.__name__} has no type argument for {name!r}: it was not " + f"constructed from a specialization" + ) + return value + + +# --- lazy imports ---------------------------------------------------------- +# below python 3.15 there is no `lazy` keyword, so an import is rewritten to a +# call: a module import becomes `_lazy_module`, and a `from` import a proxy that +# resolves the attribute the first time anything touches it +# +# `__class__` makes `isinstance(proxy, C)` work, and `__instancecheck__` makes +# `isinstance(x, proxy)` work for a lazily-imported class (`isinstance` looks +# `__instancecheck__` up on `type(classinfo)`, which is `_LazyAttr`). +# `type(proxy)` and `proxy is x` cannot be fixed by any proxy — that is exactly +# why PEP 810 is a language feature — and are documented limits of this +# polyfill + + +import importlib as _by_il, importlib.util as _by_iu, sys as _by_sys +# a relative import names its module against the package doing the importing, +# which only that module knows, so it hands `__package__` in +def _lazy_module(name, package=None): + if package is not None: + name = _by_iu.resolve_name(name, package) + mod = _by_sys.modules.get(name) + if mod is not None: + return mod + if "." in name: + return _by_il.import_module(name) + spec = _by_iu.find_spec(name) + if spec is None or spec.loader is None: + raise ImportError(f"No module named {name!r}", name=name) + spec.loader = _by_iu.LazyLoader(spec.loader) + mod = _by_iu.module_from_spec(spec) + spec.loader.exec_module(mod) + return _by_sys.modules.setdefault(name, mod) + + +def _by_forward_operators(proxy): + import operator as op + def one(f): return lambda s: f(s._by_resolve()) + def two(f): return lambda s, o: f(s._by_resolve(), o) + def rtwo(f): return lambda s, o: f(o, s._by_resolve()) + for n, f in (("add", op.add), ("sub", op.sub), ("mul", op.mul), ("matmul", op.matmul), + ("truediv", op.truediv), ("floordiv", op.floordiv), ("mod", op.mod), + ("divmod", divmod), ("pow", op.pow), ("lshift", op.lshift), + ("rshift", op.rshift), ("and", op.and_), ("xor", op.xor), ("or", op.or_)): + setattr(proxy, "__" + n + "__", two(f)) + setattr(proxy, "__r" + n + "__", rtwo(f)) + for n in ("lt", "le", "eq", "ne", "gt", "ge"): + setattr(proxy, "__" + n + "__", two(getattr(op, n))) + for n, f in (("neg", op.neg), ("pos", op.pos), ("abs", abs), ("invert", op.inv), + ("len", len), ("iter", iter), ("next", next), ("bool", bool), + ("str", str), ("repr", repr), ("bytes", bytes), ("int", int), + ("float", float), ("complex", complex), ("index", op.index), + ("hash", hash), ("reversed", reversed)): + setattr(proxy, "__" + n + "__", one(f)) + setattr(proxy, "__getitem__", two(op.getitem)) + setattr(proxy, "__contains__", two(op.contains)) + setattr(proxy, "__delitem__", two(op.delitem)) + setattr(proxy, "__setitem__", lambda s, k, v: op.setitem(s._by_resolve(), k, v)) + setattr(proxy, "__format__", lambda s, f: format(s._by_resolve(), f)) + setattr(proxy, "__round__", lambda s, *a: round(s._by_resolve(), *a)) + setattr(proxy, "__enter__", lambda s: s._by_resolve().__enter__()) + setattr(proxy, "__exit__", lambda s, *a: s._by_resolve().__exit__(*a)) + +class _LazyAttr: + __slots__ = ("_by_mod", "_by_attr", "_by_val", "_by_has") + def __init__(self, mod, attr): + object.__setattr__(self, "_by_mod", mod) + object.__setattr__(self, "_by_attr", attr) + object.__setattr__(self, "_by_val", None) + object.__setattr__(self, "_by_has", False) + def _by_resolve(self): + if not self._by_has: + m = _lazy_module(self._by_mod) + try: + v = getattr(m, self._by_attr) + except AttributeError: + # a submodule rather than an attribute: `urllib/__init__.py` never + # imports `parse`, and cpython binds it only because `__import__` is + # handed a fromlist. reading the attribute alone never triggers that + try: + v = _by_il.import_module(self._by_mod + "." + self._by_attr) + except ImportError: + # worded as the import machinery words it, down to the module's + # file: a `from x import y` that fails is something programs catch + # and report, so the report must not say where the import was + # written. `name_from` is left off — cpython's own constructor + # only took it from 3.12, and this polyfill runs on 3.9 + p = getattr(m, "__file__", None) + raise ImportError("cannot import name " + repr(self._by_attr) + + " from " + repr(self._by_mod) + + ("" if p is None else " (" + p + ")"), + name=self._by_mod, path=p) from None + object.__setattr__(self, "_by_val", v) + object.__setattr__(self, "_by_has", True) + return self._by_val + @property + def __class__(self): return self._by_resolve().__class__ + def __getattr__(self, k): return getattr(self._by_resolve(), k) + def __setattr__(self, k, v): setattr(self._by_resolve(), k, v) + def __delattr__(self, k): delattr(self._by_resolve(), k) + def __call__(self, *a, **k): return self._by_resolve()(*a, **k) + def __class_getitem__(cls, k): return cls + def __instancecheck__(self, o): return isinstance(o, self._by_resolve()) + def __subclasscheck__(self, o): return issubclass(o, self._by_resolve()) + def __mro_entries__(self, bases): + r = self._by_resolve() + m = getattr(r, "__mro_entries__", None) + if m is None: return (r,) + return m(tuple(r if b is self else b for b in bases)) + +_by_forward_operators(_LazyAttr) + +def _lazy_attr(mod, attr, package=None): + if package is not None: + mod = _by_iu.resolve_name(mod, package) + return _LazyAttr(mod, attr) + + +# a type-only marker for `ty_extensions` names, which have no runtime import +# to make. it supports the type-expression operations the language allows on +# them and nothing else + + +class _TyExtMarker: + def __class_getitem__(cls, k): return cls + + +# `Character` is a concrete `str` subclass, so the grapheme accessors build real +# instances and `isinstance(x, Character)` works. class *identity* is what that +# tests, and a module that pastes the definition in rather than importing it +# would get a class of its own — so the class is interned in a `sys.modules` +# registry and the first definer wins + + +def _by_character_class(): + import sys + import types + + registry = sys.modules.setdefault( + "_by_character_registry", types.ModuleType("_by_character_registry") + ) + if not hasattr(registry, "Character"): + class Character(str): + __slots__ = () + + registry.Character = Character + return registry.Character + + +Character = _by_character_class() + + +# --- discarded returns ----------------------------------------------------- +# the adapter a callable is wrapped in where the site declared one returning +# `None` and the callable returns something else — basedpython's coercion to +# `None`, which the checker resolves as a conversion route +# +# a bare closure would throw the result away just as well. it would also stop +# comparing equal to the callable it wraps, and python deregisters callbacks by +# value all the time — `observers.remove(cb)`, `atexit.unregister(cb)`, +# `signal.disconnect(cb)`. delegating `__eq__` and `__hash__` is what keeps a +# wrapped callback removable; delegating everything else through `__getattr__` +# is what keeps `cb.__name__` answering for a framework that reads it + + +class _by_discard: + __slots__ = ("__wrapped__",) + + def __init__(self, fn): + self.__wrapped__ = fn + + def __call__(self, *args, **kwargs): + self.__wrapped__(*args, **kwargs) + + def __getattr__(self, name): + if name == "__wrapped__": + raise AttributeError(name) + return getattr(self.__wrapped__, name) + + def __eq__(self, other): + if isinstance(other, _by_discard): + other = other.__wrapped__ + return self.__wrapped__ == other + + def __hash__(self): + return hash(self.__wrapped__) + + +# --- match statement ------------------------------------------------------- +# below 3.10 there is no `match`, so one is lowered to an `if`/`elif` chain and +# these answer the questions the chain cannot ask in an expression +# +# `_by_match_miss` stands for "this pattern did not match" where `None` would be +# ambiguous — a subject really can hold `None` + + +_by_match_miss = object() + +# the sequence types are looked up on first use and kept, rather than imported +# when this module loads: `array` and `collections.abc` are of no interest to a +# program with no sequence pattern in it +_by_match_seq_types = None + + +# python decides "is a sequence" by a type flag rather than by an ABC, and sets +# it on a handful of builtins that register no ABC of their own. str, bytes and +# bytearray carry the flag's opposite: they are sequences everywhere else, and +# never match a sequence pattern +def _by_match_seq(subject): + global _by_match_seq_types + if _by_match_seq_types is None: + import array + from collections.abc import Sequence + + _by_match_seq_types = (list, tuple, range, memoryview, array.array, Sequence) + return isinstance(subject, _by_match_seq_types) and not isinstance( + subject, (str, bytes, bytearray) + ) + + +def _by_match_map(subject): + from collections.abc import Mapping + + return isinstance(subject, Mapping) + + +def _by_match_key(subject, key): + try: + return subject[key] + except KeyError: + return _by_match_miss + + +def _by_match_rest(subject, matched): + return {key: value for key, value in subject.items() if key not in matched} + + +# a handful of builtins take one positional sub-pattern that matches the subject +# itself, in place of reading `__match_args__` +_by_match_self = (bool, bytearray, bytes, dict, float, frozenset, int, list, set, str, tuple) + + +def _by_match_args(cls, subject, count): + if cls in _by_match_self: + if count > 1: + raise TypeError(f"{cls.__name__}() accepts 1 positional sub-pattern ({count} given)") + return (subject,) + args = getattr(cls, "__match_args__", ()) + if not isinstance(args, tuple): + raise TypeError(f"{cls.__name__}.__match_args__ must be a tuple (got {type(args).__name__})") + if count > len(args): + raise TypeError(f"{cls.__name__}() accepts {len(args)} positional sub-patterns ({count} given)") + values = [] + for name in args[:count]: + if not isinstance(name, str): + raise TypeError(f"__match_args__ elements must be strings (got {type(name).__name__})") + try: + values.append(getattr(subject, name)) + except AttributeError: + return _by_match_miss + return tuple(values) + + +def _by_match_attr(subject, name): + try: + return getattr(subject, name) + except AttributeError: + return _by_match_miss diff --git a/crates/by_transforms/src/transforms/annotation.rs b/crates/by_transforms/src/transforms/annotation.rs index e6c50a5fde..f1096ad712 100644 --- a/crates/by_transforms/src/transforms/annotation.rs +++ b/crates/by_transforms/src/transforms/annotation.rs @@ -388,7 +388,9 @@ impl TypeAwarePass for TupleLiteralTypePass<'_> { ctx.required_imports .push(format!("{}\n", defs.trim_end_matches('\n'))); } - ctx.required_imports.extend(leaves.take_import_lines()); + let (imports, helpers) = leaves.take_requirements(); + ctx.required_imports.extend(imports); + ctx.runtime.extend(helpers); } } diff --git a/crates/by_transforms/src/transforms/anon_named_tuple.rs b/crates/by_transforms/src/transforms/anon_named_tuple.rs index d981a0b030..580a322cf5 100644 --- a/crates/by_transforms/src/transforms/anon_named_tuple.rs +++ b/crates/by_transforms/src/transforms/anon_named_tuple.rs @@ -895,8 +895,9 @@ impl super::ast_driver::TypeAwarePass for AnonNamedTuplePass<'_> { ctx.required_imports.push(format!("{trimmed}\n")); } } - ctx.required_imports - .extend(inner.callable.take_import_lines()); + let (imports, helpers) = inner.callable.take_requirements(); + ctx.required_imports.extend(imports); + ctx.runtime.extend(helpers); ctx.type_only_imports.extend( inner .type_only_imports @@ -1036,7 +1037,8 @@ mod tests { #[test] fn an_alias_below_its_use_does_not_coerce() { // the table is built in source order, so a name is only an anon-NT alias - // from its declaration down — the same rule the binding itself follows + // from its declaration down — the same rule the binding itself follows. + // the annotation runs before `P` is bound, so it is quoted check( indoc! {" v: P = (\"a\", 1) @@ -1048,7 +1050,7 @@ mod tests { name: str age: int - v: P = (\"a\", 1) + v: \"P\" = (\"a\", 1) P = _AnonNamedTuple_7bfb4772 "}, ); diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index add7ce5cc3..9a4394321b 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -44,12 +44,12 @@ use super::{ if_let, implicit_receiver, implicit_typing, inferred_annotation, init_method, just_float, kw_subscript, literal_string, literal_types, local_once, main_function, match_type, modifiers, module_api, mutable_defaults, none_chain, optional_type, overload, parametric_is, - postfix_await, private_method, propagate, properties, protocol_type, raises_clause, - reified_class, reified_generic, repeated_underscore, return_value_use, runtime_union, sentinel, - some_ctor, soundness, statement_expression, static_resource, string_tag, super_keyword, - symbolic_type_op, template_type, top_star, trailing_lambda, tuple_index, type_fn, type_is, - type_reification, typed_dict_literal, typed_lambda, typeof_keyword, unique_loop_bindings, - unpack, use_site_variance, + postfix_await, propagate, properties, protocol_type, raises_clause, reified_class, + reified_generic, repeated_underscore, return_value_use, runtime_union, sentinel, some_ctor, + soundness, statement_expression, static_resource, string_tag, super_keyword, symbolic_type_op, + template_type, top_star, trailing_lambda, tuple_index, type_fn, type_is, type_reification, + typed_dict_literal, typed_lambda, typeof_keyword, unique_loop_bindings, unpack, + use_site_variance, visibility_rename, }; use crate::Config; use crate::source_map::Replacement; @@ -87,6 +87,11 @@ pub(crate) struct PassContext { /// Full source lines to prepend to the file (e.g. `from typing import cast`). /// Deduped before emission. pub(crate) required_imports: Vec, + /// Runtime helpers the emitted code calls, by the name it calls them under. + /// The driver turns these into an import of the module a build writes them + /// to, or — when there is no such module — into the definitions themselves. + /// See [`crate::runtime`] + pub(crate) runtime: BTreeSet, /// Indices into the *original* module body of statements any pass /// mutated (so the driver knows to re-render them). Indices may /// repeat — the driver dedupes. @@ -154,6 +159,11 @@ pub(crate) trait AstPass { /// to mutate any statement in place, declare hoisted statements, /// and request runtime imports via [`PassContext`]. fn run(&self, module: &mut ModModule, ctx: &mut PassContext); + + /// See [`TypeAwarePass::runtime_only`]. + fn runtime_only(&self) -> bool { + false + } } /// Type-aware pass that reads semantic info from the salsa-owned parsed @@ -162,6 +172,17 @@ pub(crate) trait AstPass { /// because `inferred_type` queries bind to its exact node identities pub(crate) trait TypeAwarePass { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext); + + /// whether everything this pass emits exists for what the code does when + /// it runs — a check, a registration, an entry point — rather than to spell + /// something a checker reads. a stub is read and never run, so the driver + /// leaves such a pass out of a stub's transpile + /// + /// a pass that lowers syntax must never say so: left out, its construct + /// would reach the output as something python cannot parse + fn runtime_only(&self) -> bool { + false + } } /// Adapter: lift a [`Transformer`] (visitor that mutates AST in place) @@ -229,6 +250,19 @@ enum SubPatch { Relocating(Vec), } +/// whether a template spanning `start..end` only wraps it: its one passthrough +/// is the whole span, and everything else it emits is text around it +fn is_wrapper(frags: &[Fragment], start: usize, end: usize) -> bool { + let mut passthroughs = frags.iter().filter_map(|frag| match frag { + Fragment::Src(range) => Some(range), + Fragment::Lit(_) => None, + }); + passthroughs + .next() + .is_some_and(|range| usize::from(range.start()) == start && usize::from(range.end()) == end) + && passthroughs.next().is_none() +} + /// The sub-edits a template materializes, in position order: those nested in /// its own range, plus those its `Src` passthrough spans contain. /// @@ -362,7 +396,7 @@ fn apply_within( /// `from import X, Y, ...` line. Preserves any non-matching /// lines (e.g. `import foo`, `_MISSING = object()`) in their original /// order. Names within a merged line are sorted and deduped -fn merge_from_imports(lines: Vec) -> Vec { +fn merge_from_imports(lines: Vec) -> (Vec, Vec) { // preserve first-seen module order so tests that depend on specific // import sequence (e.g. `from typing import TypeVar, Generic` before // `from typing import Final`) stay stable. names within a module @@ -386,13 +420,13 @@ fn merge_from_imports(lines: Vec) -> Vec { } // `from` imports first (first-seen module order), then raw lines // (synthesized class defs etc.) so any class body referencing imported - // names sees them already in scope - let mut from_lines: Vec = groups + // names sees them already in scope. returned apart, so the runtime can go + // between them + let from_lines: Vec = groups .into_iter() .map(|(module, names)| format!("from {module} import {}", names.join(", "))) .collect(); - from_lines.extend(other); - from_lines + (from_lines, other) } /// Run every registered AST pass against `source` and splice the rewritten @@ -563,7 +597,8 @@ pub(crate) fn run_against_source<'a>( let dedent_string_pass = dedent_string::DedentString::new(source_ref); let super_keyword_pass = super_keyword::SuperKeyword::new(); let postfix_await_pass = postfix_await::PostfixAwait::new(source_ref); - let mutable_defaults_pass = mutable_defaults::MutableDefaultsPass::new(source_ref); + let mutable_defaults_pass = + mutable_defaults::MutableDefaultsPass::new(source_ref, config.is_stub); let unique_loop_bindings_pass = unique_loop_bindings::UniqueLoopBindingsPass::new(source_ref, config.unique_loop_bindings); let auto_quote_pass = auto_quote::AutoQuote::new( @@ -571,7 +606,8 @@ pub(crate) fn run_against_source<'a>( config.min_version, config.inject_future_annotations, ); - let init_method_pass = init_method::InitMethod::new(source_ref, config.float_literals); + let init_method_pass = + init_method::InitMethod::new(source_ref, config.float_literals, config.is_stub); let properties_pass = properties::PropertiesPass::new(source_ref, accessor_value_ranges); let local_once_pass = local_once::LocalOncePass::new(source_ref); let raises_strip_pass = raises_clause::RaisesStripPass::new(source_ref); @@ -583,8 +619,8 @@ pub(crate) fn run_against_source<'a>( raises_clause::RaisesGuardPass::new(source_ref, config.runtime_raises_checks); let type_fn_pass = type_fn::TypeFnPass::new(source_ref); let match_type_pass = match_type::MatchTypePass::new(source_ref); - let modifiers_pass = modifiers::ModifiersPass::new(source_ref); - let main_function_pass = main_function::MainFunction::new(source_ref, config.is_stub); + let modifiers_pass = modifiers::ModifiersPass::new(source_ref, config.is_stub); + let main_function_pass = main_function::MainFunction::new(source_ref); let build_stamps_pass = build_stamps::BuildStampsPass::new(source_ref, config.stamps.clone()); let empty_declarations_pass = empty_declarations::EmptyDeclarations::new(); let overload_pass = overload::Overload::new(source_ref, config.is_stub); @@ -597,11 +633,9 @@ pub(crate) fn run_against_source<'a>( let generic_call_pass = generic_call::GenericCallStripPass::new(source_ref); let reified_generic_pass = reified_generic::ReifiedGenericPass::new(source_ref, config.min_version); - let reified_class_pass = - reified_class::ReifiedClassPass::new(source_ref, config.min_version, config.is_stub); - let type_reification_pass = - type_reification::TypeReificationPass::new(config.min_version, config.is_stub); - let private_method_pass = private_method::PrivateMethodPass; + let reified_class_pass = reified_class::ReifiedClassPass::new(source_ref, config.min_version); + let type_reification_pass = type_reification::TypeReificationPass::new(config.min_version); + let visibility_rename_pass = visibility_rename::VisibilityRenamePass; let parametric_is_pass = parametric_is::ParametricIsPass::new(source_ref); let implicit_typing_pass = implicit_typing::ImplicitTypingPass::new(); let inferred_annotation_pass = inferred_annotation::InferredAnnotationPass::new(); @@ -628,7 +662,7 @@ pub(crate) fn run_against_source<'a>( let destructure_pass = destructure::DestructurePass::new(source_ref); let statement_expression_pass = statement_expression::StatementExpressionPass::new(source_ref); let context_params_pass = context_params::ContextParamsPass::new(source_ref); - let extension_block_pass = extension::ExtensionBlockPass::new(source_ref); + let extension_block_pass = extension::ExtensionBlockPass::new(source_ref, config.is_stub); let extension_call_pass = extension::ExtensionCallPass; let witness_dispatch_pass = conformance::WitnessDispatchPass; let conversion_pass = conversion::ConversionPass::new(source_ref); @@ -672,7 +706,6 @@ pub(crate) fn run_against_source<'a>( &dedent_string_pass, &super_keyword_pass, &postfix_await_pass, - &auto_quote_pass, // strip `local` / `once` parameter modifiers (source-span deletions, // like init_method's `let` handling — must read ranges before any // AST-mutation pass zeroes them) @@ -733,6 +766,9 @@ pub(crate) fn run_against_source<'a>( &static_resource_pass, ]; for pass in passes { + if config.is_stub && pass.runtime_only() { + continue; + } pass.run(&mut module, &mut ctx); } @@ -743,6 +779,10 @@ pub(crate) fn run_against_source<'a>( // order-independent; first, so a hard incompatibility surfaces // before any edit-conflict noise &frameworks_pass, + // forward references are quoted with one wrapper template per span, so + // the lowerings inside an annotation (an arrow, a `T?`) land between the + // quotes wherever they come in the list + &auto_quote_pass, // the `raises` runtime guard is a decorator inserted at the start of the // `def` line, so it composes with every edit inside the signature and // body (the clause deletion among them) @@ -768,7 +808,7 @@ pub(crate) fn run_against_source<'a>( // a `private` method is reached by its mangled name; the edit replaces // the attribute identifier alone, so it composes with any rewrite of the // receiver it is read from - &private_method_pass, + &visibility_rename_pass, // checked cast wraps ` cast? ` in `_checked_cast(...)`; its // template passes value + type through as `Src`, so lowerings inside // them (a `??` value, a `T?` type) still compose @@ -916,6 +956,9 @@ pub(crate) fn run_against_source<'a>( &anon_named_tuple_pass, ]; for pass in type_aware { + if config.is_stub && pass.runtime_only() { + continue; + } pass.run(parsed_handle.suite(), &semantic_model, &mut ctx); } @@ -983,7 +1026,22 @@ pub(crate) fn run_against_source<'a>( ctx.required_imports.sort(); ctx.required_imports.dedup(); - ctx.required_imports = merge_from_imports(std::mem::take(&mut ctx.required_imports)); + let (imports, definitions) = merge_from_imports(std::mem::take(&mut ctx.required_imports)); + ctx.required_imports = imports; + // the runtime goes after the imports and ahead of everything else: it needs + // nothing from the module, and a synthesized class may name one of its + // helpers where it is evaluated at once — `Optional` in the annotation of a + // `NamedTuple` field. never sorted: set-up code follows its definition + if !ctx.runtime.is_empty() { + let helpers = ctx.runtime.iter().copied(); + match config.runtime_module.as_deref() { + Some(module) => ctx + .required_imports + .push(crate::runtime::import_line(module, helpers)), + None => ctx.required_imports.extend(crate::runtime::inline(helpers)), + } + } + ctx.required_imports.extend(definitions); ctx.changed.sort_unstable(); ctx.changed.dedup(); @@ -1115,7 +1173,13 @@ pub(crate) fn run_against_source<'a>( // 3. at one identical span, a *relocating* edit leads: it says the // construct has moved, and the pass that moved it re-emits the span // itself, so every other edit there materializes at the new home - // 4. then a *substitution* — plain text, or a template with no `Src` + // 4. then a *wrapper* — a template whose one passthrough is its whole + // span, adding text around the construct without removing any of it. + // it claims the other edits at that span and materializes them inside + // its passthrough, so whatever the construct becomes ends up inside the + // wrapping (a quoted forward reference around an arrow type the + // callable lowering replaced as text) + // 5. then a *substitution* — plain text, or a template with no `Src` // passthrough — ahead of a *rewrite*, a template that re-emits part of // the span. a substitution says the construct does not appear here at // all, which a rewrite of it cannot outrank @@ -1131,15 +1195,18 @@ pub(crate) fn run_against_source<'a>( }); let statement = i64::from(!matches!(e.2, SubPatch::Statement(_))); let relocating = i64::from(!matches!(e.2, SubPatch::Relocating(_))); + let wraps = i64::from( + !matches!(&e.2, SubPatch::Template(frags) if is_wrapper(frags, e.0, e.1)), + ); // (start, is_replacement_not_insertion, statement-insert-first, - // neg_end-for-wider-first, relocating-first, + // neg_end-for-wider-first, relocating-first, wrapper-first, // substitution-before-rewrite) if e.1 == e.0 { - (e.0, 0i64, statement, 0i64, relocating, rewrites) // insertion + (e.0, 0i64, statement, 0i64, relocating, wraps, rewrites) // insertion } else { #[allow(clippy::cast_possible_wrap)] let neg_end = -(e.1 as i64); - (e.0, 1i64, statement, neg_end, relocating, rewrites) + (e.0, 1i64, statement, neg_end, relocating, wraps, rewrites) } }; priority(a).cmp(&priority(b)) diff --git a/crates/by_transforms/src/transforms/auto_quote.rs b/crates/by_transforms/src/transforms/auto_quote.rs index c99d52b7f4..5608faf06f 100644 --- a/crates/by_transforms/src/transforms/auto_quote.rs +++ b/crates/by_transforms/src/transforms/auto_quote.rs @@ -1,31 +1,44 @@ -//! AST pass: auto-quotes forward self-references in class definitions +//! Type-aware pass: quotes the forward references python would fail to +//! evaluate //! -//! `class A(list[A])` → `class A(list["A"])` +//! basedpython has no manual forward-reference syntax — a string in an +//! annotation is a string-literal *type* — and the checker reads every +//! annotation as deferred, so `def f() -> Later` is fine with `class Later` +//! further down. python before 3.14 evaluates an annotation as its definition +//! runs, and there the same annotation raises `NameError`. this pass quotes +//! each reference that would: //! -//! the class name appearing as a subscript slice argument in base classes or -//! the class body is replaced with a string literal — a PEP 484 forward -//! reference resolvable at runtime without deferred annotation evaluation. +//! `class A: def f(self) -> A` → `def f(self) -> "A"` //! -//! fires when the name is inside a subscript slice; direct bases (`class A(A):`) -//! are left alone — that is a runtime error regardless of quoting. +//! an annotation is evaluated when its definition runs if it is a parameter or +//! return annotation, or annotates a class-body or module-level variable. a +//! local variable's annotation is never evaluated. which names need quoting is +//! ty's to say ([`TypeInfo::is_forward_reference`]): one the program binds, but +//! not by the point the annotation runs — defined further down, the class the +//! annotation sits in, or imported only under `if TYPE_CHECKING:` //! -//! basedpython has no manual forward-reference syntax (a string in an -//! annotation is a string-literal *type*), so the transpiler is the only -//! place these self-references can be made runtime-safe. quoting is skipped -//! when it isn't needed: on python >= 3.14 annotations are deferred natively -//! (PEP 649), and a user-written or opt-in `from __future__ import annotations` -//! already defers every annotation +//! annotation quoting is skipped when annotations are not evaluated eagerly: +//! on python >= 3.14 they are deferred natively (PEP 649), and a user-written +//! or opt-in `from __future__ import annotations` defers every one. a class +//! *base*, and a value-position subscript in a class body (`list[A]()`), +//! evaluates while the class is being built on every version, so a +//! self-reference there is always quoted. a direct base (`class A(A):`) is +//! left alone — that is a runtime error regardless of quoting //! -//! per-class state (class name + PEP-695 typevar names) means each `ClassDef` -//! drives its own walk; the shared [`type_expr_walker`] traverses each -//! class's body + bases identifying type positions, and this pass's visitor -//! checks each one for self-references - -use ruff_python_ast::{Expr, ModModule, PythonVersion, Stmt, StmtClassDef}; +//! an annotation holding a forward reference is quoted whole, as one wrapper +//! template passing its source through. no lowering reaches past the annotation +//! it lowers, so whatever the annotation becomes — a callable arrow, a `T?`, a +//! renamed type parameter, even when a lowering rewrites the whole annotation +//! as text — ends up between the quotes. a base or a value-position subscript +//! cannot be a string, so there only the self-reference itself is quoted + +use ruff_python_ast::visitor::{Visitor, walk_expr}; +use ruff_python_ast::{AnyParameterRef, Expr, ExprName, PythonVersion, Stmt, StmtClassDef}; use ruff_text_size::{Ranged, TextRange}; -use super::ast_driver::{AstPass, PassContext}; +use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_one_type_expr}; +use crate::type_info::TypeInfo; pub(crate) struct AutoQuote<'src> { source: &'src str, @@ -43,19 +56,25 @@ impl<'src> AutoQuote<'src> { } } -impl AstPass for AutoQuote<'_> { - fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { - // annotation positions need no quoting when they won't be eagerly - // evaluated: native deferral on 3.14+ (PEP 649), or a future import - // that defers them all. class *bases* and value-position subscripts - // (`class A(list[A])`, `list[A]()`) evaluate eagerly regardless, so - // their self-references are always quoted +impl TypeAwarePass for AutoQuote<'_> { + // nothing in a stub is evaluated, and a checker reads a forward reference + // in one without quotes + fn runtime_only(&self) -> bool { + true + } + + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let quote_annotations = !(self.min_version.defers_annotations() || self.inject_future - || has_future_annotations(&module.body)); - let mut edits: Vec<(TextRange, String)> = Vec::new(); - process_stmts(&module.body, self.source, &mut edits, quote_annotations); - ctx.text_edits.extend(edits); + || has_future_annotations(stmts)); + let mut walk = Walk { + source: self.source, + types, + quote_annotations, + edits: Vec::new(), + }; + walk.block(stmts, Scope::Module); + ctx.template_edits.extend(walk.edits); } } @@ -67,341 +86,244 @@ fn has_future_annotations(stmts: &[Stmt]) -> bool { }) } -fn process_stmts( - stmts: &[Stmt], - source: &str, - edits: &mut Vec<(TextRange, String)>, - quote_annotations: bool, -) { - for stmt in stmts { - if let Stmt::ClassDef(c) = stmt { - process_class(c, source, edits, quote_annotations); - // nested classes inside this one's body are recursed into by - // process_class so the inner ClassDef walks with its own name - } else { - // top-level non-class statements may contain nested classes via - // function bodies — descend - walk_for_nested_classes(stmt, source, edits, quote_annotations); - } - } +/// the kind of scope a block of statements runs in +#[derive(Clone, Copy, PartialEq, Eq)] +enum Scope { + Module, + Class, + Function, } -fn walk_for_nested_classes( - stmt: &Stmt, - source: &str, - edits: &mut Vec<(TextRange, String)>, +struct Walk<'a> { + source: &'a str, + types: &'a dyn TypeInfo, quote_annotations: bool, -) { - // only walk into structures that may contain nested class defs. - // function bodies, if/while/try blocks, etc. - match stmt { - Stmt::FunctionDef(f) => process_stmts(&f.body, source, edits, quote_annotations), - Stmt::If(i) => { - process_stmts(&i.body, source, edits, quote_annotations); - for clause in &i.elif_else_clauses { - process_stmts(&clause.body, source, edits, quote_annotations); - } - } - Stmt::While(w) => process_stmts(&w.body, source, edits, quote_annotations), - Stmt::For(f) => { - process_stmts(&f.body, source, edits, quote_annotations); - process_stmts(&f.orelse, source, edits, quote_annotations); - } - Stmt::With(w) => process_stmts(&w.body, source, edits, quote_annotations), - Stmt::Try(t) => { - process_stmts(&t.body, source, edits, quote_annotations); - for h in &t.handlers { - let ruff_python_ast::ExceptHandler::ExceptHandler(eh) = h; - process_stmts(&eh.body, source, edits, quote_annotations); - } - process_stmts(&t.orelse, source, edits, quote_annotations); - process_stmts(&t.finalbody, source, edits, quote_annotations); - } - _ => {} - } + edits: Vec<(TextRange, Vec)>, } -fn process_class( - class: &StmtClassDef, - source: &str, - edits: &mut Vec<(TextRange, String)>, - quote_annotations: bool, -) { - let class_name = class.name.id.as_str(); - let typevar_names: Vec = class - .type_params - .as_deref() - .map(|tps| { - tps.type_params - .iter() - .map(|tp| tp.name().id.as_str().to_owned()) - .collect() - }) - .unwrap_or_default(); - - let mut visitor = Visitor { - source, - class_name, - typevars: &typevar_names, - edits, - skip_self_ref_root: false, - }; - - if let Some(args) = &class.arguments { - for base in &args.args { - // a direct `class A(A)` base must not be quoted — that would - // mask a runtime error rather than fix it. for a base, the - // root self-ref is the bare-name direct base; descend into the - // subscript slice / union arms but skip a root self-ref name - visitor.skip_self_ref_root = true; - walk_one_type_expr(base, &mut visitor); - visitor.skip_self_ref_root = false; +impl Walk<'_> { + fn block(&mut self, stmts: &[Stmt], scope: Scope) { + for stmt in stmts { + self.stmt(stmt, scope); } } - // body: AnnAssign annotations + function annotations are type positions - // (handled by walker), method bodies need separate descent for - // `list[A]()` patterns - for stmt in &class.body { - process_class_body_stmt(stmt, &mut visitor, quote_annotations); - } - - // recurse into nested classes inside the body so they get their own - // class-context walk (the `visitor` borrow of `edits` ends above) - process_stmts(&class.body, source, edits, quote_annotations); -} - -fn process_class_body_stmt(stmt: &Stmt, visitor: &mut Visitor<'_>, quote_annotations: bool) { - match stmt { - Stmt::Expr(e) => walk_value_subscripts(e.value.as_ref(), visitor), - Stmt::Assign(a) => walk_value_subscripts(a.value.as_ref(), visitor), - Stmt::AnnAssign(a) => { - if quote_annotations { - visitor.quote_annotation(a.annotation.as_ref()); - } - if let Some(value) = &a.value { - walk_value_subscripts(value.as_ref(), visitor); + fn stmt(&mut self, stmt: &Stmt, scope: Scope) { + match stmt { + Stmt::ClassDef(class) => self.class(class), + Stmt::FunctionDef(function) => { + if self.quote_annotations { + for parameter in function + .parameters + .iter() + .map(AnyParameterRef::as_parameter) + { + if let Some(annotation) = parameter.annotation.as_deref() { + self.annotation(annotation); + } + } + if let Some(returns) = &function.returns { + self.annotation(returns); + } + } + self.block(&function.body, Scope::Function); } - } - Stmt::FunctionDef(f) => { - if !quote_annotations { - return; + // a local variable's annotation is never evaluated + Stmt::AnnAssign(assign) if scope != Scope::Function && self.quote_annotations => { + self.annotation(&assign.annotation); } - for param in f.parameters.iter_non_variadic_params() { - if let Some(ann) = ¶m.parameter.annotation { - visitor.quote_annotation(ann); + Stmt::If(node) => { + self.block(&node.body, scope); + for clause in &node.elif_else_clauses { + self.block(&clause.body, scope); } } - if let Some(var) = &f.parameters.vararg - && let Some(ann) = &var.annotation - { - visitor.quote_annotation(ann); + Stmt::While(node) => { + self.block(&node.body, scope); + self.block(&node.orelse, scope); } - if let Some(kwarg) = &f.parameters.kwarg - && let Some(ann) = &kwarg.annotation - { - visitor.quote_annotation(ann); + Stmt::For(node) => { + self.block(&node.body, scope); + self.block(&node.orelse, scope); } - if let Some(ret) = &f.returns { - visitor.quote_annotation(ret); + Stmt::With(node) => self.block(&node.body, scope), + Stmt::Try(node) => { + self.block(&node.body, scope); + for ruff_python_ast::ExceptHandler::ExceptHandler(handler) in &node.handlers { + self.block(&handler.body, scope); + } + self.block(&node.orelse, scope); + self.block(&node.finalbody, scope); + } + Stmt::Match(node) => { + for case in &node.cases { + self.block(&case.body, scope); + } } + _ => {} + } + } + + fn annotation(&mut self, annotation: &Expr) { + let types = self.types; + let is_forward = |name: &ExprName| types.is_forward_reference(name) == Some(true); + let mut quoter = Quoter { + source: self.source, + is_forward: &is_forward, + edits: &mut self.edits, + skip_root: false, + }; + if quoter.contains_forward_reference(annotation) { + quoter.quote(annotation.range()); } - _ => {} } -} -/// `list[A]()` and similar — quote a self-ref inside a value-position -/// subscript on the LHS of a Call. doesn't descend into Call args -fn walk_value_subscripts(expr: &Expr, visitor: &mut Visitor<'_>) { - match expr { - Expr::Subscript(s) => { - walk_one_type_expr(s.slice.as_ref(), visitor); - walk_value_subscripts(&s.value, visitor); + /// a class's bases and the value-position subscripts in its body run while + /// the class is being built, whatever the version, and the class's own name + /// is not bound until it is + fn class(&mut self, class: &StmtClassDef) { + let class_name = class.name.id.as_str(); + let is_self = |name: &ExprName| name.id.as_str() == class_name; + if let Some(arguments) = &class.arguments { + for base in &arguments.args { + // a direct `class A(A)` base must not be quoted — that would + // mask a runtime error rather than fix it + let mut quoter = Quoter { + source: self.source, + is_forward: &is_self, + edits: &mut self.edits, + skip_root: true, + }; + walk_one_type_expr(base, &mut quoter); + } } - Expr::Call(c) => walk_value_subscripts(&c.func, visitor), - Expr::Attribute(a) => walk_value_subscripts(&a.value, visitor), - _ => {} + for stmt in &class.body { + let value = match stmt { + Stmt::Expr(node) => Some(node.value.as_ref()), + Stmt::Assign(node) => Some(node.value.as_ref()), + Stmt::AnnAssign(node) => node.value.as_deref(), + _ => None, + }; + if let Some(value) = value { + let mut quoter = Quoter { + source: self.source, + is_forward: &is_self, + edits: &mut self.edits, + skip_root: false, + }; + quoter.value_subscripts(value); + } + } + self.block(&class.body, Scope::Class); } } -struct Visitor<'a> { +/// quotes the forward references in one type expression +struct Quoter<'a> { source: &'a str, - class_name: &'a str, - typevars: &'a [String], - edits: &'a mut Vec<(TextRange, String)>, - /// when walking a class base, the root expression must not be quoted - /// even if it's a bare self-ref name — that would mask the runtime error - skip_self_ref_root: bool, + is_forward: &'a dyn Fn(&ExprName) -> bool, + edits: &'a mut Vec<(TextRange, Vec)>, + /// when walking a class base, the root expression must not be quoted even + /// if it's a bare self-reference — that would mask the runtime error + skip_root: bool, } -impl TypeExprVisitor for Visitor<'_> { +impl TypeExprVisitor for Quoter<'_> { fn visit(&mut self, expr: &Expr, _pos: TypePos) -> Recurse { - // class base: the root expression must not be quoted even if it - // contains a self-ref. for a bare-name direct base (`class A(A):`) - // quoting would mask a runtime error; for a subscript/binop, the - // self-ref lives inside and the walker descends into it - if self.skip_self_ref_root { - self.skip_self_ref_root = false; + if std::mem::take(&mut self.skip_root) { return match expr { Expr::Subscript(_) | Expr::BinOp(_) => Recurse::Descend, _ => Recurse::Stop, }; } - if !contains_self_ref(expr, self.class_name) { + if !self.contains_forward_reference(expr) { return Recurse::Stop; } match expr { - // unparenthesized tuple in a subscript slice (the walker - // already passes individual elts; this fires when we somehow - // see the Tuple directly — descend) Expr::Tuple(_) => Recurse::Descend, - // `A | B` arms: quote the whole union since the original - // behaviour collapsed it into one string (`"A | None"`) to - // avoid the runtime `str | NoneType` computation that quoting - // only the self-ref arm would produce - Expr::BinOp(_) => { - self.emit_quote(expr.range()); - Recurse::Stop + // a generic whose base is not itself a forward reference: quote the + // references inside its arguments at their own level + Expr::Subscript(subscript) if !self.contains_forward_reference(&subscript.value) => { + Recurse::Descend } - // `A[T]` where A is the class name — quote the whole subscript - // since the base name itself is a forward reference - Expr::Subscript(s) if is_self_ref_root(&s.value, self.class_name) => { - self.emit_quote(expr.range()); - Recurse::Stop - } - // generic subscript whose base isn't a self-ref: descend into - // the slice so nested self-refs get quoted at their own level - Expr::Subscript(_) => Recurse::Descend, - // bare-name or any other expression containing a self-ref: - // quote whole as a forward reference + // a union is quoted whole: quoting one arm alone would evaluate + // `str | NoneType` at runtime. anything else that holds a reference + // — a name, a generic rooted at one, an arrow type — is quoted whole + // as the reference it is _ => { - self.emit_quote(expr.range()); + self.quote(expr.range()); Recurse::Stop } } } } -impl Visitor<'_> { - /// Quote the self-references in one annotation. - /// - /// A [callable type](super::callable) is basedpython syntax, so a - /// self-reference inside one cannot be quoted where it is written — - /// `"(A) -> None"` is not a python type. The whole annotation is quoted - /// instead, as a pair of boundary insertions: the callable lowering's own - /// replacement sits strictly inside them and still applies, so what ends up - /// between the quotes is the *lowered* `Callable[…]`. Every other - /// annotation keeps the narrow leaf quoting. - fn quote_annotation(&mut self, annotation: &Expr) { - if contains_callable_self_ref(annotation, self.class_name) { - let range = annotation.range(); - self.edits - .push((TextRange::empty(range.start()), "\"".to_owned())); - self.edits - .push((TextRange::empty(range.end()), "\"".to_owned())); - return; +impl Quoter<'_> { + fn contains_forward_reference(&self, expr: &Expr) -> bool { + struct Finder<'a> { + is_forward: &'a dyn Fn(&ExprName) -> bool, + found: bool, } - walk_one_type_expr(annotation, self); - } - - fn emit_quote(&mut self, range: TextRange) { - let raw = &self.source[usize::from(range.start())..usize::from(range.end())]; - // basedpython renames PEP 695 typevars (`T` → `_T`) when polyfilling - // for runtime. quoting a forward-reference verbatim from source would - // capture the pre-rename name and leave it unresolved inside the - // string. apply the rename here so the quoted form stays correct - let body = if self.typevars.is_empty() { - raw.to_owned() - } else { - substitute_typevars(raw, self.typevars) - }; - self.edits.push((range, format!("\"{body}\""))); - } -} - -fn is_self_ref_root(expr: &Expr, class_name: &str) -> bool { - matches!(expr, Expr::Name(n) if n.id.as_str() == class_name) -} - -fn contains_self_ref(expr: &Expr, class_name: &str) -> bool { - match expr { - Expr::Name(n) => n.id.as_str() == class_name, - Expr::Subscript(s) => { - contains_self_ref(&s.value, class_name) || contains_self_ref(&s.slice, class_name) - } - Expr::BinOp(b) => { - contains_self_ref(&b.left, class_name) || contains_self_ref(&b.right, class_name) - } - Expr::Tuple(t) => t.elts.iter().any(|e| contains_self_ref(e, class_name)), - Expr::CallableType(c) => { - c.receiver - .iter() - .any(|receiver| contains_self_ref(receiver, class_name)) - || c.args - .iter() - .any(|argument| contains_self_ref(argument, class_name)) - || contains_self_ref(&c.returns, class_name) + impl<'ast> Visitor<'ast> for Finder<'_> { + fn visit_expr(&mut self, expr: &'ast Expr) { + if self.found { + return; + } + if let Expr::Name(name) = expr + && (self.is_forward)(name) + { + self.found = true; + return; + } + walk_expr(self, expr); + } } - _ => false, + let mut finder = Finder { + is_forward: self.is_forward, + found: false, + }; + finder.visit_expr(expr); + finder.found } -} -/// whether `expr` holds a callable type that names the enclosing class. -/// -/// A callable type is basedpython syntax, so its self-reference cannot be -/// quoted where it is written — `"(A) -> None"` is not a python type. The whole -/// annotation is quoted instead, *after* the callable lowering has rewritten it -/// to `Callable[…]`, which is what [`quote_lowered_annotation`] arranges. -fn contains_callable_self_ref(expr: &Expr, class_name: &str) -> bool { - match expr { - Expr::CallableType(_) => contains_self_ref(expr, class_name), - Expr::Subscript(s) => { - contains_callable_self_ref(&s.value, class_name) - || contains_callable_self_ref(&s.slice, class_name) - } - Expr::BinOp(b) => { - contains_callable_self_ref(&b.left, class_name) - || contains_callable_self_ref(&b.right, class_name) + /// `list[A]()` and similar — quote a self-reference inside a value-position + /// subscript on the LHS of a call. doesn't descend into call arguments + fn value_subscripts(&mut self, expr: &Expr) { + match expr { + Expr::Subscript(subscript) => { + walk_one_type_expr(subscript.slice.as_ref(), self); + self.value_subscripts(&subscript.value); + } + Expr::Call(call) => self.value_subscripts(&call.func), + Expr::Attribute(attribute) => self.value_subscripts(&attribute.value), + _ => {} } - Expr::Tuple(t) => t - .elts - .iter() - .any(|e| contains_callable_self_ref(e, class_name)), - _ => false, } -} -/// Replace each occurrence of `name` with `_name` (the mangled form) when -/// `name` appears as an identifier token. Identifier boundaries are detected -/// against the surrounding bytes — `T` matches `T`, `[T]`, `T |`, but not -/// `Tree` or `_T`. Only matches names that are NOT already prefixed with `_`. -fn substitute_typevars(text: &str, typevars: &[String]) -> String { - let bytes = text.as_bytes(); - let mut out = String::with_capacity(text.len() + typevars.len()); - let mut i = 0; - while i < bytes.len() { - let b = bytes[i]; - let starts_ident = b.is_ascii_alphabetic() || b == b'_'; - let prev_ident = i > 0 && (bytes[i - 1].is_ascii_alphanumeric() || bytes[i - 1] == b'_'); - if starts_ident && !prev_ident { - let mut j = i; - while j < bytes.len() && (bytes[j].is_ascii_alphanumeric() || bytes[j] == b'_') { - j += 1; - } - let ident = &text[i..j]; - if typevars.iter().any(|tv| tv == ident) { - out.push('_'); - out.push_str(ident); - } else { - out.push_str(ident); - } - i = j; - continue; - } - out.push(b as char); - i += 1; + /// wrap `range` in quotes, as one template passing the source through so + /// the lowerings inside it land between the quotes. the delimiter is one the + /// source it wraps does not already use + fn quote(&mut self, range: TextRange) { + let text = &self.source[range]; + let delimiter = ["\"", "'", "\"\"\"", "'''"] + .into_iter() + .find(|delimiter| { + !text.contains(delimiter) + && delimiter + .chars() + .next() + .is_none_or(|quote| !text.ends_with(quote)) + }) + .unwrap_or("\""); + self.edits.push(( + range, + vec![ + Fragment::Lit(delimiter.to_owned()), + Fragment::Src(range), + Fragment::Lit(delimiter.to_owned()), + ], + )); } - out } #[cfg(test)] @@ -419,9 +341,9 @@ mod tests { } /// a callable type is basedpython syntax, so a self-reference inside one - /// cannot be quoted where it is written. the whole annotation is quoted - /// around the *lowered* `Callable[…]` instead — without it the annotation - /// evaluates `Tag` while the class body is still running + /// cannot be quoted where it is written. the quote wraps the arrow, and the + /// callable lowering's `Callable[…]` lands inside it — without it the + /// annotation evaluates `Tag` while the class body is still running #[test] fn a_callable_annotation_naming_its_class_is_quoted() { check( @@ -545,7 +467,7 @@ mod tests { "}, indoc! {" class A(list[\"A\"]): - x: list[\"A\"] = list[\"A\"]() + x: \"list[A]\" = list[\"A\"]() "}, ); } @@ -559,7 +481,7 @@ mod tests { "}, indoc! {" class A(list[\"A\"]): - def method(self, x: list[\"A\"]) -> list[\"A\"]: ... + def method(self, x: \"list[A]\") -> \"list[A]\": ... "}, ); } @@ -608,7 +530,7 @@ mod tests { "}, indoc! {" class Tree: - children: list[\"Tree[int]\"] + children: \"list[Tree[int]]\" "}, ); } @@ -627,6 +549,159 @@ mod tests { ); } + /// a class defined further down is not bound when a signature above it runs, + /// whether the signature is a module-level function's or a method's + #[test] + fn a_class_defined_later_is_quoted() { + check( + indoc! {" + def later() -> Later: + return Later() + + + class Plain: + def other(self, x: Later) -> Later: ... + + + class Later: ... + "}, + indoc! {" + def later() -> \"Later\": + return Later() + + + class Plain: + def other(self, x: \"Later\") -> \"Later\": ... + + + class Later: ... + "}, + ); + } + + /// a name bound by the time the annotation runs is left as it is + #[test] + fn a_class_defined_earlier_is_not_quoted() { + check( + indoc! {" + class Earlier: ... + + + def f(x: Earlier) -> list[Earlier]: ... + + + y: Earlier = Earlier() + "}, + indoc! {" + class Earlier: ... + + + def f(x: Earlier) -> list[Earlier]: ... + + + y: Earlier = Earlier() + "}, + ); + } + + /// a module-level variable annotation runs too + #[test] + fn a_module_level_variable_annotation_is_quoted() { + check( + indoc! {" + x: Later + + + class Later: ... + "}, + indoc! {" + x: \"Later\" + + + class Later: ... + "}, + ); + } + + /// a local variable's annotation is never evaluated + #[test] + fn a_local_variable_annotation_is_not_quoted() { + check( + indoc! {" + def f() -> None: + x: Later = Later() + + + class Later: ... + "}, + indoc! {" + def f() -> None: + x: Later = Later() + + + class Later: ... + "}, + ); + } + + /// an import made only under `if TYPE_CHECKING:` never runs, so the + /// annotation has nothing to find + #[test] + fn a_type_checking_import_is_quoted() { + check( + indoc! {" + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from collections import OrderedDict + + + def f(x: OrderedDict[str, int]) -> None: ... + "}, + indoc! {" + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from collections import OrderedDict + + + def f(x: \"OrderedDict[str, int]\") -> None: ... + "}, + ); + } + + /// the lowerings inside a quoted annotation land between the quotes + #[test] + fn a_lowering_inside_the_reference_is_quoted_with_it() { + check( + indoc! {" + def f(x: Later?) -> list[Later?]: ... + + + class Later: ... + "}, + indoc! {" + def f(x: \"Later | None\") -> \"list[Later | None]\": ... + + + class Later: ... + "}, + ); + } + + /// a name basedpython supplies itself is the transpiler's to make available, + /// not a reference to quote + #[test] + fn a_name_basedpython_supplies_is_not_quoted() { + check( + "def f(x: dynamic) -> None: ...\n", + indoc! {" + from typing import Any + def f(x: Any) -> None: ... + "}, + ); + } + fn transpile_with(input: &str, config: &Config) -> String { transpile(input, config).unwrap() } @@ -692,4 +767,16 @@ mod tests { "should leave the self-ref bare when future is injected, got: {out}" ); } + + /// nothing in a stub is evaluated, and a checker reads a forward reference in + /// one without quotes + #[test] + fn a_stub_quotes_nothing() { + let source = "class A(list[A]):\n def f(self, other: list[A]) -> A: ...\n"; + let config = Config { + is_stub: true, + ..Config::test_default() + }; + assert_eq!(transpile(source, &config).unwrap(), source); + } } diff --git a/crates/by_transforms/src/transforms/callable.rs b/crates/by_transforms/src/transforms/callable.rs index fa50b51b81..e12e31d789 100644 --- a/crates/by_transforms/src/transforms/callable.rs +++ b/crates/by_transforms/src/transforms/callable.rs @@ -24,13 +24,13 @@ use std::fmt::Write as _; use std::hash::{Hash, Hasher}; use ruff_diagnostics::{Edit, Fix}; +use ruff_python_ast::helpers::{is_classvar_marker_id, is_final_marker_id, is_let_marker_id}; use ruff_python_ast::{Expr, ExprCallableType, Stmt, UnaryOp}; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{PassContext, TypeAwarePass}; use super::intersection::{collect_intersect, collect_union, is_intersection_node}; use super::just_float::rewrite_type_expr_with_imports; -use super::wrapped_runtime::OPTIONAL_RUNTIME; use crate::config::FloatLiteralLowering; use crate::type_info::{TypeInfo, UnpackedKwargsLowering}; @@ -134,9 +134,14 @@ impl<'src> CallableSyntax<'src> { &self.protocol_class_defs } - /// The import lines everything this lowerer emitted needs, including the - /// per-leaf rewrites it folded into its own wide replacements. - pub(crate) fn take_import_lines(&mut self) -> Vec { + /// What everything this lowerer emitted needs: import lines, and the names + /// of any runtime helpers, including for the per-leaf rewrites it folded + /// into its own wide replacements. + /// + /// the two come back together because they are one answer: a caller that + /// took the imports and dropped the helpers would emit code calling a name + /// nothing defines + pub(crate) fn take_requirements(&mut self) -> (Vec, Vec) { let mut lines = Vec::new(); for (needed, line) in [ (self.needs_import, "from typing import Callable"), @@ -152,13 +157,16 @@ impl<'src> CallableSyntax<'src> { (self.needs_typeof_import, "from ty_extensions import TypeOf"), (self.needs_not_import, "from ty_extensions import Not"), (self.needs_annotated_import, "from typing import Annotated"), - (self.needs_optional_runtime, OPTIONAL_RUNTIME), ] { if needed { lines.push(line.to_owned()); } } lines.append(&mut self.extra_imports); + let mut helpers = Vec::new(); + if self.needs_optional_runtime { + helpers.push(crate::runtime::OPTIONAL); + } // reset so a second call is a no-op rather than re-emitting every line self.needs_import = false; self.needs_concatenate_import = false; @@ -168,7 +176,7 @@ impl<'src> CallableSyntax<'src> { self.needs_not_import = false; self.needs_annotated_import = false; self.needs_optional_runtime = false; - lines + (lines, helpers) } /// Lower a single type expression to python source: the structural forms @@ -848,7 +856,10 @@ pub(crate) fn lower_type_expr_full( fn synthetic_let_slice(expr: &Expr) -> Option<&Expr> { if let Expr::Subscript(s) = expr { if let Expr::Name(n) = s.value.as_ref() { - if matches!(n.id.as_str(), "__let__" | "__classvar__" | "__final__") { + if is_let_marker_id(n.id.as_str()) + || is_final_marker_id(n.id.as_str()) + || is_classvar_marker_id(n.id.as_str()) + { return Some(s.slice.as_ref()); } } @@ -978,7 +989,9 @@ impl TypeAwarePass for CallableSyntaxPass<'_> { // callable type) — the dedicated leaf passes' own import requests are // dropped along with their edits when our edit wins the overlap, so // they are re-requested here - ctx.required_imports.extend(inner.take_import_lines()); + let (imports, helpers) = inner.take_requirements(); + ctx.required_imports.extend(imports); + ctx.runtime.extend(helpers); let defs = inner.class_defs().to_owned(); for fix in inner.edits { for edit in fix.edits() { diff --git a/crates/by_transforms/src/transforms/checked_cast.rs b/crates/by_transforms/src/transforms/checked_cast.rs index f26717d3af..9053fb3b2d 100644 --- a/crates/by_transforms/src/transforms/checked_cast.rs +++ b/crates/by_transforms/src/transforms/checked_cast.rs @@ -57,39 +57,6 @@ use crate::type_info::{CastCheck, SoundnessCheck, TypeInfo}; /// is evaluated exactly once and the predicate can reference it const CAST_VALUE_PARAM: &str = "_by_cast_value"; -// ` cast! `: verify at runtime, raise on mismatch. -const CHECKED_CAST_HELPER: &str = "\ -def _checked_cast(_v, _t): - if not isinstance(_v, _t): - raise TypeError( - f\"cast to {getattr(_t, '__name__', _t)} failed: value is {type(_v).__name__}\" - ) - return _v -"; - -// ` cast? `: yield the value when it matches, else `None`. -const TRY_CAST_HELPER: &str = "\ -def _try_cast(_v, _t): - return _v if isinstance(_v, _t) else None -"; - -// predicate forms, for any target the shared parametric engine can decide at -// runtime — a reified-cell comparison (`T == int`), an `__orig_class__` probe, a -// structural protocol check, or a disjunction of those across a union's arms. -// the predicate is a lambda so the value is evaluated exactly once (as `_v`) and -// referenced from inside the test -const CHECKED_CAST_PRED_HELPER: &str = "\ -def _checked_cast_pred(_v, _pred): - if not _pred(_v): - raise TypeError(f\"cast failed: value is {type(_v).__name__}\") - return _v -"; - -const TRY_CAST_PRED_HELPER: &str = "\ -def _try_cast_pred(_v, _pred): - return _v if _pred(_v) else None -"; - /// the runtime helper a cast occurrence lowers to #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] enum Helper { @@ -132,13 +99,15 @@ impl Helper { } /// the preamble this helper's call needs - fn runtime(self) -> &'static str { + /// the runtime helper this form calls, or `None` for the one form that + /// calls nothing of ours — `typing.cast`, which is an import + fn runtime(self) -> Option { match self { - Self::Checked => CHECKED_CAST_HELPER, - Self::Try => TRY_CAST_HELPER, - Self::CheckedPredicate => CHECKED_CAST_PRED_HELPER, - Self::TryPredicate => TRY_CAST_PRED_HELPER, - Self::TypingCast => "from typing import cast", + Self::Checked => Some(crate::runtime::CHECKED_CAST), + Self::Try => Some(crate::runtime::TRY_CAST), + Self::CheckedPredicate => Some(crate::runtime::CHECKED_CAST_PRED), + Self::TryPredicate => Some(crate::runtime::TRY_CAST_PRED), + Self::TypingCast => None, } } } @@ -283,10 +252,17 @@ impl TypeAwarePass for CheckedCastPass { // `_parametric_is` / `_by_protocol_is` must precede the predicates that // call them for runtime in inner.runtimes { - ctx.required_imports.push(runtime.source().to_owned()); + ctx.runtime.extend(runtime.helpers()); } for helper in &inner.used { - ctx.required_imports.push(helper.runtime().to_owned()); + match helper.runtime() { + Some(name) => { + ctx.runtime.insert(name); + } + None => ctx + .required_imports + .push("from typing import cast".to_owned()), + } } ctx.template_edits.extend(inner.edits); } diff --git a/crates/by_transforms/src/transforms/conformance.rs b/crates/by_transforms/src/transforms/conformance.rs index aa7d90891a..ed5cd9a5ac 100644 --- a/crates/by_transforms/src/transforms/conformance.rs +++ b/crates/by_transforms/src/transforms/conformance.rs @@ -17,7 +17,7 @@ //! Two things then read that table. A requirement accessed on a receiver the //! checker typed as the interface cannot be a plain attribute — the value may //! be a conforming type that carries no such member — so it goes through -//! [`WITNESS_RUNTIME`]'s dispatcher, which falls back to the attribute when +//! the runtime's dispatcher, which falls back to the attribute when //! nothing registered one. And `x is A` answers from the table first, so a //! conforming value tests positive even though `isinstance` would not. //! @@ -37,94 +37,18 @@ use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::extension::spine_has_optional; use crate::type_info::TypeInfo; -/// the runtime a conformance needs: the registry, the per-member lookup, the -/// `is`-test, and the two dispatchers (a method is fetched and called by the -/// parentheses that already follow the access; a data member is read). +/// The registry helpers a module touching conformances calls. /// -/// three things here are load-bearing and were each a bug before: -/// -/// - **the registry is per *process*, not per module.** every transpiled module -/// carries its own copy of this preamble, so a module-level `{}` would give -/// each one a private registry and a conformance would never be visible to the -/// module that uses it. it is parked in `sys.modules` instead, which is the one -/// namespace every module already shares -/// - **the lookup is per *member*.** walking the MRO for the first class with -/// *any* table would let a base's conformance beat a subclass's own method — -/// the same object answering two ways depending on its static type. whichever -/// comes first in the MRO wins: a table entry for this member, or a class that -/// defines it -/// - **a conformance registers under every interface it implies.** conforming to -/// `Loud(Show)` conforms to `Show`, and a receiver typed as `Show` looks up -/// under `Show` -pub(crate) const WITNESS_RUNTIME: &str = "\ -def _by_registry(): - # one registry per process: each transpiled module carries its own copy of - # this preamble, and a conformance registered by any of them has to be - # visible to all of them. `sys.modules` is the namespace they already share. - # imported inside the function so the lazy-import pass has no statement to - # rewrite - import sys - import types - module = sys.modules.get(\"_by_conformance_registry\") - if module is None: - module = types.ModuleType(\"_by_conformance_registry\") - module.table = {} - sys.modules[\"_by_conformance_registry\"] = module - return module.table - -_by_conformances = _by_registry() - -def _by_conform(interface, cls, witness): - # conforming to an interface conforms to everything it derives, so a - # receiver typed as a supertype finds the same witness - for base in getattr(interface, \"__mro__\", (interface,)): - if base is object or getattr(base, \"__module__\", None) == \"typing\": - continue - _by_conformances.setdefault(base, {}).setdefault(cls, {}).update(witness) - -def _by_witness_entry(value, interface, name): - table = _by_conformances.get(interface) - if table is None: - return None - for cls in type(value).__mro__: - witness = table.get(cls) - if witness is not None and name in witness: - return witness[name] - # a class that defines the member itself answers it, and beats any - # conformance registered further up the mro - if name in cls.__dict__: - return None - return None - -def _by_conforms(value, interface, members=None): - table = _by_conformances.get(interface) - if table is not None: - for cls in type(value).__mro__: - if cls in table: - return True - if members is None: - return isinstance(value, interface) - return all(hasattr(value, name) for name in members) - -def _by_witness(value, interface, name): - function = _by_witness_entry(value, interface, name) - if function is None: - return getattr(value, name) - return lambda *args, **kwargs: function(value, *args, **kwargs) - -def _by_witness_class(value, interface, name): - function = _by_witness_entry(value, interface, name) - if function is None: - return getattr(value, name) - owner = value if isinstance(value, type) else type(value) - return lambda *args, **kwargs: function(owner, *args, **kwargs) - -def _by_witness_get(value, interface, name): - function = _by_witness_entry(value, interface, name) - if function is None: - return getattr(value, name) - return function(value) -"; +/// The whole set at either site: a module that declares a conformance and one +/// that tests one are reading and writing the same registry, so the split is +/// not worth the risk of a site missing the one name it turned out to need +pub(crate) const WITNESS_HELPERS: &[crate::runtime::Helper] = &[ + crate::runtime::CONFORM, + crate::runtime::CONFORMS, + crate::runtime::WITNESS, + crate::runtime::WITNESS_CLASS, + crate::runtime::WITNESS_GET, +]; /// the `from import as ` a cross-module interface /// spelling needs @@ -157,7 +81,7 @@ pub(crate) fn registration_fragments( if registrations.is_empty() { return; } - ctx.required_imports.push(WITNESS_RUNTIME.to_owned()); + ctx.runtime.extend(WITNESS_HELPERS.iter().copied()); let mut first = !had_members; for registration in registrations { if let Some(import) = ®istration.import { @@ -260,7 +184,7 @@ impl TypeAwarePass for WitnessDispatchPass { if inner.edits.is_empty() { return; } - ctx.required_imports.push(WITNESS_RUNTIME.to_owned()); + ctx.runtime.extend(WITNESS_HELPERS.iter().copied()); ctx.required_imports.extend(inner.imports); ctx.template_edits.extend(inner.edits); } @@ -405,4 +329,19 @@ mod tests { "got:\n{out}" ); } + + /// the witness table is registered as the declaring module is imported, and + /// a stub is never imported + #[test] + fn a_stub_registers_no_conformance() { + let out = transpile( + "protocol Show:\n def show(self) -> str\n\nextension str(Show):\n override def show(self) -> str\n", + &Config { + is_stub: true, + ..Config::test_default() + }, + ) + .unwrap(); + assert!(!out.contains("_by_conform"), "got:\n{out}"); + } } diff --git a/crates/by_transforms/src/transforms/conversion.rs b/crates/by_transforms/src/transforms/conversion.rs index 18114f4769..64d5098d4a 100644 --- a/crates/by_transforms/src/transforms/conversion.rs +++ b/crates/by_transforms/src/transforms/conversion.rs @@ -29,7 +29,6 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use ty_python_semantic::{ConversionInfo, ConversionRuntime, PreludeDunderReceiver}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; -use super::wrapped_runtime::discard_return_runtime; use crate::type_info::TypeInfo; /// emit the conversion the checker resolved at every conversion site @@ -142,7 +141,7 @@ impl TypeAwarePass for ConversionPass<'_> { // same adapter still define it once match runtime { Some(ConversionRuntime::DiscardReturn) => { - ctx.required_imports.push(discard_return_runtime()); + ctx.runtime.insert(crate::runtime::DISCARD); } None => {} } diff --git a/crates/by_transforms/src/transforms/enums.rs b/crates/by_transforms/src/transforms/enums.rs index 635a54f835..e5e9e21a9d 100644 --- a/crates/by_transforms/src/transforms/enums.rs +++ b/crates/by_transforms/src/transforms/enums.rs @@ -33,6 +33,7 @@ use ruff_python_parser::parse_unchecked_source; use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; use super::source_util::{is_synthetic_decorator, line_indent, line_start}; +use crate::Config; /// Result of the enum-lowering phase: the rewritten source and an output-line → /// original-`.by`-line table (`None` for generated lines). @@ -45,7 +46,7 @@ pub(crate) struct EnumLowering<'a> { /// Lower every module-level `enum` declaration in `source`. Returns the source /// unchanged (borrowed) when there are no based enums or the source fails to /// parse — in which case the normal pipeline surfaces the parse error. -pub(crate) fn lower(source: &str, min_version: PythonVersion) -> EnumLowering<'_> { +pub(crate) fn lower<'a>(source: &'a str, config: &Config) -> EnumLowering<'a> { let parsed = parse_unchecked_source(source, PySourceType::BasedPython); if !parsed.errors().is_empty() { return borrowed(source); @@ -124,12 +125,17 @@ pub(crate) fn lower(source: &str, min_version: PythonVersion) -> EnumLowering<'_ // a sealed hierarchy is mutually recursive (base methods reference variants // and the union alias; recursive enums reference themselves), so annotations // must be lazy. emit `from __future__ import annotations` and skip the - // user's own leading copy if they wrote one - let future_skip = leading_future_skip(suite, source); + // user's own leading copy if they wrote one. a stub's annotations are never + // evaluated, so it gets neither + let future_skip = if config.is_stub { + None + } else { + leading_future_skip(suite, source) + }; let mut cursor = future_skip.unwrap_or_default(); for enum_def in &enums { out.push_verbatim(source, TextRange::new(cursor, enum_def.range().start())); - emit_enum(&mut out, source, enum_def, &mut imports, min_version); + emit_enum(&mut out, source, enum_def, &mut imports, config); cursor = enum_def.range().end(); } out.push_verbatim(source, TextRange::new(cursor, source.text_len())); @@ -139,7 +145,11 @@ pub(crate) fn lower(source: &str, min_version: PythonVersion) -> EnumLowering<'_ // prologue: the `__future__` import (always first) then the deduplicated // imports the lowered classes need, prepended ahead of the rewritten body - let prologue = format!("from __future__ import annotations\n{}", imports.render()); + let prologue = if config.is_stub { + imports.render() + } else { + format!("from __future__ import annotations\n{}", imports.render()) + }; let mut text = String::with_capacity(prologue.len() + body.len()); let mut line_map = Vec::with_capacity(body_map.len()); for _ in prologue.bytes().filter(|&b| b == b'\n') { @@ -312,7 +322,7 @@ fn emit_enum( source: &str, class: &StmtClassDef, imports: &mut ImportSet, - min_version: PythonVersion, + config: &Config, ) { let (variants, members) = partition(class, source); let name = class.name.as_str(); @@ -333,15 +343,7 @@ fn emit_enum( let vis = enum_visibility_prefix(class, source); emit_plain_enum(out, source, name, vis, &variants, &members, imports); } else { - emit_sealed_hierarchy( - out, - source, - class, - &variants, - &members, - imports, - min_version, - ); + emit_sealed_hierarchy(out, source, class, &variants, &members, imports, config); } // the replaced source range excludes its trailing newline, so the lowered @@ -392,7 +394,7 @@ fn emit_sealed_hierarchy( variants: &[Variant], members: &[&Stmt], imports: &mut ImportSet, - min_version: PythonVersion, + config: &Config, ) { // visibility prefix derived from the class (see `lower`): `private`/`export` // ride through to phase-1's `modifiers` pass on the synthesized base line @@ -447,18 +449,17 @@ fn emit_sealed_hierarchy( // variant subclasses, emitted at module level and attached to the enum for variant in variants { out.push_gen("\n"); - emit_variant_class(out, name, variant, min_version); + emit_variant_class(out, name, variant, config); } } /// Emit one variant as a module-level subclass of the enum and attach it (or, for /// a unit variant, its singleton instance) as `EnumName.Variant`. -fn emit_variant_class( - out: &mut Out, - enum_name: &str, - variant: &Variant, - min_version: PythonVersion, -) { +/// +/// A stub gets the subclass alone. The attachment and the name reset run as the +/// module does, and the enum's body already declares `EnumName.Variant` to a +/// checker +fn emit_variant_class(out: &mut Out, enum_name: &str, variant: &Variant, config: &Config) { // a private module-level name holds the subclass; the public binding is the // attached `EnumName.Variant` let mangled = format!("_{enum_name}_{}", variant.name); @@ -479,14 +480,16 @@ fn emit_variant_class( // returning a name makes both return the original object, which is // what the idiomatic `Enum` lowering of an all-unit enum already does out.push_gen(" def __reduce__(self): return type(self).__qualname__\n"); - emit_variant_name_reset(out, enum_name, &variant.name, &mangled); - out.push_gen(&format!("{enum_name}.{} = {mangled}()\n", variant.name)); + if !config.is_stub { + emit_variant_name_reset(out, enum_name, &variant.name, &mangled); + out.push_gen(&format!("{enum_name}.{} = {mangled}()\n", variant.name)); + } } VariantKind::Tuple => { // `slots=True` is a dataclass option only on python 3.10+; a frozen // dataclass already blocks attribute mutation, so on older targets // we simply omit it - let slots = if min_version >= PythonVersion::PY310 { + let slots = if config.min_version >= PythonVersion::PY310 { ", slots=True" } else { "" @@ -510,8 +513,10 @@ fn emit_variant_class( } } } - emit_variant_name_reset(out, enum_name, &variant.name, &mangled); - out.push_gen(&format!("{enum_name}.{} = {mangled}\n", variant.name)); + if !config.is_stub { + emit_variant_name_reset(out, enum_name, &variant.name, &mangled); + out.push_gen(&format!("{enum_name}.{} = {mangled}\n", variant.name)); + } } } } @@ -1182,4 +1187,43 @@ mod tests { .unwrap_err(); assert!(err.contains("without a default"), "got: {err}"); } + + /// the enum's body declares each variant. attaching it and resetting its name + /// happen as the module runs, and so does the `__future__` import that keeps + /// the annotations naming the variants lazy — a stub never runs + #[test] + fn a_stub_declares_variants_without_attaching_them() { + let out = transpile( + indoc! {" + enum class Shape: + case Circle(radius: int) + case Point + "}, + &Config { + is_stub: true, + ..Config::test_default() + }, + ) + .unwrap(); + assert_eq!( + out, + indoc! {" + from dataclasses import dataclass + from typing import final, ClassVar + class Shape: + Circle: ClassVar[type[_Shape_Circle]] + Point: ClassVar[_Shape_Point] + + @final + @dataclass(frozen=True, slots=True) + class _Shape_Circle(Shape): + radius: int + + class _Shape_Point(Shape): + __slots__ = () + def __repr__(self): return \"Point\" + def __reduce__(self): return type(self).__qualname__ + "} + ); + } } diff --git a/crates/by_transforms/src/transforms/erased_union.rs b/crates/by_transforms/src/transforms/erased_union.rs index 2f3f90eadc..8d2ce380aa 100644 --- a/crates/by_transforms/src/transforms/erased_union.rs +++ b/crates/by_transforms/src/transforms/erased_union.rs @@ -380,4 +380,17 @@ mod tests { "a collision leaves the annotation untouched: {out}" ); } + + /// the reified parameter carries a call's specialization into the body, and a + /// stub has no body to carry it to. it declares the union as written + #[test] + fn a_stub_keeps_the_union() { + let source = "def f(data: list[int] | list[str]) -> None: ...\n"; + let config = Config { + is_stub: true, + min_version: PythonVersion::PY313, + ..Config::test_default() + }; + assert_eq!(transpile(source, &config).unwrap(), source); + } } diff --git a/crates/by_transforms/src/transforms/extension.rs b/crates/by_transforms/src/transforms/extension.rs index 4fb2b93cde..b9ce87ae43 100644 --- a/crates/by_transforms/src/transforms/extension.rs +++ b/crates/by_transforms/src/transforms/extension.rs @@ -100,16 +100,6 @@ fn member_kind(func: &ast::StmtFunctionDef, source: &str) -> ExtensionMemberKind ExtensionMemberKind::Method } -/// Whether `func` is the getter the parser synthesized for a property accessor -/// block, rather than a member the author wrote as a `def`. -fn is_accessor_block_member(func: &ast::StmtFunctionDef, source: &str) -> bool { - func.decorator_list.iter().any(|decorator| { - is_synthetic_decorator(source, decorator) - && matches!(&decorator.expression, Expr::Name(name) - if matches!(name.id.as_str(), "__property__" | "__static_property__")) - }) -} - fn kind_word(kind: ExtensionMemberKind) -> &'static str { match kind { ExtensionMemberKind::Method => "method", @@ -186,11 +176,12 @@ fn parameter_fragments(parameters: &ast::Parameters, fragments: &mut Vec { source: &'a str, + is_stub: bool, } impl<'a> ExtensionBlockPass<'a> { - pub(crate) fn new(source: &'a str) -> Self { - Self { source } + pub(crate) fn new(source: &'a str, is_stub: bool) -> Self { + Self { source, is_stub } } /// lower one extension block to its backing functions, in place. the block @@ -294,7 +285,7 @@ impl<'a> ExtensionBlockPass<'a> { // `return` is nowhere in the source), so it has to be rendered rather // than passed through. same trade-off the properties pass already // makes: a basedpython construct inside an accessor body is not lowered - if is_accessor_block_member(func, source) { + if func.property_construct_range().is_some() { fragments.push(Fragment::Lit(format!( ": {marker}\n{}", super::properties::render_body(&func.body, " ") @@ -322,14 +313,17 @@ impl<'a> ExtensionBlockPass<'a> { } // a conformance extension also registers its witness table, after the - // backing functions its entries name - super::conformance::registration_fragments( - class, - types, - ctx, - &mut fragments, - !first_member, - ); + // backing functions its entries name. the registration runs as the + // declaring module is imported, and a stub is never imported + if !self.is_stub { + super::conformance::registration_fragments( + class, + types, + ctx, + &mut fragments, + !first_member, + ); + } ctx.template_edits.push((class.range, fragments)); } diff --git a/crates/by_transforms/src/transforms/force_unwrap.rs b/crates/by_transforms/src/transforms/force_unwrap.rs index 93055740a7..849ec153e3 100644 --- a/crates/by_transforms/src/transforms/force_unwrap.rs +++ b/crates/by_transforms/src/transforms/force_unwrap.rs @@ -25,25 +25,8 @@ use ruff_python_ast::{Expr, Stmt, UnaryOp}; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{PassContext, TypeAwarePass}; -use super::wrapped_runtime::OPTIONAL_RUNTIME; use crate::type_info::TypeInfo; -// peels one absent layer. a present wrapped value (`Some(x)` → `Optional(x)`) -// yields its inner `.value`; a plain `T | None` yields the value or raises on -// `None`; a result-like `T | E` raises on a `BaseException` value, chaining it -// as `__cause__`. referencing `Optional` means the runtime class is co-injected -// below. -const FORCE_HELPER: &str = "\ -def _force_unwrap(_v): - if isinstance(_v, Optional): - return _v.value - if _v is None: - raise RuntimeError(\"force-unwrap of absent value\") - if isinstance(_v, BaseException): - raise RuntimeError(\"force-unwrap of absent value\") from _v - return _v -"; - struct ForceUnwrap { edits: Vec<(TextRange, String)>, used: bool, @@ -107,8 +90,8 @@ impl TypeAwarePass for ForceUnwrapPass<'_> { if inner.used { // the helper unwraps the `Optional` value wrapper, so its runtime // class must be present (deduped if `Some`/`int??` already added it) - ctx.required_imports.push(OPTIONAL_RUNTIME.to_owned()); - ctx.required_imports.push(FORCE_HELPER.to_owned()); + ctx.runtime.insert(crate::runtime::OPTIONAL); + ctx.runtime.insert(crate::runtime::FORCE_UNWRAP); } ctx.text_edits.extend(inner.edits); } diff --git a/crates/by_transforms/src/transforms/generics.rs b/crates/by_transforms/src/transforms/generics.rs index 24702a08d7..09708a6bed 100644 --- a/crates/by_transforms/src/transforms/generics.rs +++ b/crates/by_transforms/src/transforms/generics.rs @@ -69,6 +69,15 @@ pub(crate) struct GenericPolyfill<'src> { /// ranges of `pending_edits` this pass re-rendered; the driver drops them so /// the stale un-renamed text cannot win the overlap race superseded: Vec, + /// the `TypeVar` definitions a polyfilled `class` / `def` needs, keyed on the + /// start of the line its definition begins on + /// + /// these are statements, so they go through the driver's statement-insert + /// channel rather than an ordinary text edit: that channel leads every other + /// insertion at the same offset, which keeps a decorator another lowering + /// writes there — the `raises` runtime guard on a top-level `def` — below + /// them. a statement between a decorator and its `def` is not python at all + statement_prefixes: Vec<(TextSize, String)>, } #[derive(Default)] @@ -168,6 +177,7 @@ impl<'src> GenericPolyfill<'src> { symbolic_substitutions, pending_edits, superseded: Vec::new(), + statement_prefixes: Vec::new(), } } @@ -728,8 +738,7 @@ impl<'src> GenericPolyfill<'src> { let indent = indent.to_owned(); let prefix = self.dedupe_defs(&defs, &indent); if !prefix.is_empty() { - self.edits - .push(Fix::safe_edit(Edit::insertion(prefix, line_start))); + self.statement_prefixes.push((line_start, prefix)); } // Rename type param references in class body. @@ -793,8 +802,7 @@ impl<'src> GenericPolyfill<'src> { let indent = indent.to_owned(); let prefix = self.dedupe_defs(&defs, &indent); if !prefix.is_empty() { - self.edits - .push(Fix::safe_edit(Edit::insertion(prefix, line_start))); + self.statement_prefixes.push((line_start, prefix)); } // Rename type param references in parameter annotations, return type, and body. @@ -1326,6 +1334,10 @@ impl super::ast_driver::TypeAwarePass for GenericPolyfillPass<'_> { ctx.required_imports .push("from typing import Any".to_owned()); } + for (at, prefix) in std::mem::take(&mut inner.statement_prefixes) { + ctx.statement_inserts + .push((at, vec![super::ast_driver::Fragment::Lit(prefix)])); + } for fix in inner.edits { for edit in fix.edits() { let range = edit.range(); diff --git a/crates/by_transforms/src/transforms/grapheme_string.rs b/crates/by_transforms/src/transforms/grapheme_string.rs index 24c221c591..16fc7da2e7 100644 --- a/crates/by_transforms/src/transforms/grapheme_string.rs +++ b/crates/by_transforms/src/transforms/grapheme_string.rs @@ -53,41 +53,6 @@ use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::coalesce::is_trivially_pure; use crate::type_info::TypeInfo; -/// runtime helper injected when `character_count` / `first` / `last` / -/// `characters` / `character_at` are lowered. splits a string into extended -/// grapheme clusters (one `Character` each) via the `regex` module's `\X` — the only widely -/// available python engine that implements UAX #29 correctly (including ZWJ -/// emoji sequences and regional-indicator flags). `regex` is therefore a -/// runtime dependency of the grapheme surface: if it is missing we raise an -/// actionable error rather than silently miscounting with `list()`, whose -/// code-point split gives a wrong answer for any multi-code-point grapheme -const GRAPHEME_HELPER: &str = "\ -def _by_graphemes(_text): - try: - import regex as _regex - except ImportError as _err: - raise ImportError( - \"basedpython's grapheme string surface (character_count / first / last / \" - \"characters / character_at / ...) needs the 'regex' package: uv add regex\" - ) from _err - return _regex.findall(r\"\\X\", _text) -"; - -/// `s.prefix(n)` — the first `n` grapheme clusters, joined. clamps `n` to `>= 0`, -/// so `prefix(0)` is empty and `prefix(large)` is the whole string -const PREFIX_HELPER: &str = "\ -def _by_prefix(_text, _n): - return \"\".join(_by_graphemes(_text)[:max(0, _n)]) -"; - -/// `s.suffix(n)` — the last `n` grapheme clusters, joined. computed from the -/// front (not `[-n:]`) so `suffix(0)` is empty rather than the whole string -const SUFFIX_HELPER: &str = "\ -def _by_suffix(_text, _n): - _g = _by_graphemes(_text) - return \"\".join(_g[max(0, len(_g) - _n):]) -"; - #[expect( clippy::struct_excessive_bools, reason = "independent which-helpers-to-inject flags, not a state machine" @@ -339,7 +304,7 @@ impl TypeAwarePass for GraphemeStringPass { // the prefix / suffix helpers call `_by_graphemes`, so the base helper // is always injected first when either is used if inner.needs_grapheme_helper { - ctx.required_imports.push(GRAPHEME_HELPER.to_owned()); + ctx.runtime.insert(crate::runtime::GRAPHEMES); } // `Character`-producing accessors construct real instances; import the // name so the lazy-import phase materialises `class Character(str)` @@ -348,10 +313,10 @@ impl TypeAwarePass for GraphemeStringPass { .push("from ty_extensions import Character".to_owned()); } if inner.needs_prefix_helper { - ctx.required_imports.push(PREFIX_HELPER.to_owned()); + ctx.runtime.insert(crate::runtime::PREFIX); } if inner.needs_suffix_helper { - ctx.required_imports.push(SUFFIX_HELPER.to_owned()); + ctx.runtime.insert(crate::runtime::SUFFIX); } ctx.template_edits.extend(inner.template_edits); } diff --git a/crates/by_transforms/src/transforms/init_method.rs b/crates/by_transforms/src/transforms/init_method.rs index a22b68bf84..74748569ba 100644 --- a/crates/by_transforms/src/transforms/init_method.rs +++ b/crates/by_transforms/src/transforms/init_method.rs @@ -22,13 +22,15 @@ use std::cell::RefCell; +use ruff_python_ast::helpers::MemberVisibility; use ruff_python_ast::visitor::{Visitor, walk_stmt}; use ruff_python_ast::{Expr, Parameter, Stmt, StmtFunctionDef}; +use ruff_python_stdlib::basedpython::visibility_rename; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::callable::lower_type_expr_full; -use super::mutable_defaults::parameter_guards; +use super::mutable_defaults::{parameter_guards, undeclarable_error}; use super::source_util::{PrologueStatement, first_body_statement}; use crate::config::FloatLiteralLowering; use crate::type_info::TypeInfo; @@ -36,13 +38,19 @@ use crate::type_info::TypeInfo; pub(crate) struct InitMethod<'src> { source: &'src str, float_literals: FloatLiteralLowering, + is_stub: bool, } impl<'src> InitMethod<'src> { - pub(crate) fn new(source: &'src str, float_literals: FloatLiteralLowering) -> Self { + pub(crate) fn new( + source: &'src str, + float_literals: FloatLiteralLowering, + is_stub: bool, + ) -> Self { Self { source, float_literals, + is_stub, } } } @@ -54,6 +62,7 @@ impl TypeAwarePass for InitMethod<'_> { types, symbolic_substitutions: ctx.symbolic_substitutions.clone(), float_literals: self.float_literals, + is_stub: self.is_stub, edits: RefCell::new(Vec::new()), templates: RefCell::new(Vec::new()), relocating: RefCell::new(Vec::new()), @@ -82,7 +91,7 @@ impl TypeAwarePass for InitMethod<'_> { fn is_acceptable_init_param_modifier(word: &str) -> bool { matches!( word, - "let" | "var" | "private" | "public" | "local" | "once" + "let" | "var" | "private" | "protected" | "public" | "local" | "once" ) } @@ -90,7 +99,7 @@ fn is_acceptable_init_param_modifier(word: &str) -> bool { /// opposed to a `local` / `once` lifetime modifier, which the `local_once` pass /// strips). a prefix carrying none of these is not this transform's to rewrite fn is_init_owned_modifier(word: &str) -> bool { - matches!(word, "let" | "var" | "private" | "public") + matches!(word, "let" | "var" | "private" | "protected" | "public") } struct State<'src> { @@ -101,6 +110,7 @@ struct State<'src> { /// unless it is spliced in here symbolic_substitutions: Vec<(TextRange, String)>, float_literals: FloatLiteralLowering, + is_stub: bool, edits: RefCell>, templates: RefCell)>>, /// the `_MISSING` substitutions, whose defaults the guards re-evaluate @@ -222,7 +232,14 @@ impl State<'_> { }; let name = param.name.as_str(); let declares = words.iter().any(|w| matches!(*w, "let" | "var")); - let is_private = words.contains(&"private"); + let visibility = if words.contains(&"private") { + MemberVisibility::Private + } else if words.contains(&"protected") { + MemberVisibility::Protected + } else { + MemberVisibility::Public + }; + let is_hidden = visibility != MemberVisibility::Public; let is_public = words.contains(&"public"); for word in &words { @@ -232,14 +249,15 @@ impl State<'_> { )); } } - if is_private && is_public { + if is_hidden && is_public { self.error(format!( - "`init` parameter `{name}` cannot be both `private` and `public`" + "`init` parameter `{name}` cannot be both `{keyword}` and `public`", + keyword = visibility.keyword() )); } - if (is_private || is_public) && !declares { + if (is_hidden || is_public) && !declares { self.error(format!( - "`private` / `public` on `init` parameter `{name}` requires `let` or `var`" + "a visibility keyword on `init` parameter `{name}` requires `let` or `var`" )); } @@ -255,13 +273,11 @@ impl State<'_> { return; } - // a `private` attribute is name-mangled (`self.__name`); the - // parameter itself keeps its declared name - let attr = if is_private { - format!("__{name}") - } else { - name.to_owned() - }; + // the attribute's visibility is spelled in its name — `self.__name` + // for `private`, which python name-mangles, `self._name` for + // `protected`. the parameter itself keeps its declared name + let attr = visibility_rename(name, visibility.name_prefix()) + .unwrap_or_else(|| name.to_owned()); let line = if let Some(ann) = ¶m.annotation { let ann_src = self.lower_annotation(ann); format!("self.{attr}: {ann_src} = {name}") @@ -310,7 +326,14 @@ impl State<'_> { sentinels, written, guards, - } = parameter_guards(func, self.types); + undeclarable, + } = parameter_guards(func, self.types, self.is_stub); + if let Some(parameter) = undeclarable.first() { + self.errors + .borrow_mut() + .push(undeclarable_error(func.name.as_str(), parameter)); + return; + } let header_indent = self.line_indent(func.range.start()).to_owned(); let body_indent = format!("{header_indent} "); let mut frags = vec![Fragment::Lit(":".to_owned())]; @@ -459,6 +482,21 @@ mod tests { // modifiers compose with self-omission: self is injected and the mangled // attribute is still emitted + #[test] + fn protected_var_param_prefixes_attribute() { + check( + indoc! {" + class A: + init(protected var a: int) + "}, + indoc! {" + class A: + def __init__(self, a: int): + self._a: int = a + "}, + ); + } + #[test] fn private_var_param_self_omitted() { check( @@ -740,4 +778,28 @@ mod tests { "}, ); } + + /// a stub is never run, so the `__init__` it declares re-evaluates no default + #[test] + fn a_stub_declares_a_default_without_a_guard() { + let out = transpile( + indoc! {" + class C: + init(self, let xs: list[int] = []) + "}, + &Config { + is_stub: true, + ..Config::test_default() + }, + ) + .unwrap(); + assert_eq!( + out, + indoc! {" + class C: + def __init__(self, xs: list[int] = []): + self.xs: list[int] = xs + "} + ); + } } diff --git a/crates/by_transforms/src/transforms/lazy_import.rs b/crates/by_transforms/src/transforms/lazy_import.rs index eb15bedacf..43306d6075 100644 --- a/crates/by_transforms/src/transforms/lazy_import.rs +++ b/crates/by_transforms/src/transforms/lazy_import.rs @@ -4,8 +4,9 @@ //! - **`min_version >= 3.15`** — prepend the `lazy` keyword (PEP 810) //! - **`min_version < 3.15`** — rewrite the statement to call a runtime //! polyfill (`_lazy_module` for module imports, `_lazy_attr` for `from` -//! imports). The polyfill defines helpers in the preamble that wrap -//! `importlib.util.LazyLoader` and a small proxy class +//! imports). The helpers wrap `importlib.util.LazyLoader` and a small proxy +//! class, and live in [`crate::runtime`] with the rest of what the emitted +//! python calls //! //! Both modes skip: //! - `from __future__ import ...` — compiler directive @@ -21,12 +22,27 @@ //! //! A multi-name `import a, b` mixing the two is split, keeping a plain import //! for the names that stay eager. +//! +//! A stub defers nothing — see [`Deferral::Never`]. use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::visitor::{Visitor, walk_stmt}; use ruff_python_ast::{Stmt, StmtImport, StmtImportFrom}; use ruff_text_size::{Ranged, TextRange, TextSize}; +/// How a module-level import is deferred. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Deferral { + /// the PEP 810 `lazy` keyword, which python 3.15 and later parse + Keyword, + /// a call into the runtime's polyfill, for every target before that + Polyfill, + /// not at all. a stub is read by a checker and never executed, so it has + /// no execution to defer, and what a checker reads off each import is the + /// binding it makes. every import stays as written, less any `lazy` + Never, +} + #[expect( clippy::struct_excessive_bools, reason = "independent lazy-import state flags, not a state machine" @@ -46,9 +62,7 @@ pub(crate) struct LazyImport<'src> { /// `BaseException` and never consults `__instancecheck__`, so an exception /// class reached through the proxy raises `TypeError` from the handler eager_names: Vec, - /// True when the target Python version supports PEP 810 (3.15+). When - /// false, the transform uses the runtime polyfill instead - keyword_supported: bool, + deferral: Deferral, pub(crate) edits: Vec, /// True when at least one statement was rewritten to call /// `_lazy_module`; the preamble must define the module helper @@ -70,7 +84,7 @@ pub(crate) struct LazyImport<'src> { impl<'src> LazyImport<'src> { pub(crate) fn new( source: &'src str, - keyword_supported: bool, + deferral: Deferral, eager: &[String], eager_names: &[String], ) -> Self { @@ -79,7 +93,7 @@ impl<'src> LazyImport<'src> { at_module_level: true, eager: eager.to_vec(), eager_names: eager_names.to_vec(), - keyword_supported, + deferral, edits: Vec::new(), needs_module_helper: false, needs_attr_helper: false, @@ -143,19 +157,20 @@ impl<'src> LazyImport<'src> { } fn process_import(&mut self, node: &StmtImport) { - // a module whose execution is the point of the import is never deferred, - // whichever mechanism this target uses - if node - .names - .iter() - .any(|alias| self.is_eager(alias.name.id.as_str())) + // a stub defers nothing, and a module whose execution is the point of the + // import is never deferred, whichever mechanism this target uses + if self.deferral == Deferral::Never + || node + .names + .iter() + .any(|alias| self.is_eager(alias.name.id.as_str())) { if node.is_lazy { self.strip_lazy_keyword(node.range()); } return; } - if self.keyword_supported { + if self.deferral == Deferral::Keyword { if !node.is_lazy { self.insert_lazy_keyword(node.range().start()); } @@ -209,6 +224,12 @@ impl<'src> LazyImport<'src> { } fn process_from(&mut self, node: &StmtImportFrom) { + if self.deferral == Deferral::Never { + if node.is_lazy { + self.strip_lazy_keyword(node.range()); + } + return; + } let is_future = node .module .as_ref() @@ -240,7 +261,7 @@ impl<'src> LazyImport<'src> { return; } - if self.keyword_supported { + if self.deferral == Deferral::Keyword { if is_future || is_star { if node.is_lazy { self.strip_lazy_keyword(node.range()); @@ -336,16 +357,14 @@ impl<'src> LazyImport<'src> { // package. Resolve the relative target at runtime self.needs_module_helper = true; let rel = format!("{dots}{name}"); - lines.push(format!( - "{bind} = _lazy_module(_by_iu.resolve_name(\"{rel}\", __package__))" - )); + lines.push(format!("{bind} = _lazy_module(\"{rel}\", __package__)")); } else if is_relative { // `from .pkg import x` — lazy attribute on the resolved // parent, matching the `from pkg import x` shape self.needs_attr_helper = true; let rel = format!("{dots}{module_part}"); lines.push(format!( - "{bind} = _lazy_attr(_by_iu.resolve_name(\"{rel}\", __package__), \"{name}\")" + "{bind} = _lazy_attr(\"{rel}\", \"{name}\", __package__)" )); } else { self.needs_attr_helper = true; @@ -424,194 +443,36 @@ impl<'ast> Visitor<'ast> for LazyImport<'_> { } } -/// `from x import y` proxy for the polyfill mode (python < 3.15, which has no -/// PEP 810 `lazy` keyword). deferring the *attribute read* is what needs a -/// proxy at all: `_lazy_module` already defers the module's execution, but -/// reading `y` off it would force that execution immediately. -/// -/// the proxy must be *transparent*: python looks special methods up on the -/// type, never through `__getattr__`, so a dunder that isn't forwarded here -/// silently falls back to `object`'s version. that is not a missing feature -/// but a correctness bug — an unforwarded `__eq__` makes `a == b` compare -/// proxy identity and answer `False` for equal values. the forwarding table -/// below is generated in a loop rather than hand-written so the set stays -/// auditable, and each operator is applied to the *resolved* value so -/// python's full binary-op protocol (reflected operands, `NotImplemented`) -/// still runs. -/// -/// `__class__` makes `isinstance(proxy, C)` work, and `__instancecheck__` -/// makes `isinstance(x, proxy)` work for a lazily-imported class (`isinstance` -/// looks `__instancecheck__` up on `type(classinfo)`, which is `_LazyAttr`). -/// `type(proxy)` and `proxy is x` cannot be fixed by any proxy — that is -/// exactly why PEP 810 is a language feature — and are documented limits of -/// this polyfill -const LAZY_ATTR_PROXY: &str = r#"class _LazyAttr: - __slots__ = ("_by_mod", "_by_attr", "_by_val", "_by_has") - def __init__(self, mod, attr): - object.__setattr__(self, "_by_mod", mod) - object.__setattr__(self, "_by_attr", attr) - object.__setattr__(self, "_by_val", None) - object.__setattr__(self, "_by_has", False) - def _by_resolve(self): - if not self._by_has: - m = _lazy_module(self._by_mod) - try: - v = getattr(m, self._by_attr) - except AttributeError: - # a submodule rather than an attribute: `urllib/__init__.py` never - # imports `parse`, and cpython binds it only because `__import__` is - # handed a fromlist. reading the attribute alone never triggers that - try: - v = _by_il.import_module(self._by_mod + "." + self._by_attr) - except ImportError: - # worded as the import machinery words it, down to the module's - # file: a `from x import y` that fails is something programs catch - # and report, so the report must not say where the import was - # written. `name_from` is left off — cpython's own constructor - # only took it from 3.12, and this polyfill runs on 3.9 - p = getattr(m, "__file__", None) - raise ImportError("cannot import name " + repr(self._by_attr) + - " from " + repr(self._by_mod) + - ("" if p is None else " (" + p + ")"), - name=self._by_mod, path=p) from None - object.__setattr__(self, "_by_val", v) - object.__setattr__(self, "_by_has", True) - return self._by_val - @property - def __class__(self): return self._by_resolve().__class__ - def __getattr__(self, k): return getattr(self._by_resolve(), k) - def __setattr__(self, k, v): setattr(self._by_resolve(), k, v) - def __delattr__(self, k): delattr(self._by_resolve(), k) - def __call__(self, *a, **k): return self._by_resolve()(*a, **k) - def __class_getitem__(cls, k): return cls - def __instancecheck__(self, o): return isinstance(o, self._by_resolve()) - def __subclasscheck__(self, o): return issubclass(o, self._by_resolve()) - def __mro_entries__(self, bases): - r = self._by_resolve() - m = getattr(r, "__mro_entries__", None) - if m is None: return (r,) - return m(tuple(r if b is self else b for b in bases)) -def _by_lazy_forward(): - import operator as op - def one(f): return lambda s: f(s._by_resolve()) - def two(f): return lambda s, o: f(s._by_resolve(), o) - def rtwo(f): return lambda s, o: f(o, s._by_resolve()) - for n, f in (("add", op.add), ("sub", op.sub), ("mul", op.mul), ("matmul", op.matmul), - ("truediv", op.truediv), ("floordiv", op.floordiv), ("mod", op.mod), - ("divmod", divmod), ("pow", op.pow), ("lshift", op.lshift), - ("rshift", op.rshift), ("and", op.and_), ("xor", op.xor), ("or", op.or_)): - setattr(_LazyAttr, "__" + n + "__", two(f)) - setattr(_LazyAttr, "__r" + n + "__", rtwo(f)) - for n in ("lt", "le", "eq", "ne", "gt", "ge"): - setattr(_LazyAttr, "__" + n + "__", two(getattr(op, n))) - for n, f in (("neg", op.neg), ("pos", op.pos), ("abs", abs), ("invert", op.inv), - ("len", len), ("iter", iter), ("next", next), ("bool", bool), - ("str", str), ("repr", repr), ("bytes", bytes), ("int", int), - ("float", float), ("complex", complex), ("index", op.index), - ("hash", hash), ("reversed", reversed)): - setattr(_LazyAttr, "__" + n + "__", one(f)) - setattr(_LazyAttr, "__getitem__", two(op.getitem)) - setattr(_LazyAttr, "__contains__", two(op.contains)) - setattr(_LazyAttr, "__delitem__", two(op.delitem)) - setattr(_LazyAttr, "__setitem__", lambda s, k, v: op.setitem(s._by_resolve(), k, v)) - setattr(_LazyAttr, "__format__", lambda s, f: format(s._by_resolve(), f)) - setattr(_LazyAttr, "__round__", lambda s, *a: round(s._by_resolve(), *a)) - setattr(_LazyAttr, "__enter__", lambda s: s._by_resolve().__enter__()) - setattr(_LazyAttr, "__exit__", lambda s, *a: s._by_resolve().__exit__(*a)) -_by_lazy_forward() -def _lazy_attr(mod, attr): return _LazyAttr(mod, attr) -"#; - -/// `Character` — a concrete `str` subclass, so the grapheme accessors can -/// construct genuine instances and `isinstance(x, Character)` works. -/// -/// it is interned in a `sys.modules` registry rather than defined per file -/// because class *identity* is what `isinstance` tests: a plain -/// `class Character(str)` in each transpiled module would make every module's -/// `Character` a distinct class object, so a value built in one module would -/// fail `isinstance(v, Character)` in another. the registry gives every module -/// in a process the one class — first definer wins, the rest reuse it. -/// -/// this keeps transpiled output self-contained, which a shared import would -/// not: `x: Character = "a"` currently needs nothing installed. if `Character` -/// ever moves into a shipped `basedpython.by`, it becomes an ordinary import -/// from that module and this registry goes away -const CHARACTER_CLASS: &str = r#"import types as _by_types -_by_rt = _by_sys.modules.setdefault("_by_runtime", _by_types.ModuleType("_by_runtime")) -if not hasattr(_by_rt, "Character"): - class Character(str): - __slots__ = () - _by_rt.Character = Character -Character = _by_rt.Character -"#; - -/// Preamble snippet defining the runtime helpers used by polyfill-mode -/// lazified imports. Emitted once per file when any lazification fires +/// The runtime helpers a polyfill-mode lazified module calls, by the name it +/// calls them under. What each of those needs in turn — the proxy class, the +/// `sys` binding, the operator forwarding — is settled in [`crate::runtime`], +/// off the calls in their own bodies. #[expect( clippy::fn_params_excessive_bools, - reason = "independent which-helpers-to-emit flags, not a state machine" + reason = "independent which-helpers-are-needed flags, not a state machine" )] -pub(crate) fn polyfill_preamble( +pub(crate) fn polyfill_helpers( needs_module: bool, needs_attr: bool, needs_ty_ext: bool, needs_character_class: bool, -) -> String { - if !needs_module && !needs_attr && !needs_ty_ext && !needs_character_class { - return String::new(); - } - let needs_module = needs_module || needs_attr; - let mut out = String::new(); +) -> Vec { + let mut helpers = Vec::new(); + // each only where the emitted code names it: a `from` import calls + // `_lazy_attr` alone, and what that calls in turn is the runtime's business if needs_module { - out.push_str("import importlib as _by_il, importlib.util as _by_iu, sys as _by_sys\n"); - out.push_str("def _lazy_module(name):\n"); - out.push_str(" mod = _by_sys.modules.get(name)\n"); - out.push_str(" if mod is not None:\n"); - out.push_str(" return mod\n"); - // a dotted submodule can't take the `LazyLoader` path: `find_spec` - // needs the parent package imported, and a frozen alias like - // `collections.abc` (-> `_collections_abc`) fails to re-execute under a - // lazy load. import it eagerly — laziness is still preserved at the - // attribute level, since `_LazyAttr` only calls this on first use - out.push_str(" if \".\" in name:\n"); - out.push_str(" return _by_il.import_module(name)\n"); - out.push_str(" spec = _by_iu.find_spec(name)\n"); - // `find_spec` returns `None` when the module isn't installed; raise - // a clean `ImportError` instead of letting the next line crash with - // `AttributeError: 'NoneType' object has no attribute 'loader'` - out.push_str(" if spec is None or spec.loader is None:\n"); - out.push_str(" raise ImportError(f\"No module named {name!r}\", name=name)\n"); - out.push_str(" spec.loader = _by_iu.LazyLoader(spec.loader)\n"); - out.push_str(" mod = _by_iu.module_from_spec(spec)\n"); - // publishing before `exec_module` leaves a module nothing has executed - // in `sys.modules`, and `exec_module` is not quiet: it opens with - // `import threading`, which through 3.12 reached `functools` and its - // `from collections import namedtuple`. lazifying `collections` then - // handed that shell back and the import failed. which stdlib module - // sits in the window is an accident of the version, so the name is - // claimed only once the module really is lazy, and `setdefault` yields - // to whoever imported it for real while the window was open - out.push_str(" spec.loader.exec_module(mod)\n"); - out.push_str(" return _by_sys.modules.setdefault(name, mod)\n"); - } else if needs_character_class { - // the `Character` registry reads `sys.modules`, so `sys` must be bound - // even when no import was lazified - out.push_str("import sys as _by_sys\n"); + helpers.push(crate::runtime::LAZY_MODULE); } if needs_attr { - out.push_str(LAZY_ATTR_PROXY); + helpers.push(crate::runtime::LAZY_ATTR); } if needs_ty_ext { - // type-only marker for `ty_extensions` imports. Supports the type - // expression operations the language allows on these names without - // attempting a (non-existent) runtime import - out.push_str("class _TyExtMarker:\n"); - out.push_str(" def __class_getitem__(cls, k): return cls\n"); + helpers.push(crate::runtime::TY_EXT_MARKER); } if needs_character_class { - out.push_str(CHARACTER_CLASS); + helpers.push(crate::runtime::CHARACTER); } - out + helpers } #[cfg(test)] @@ -721,7 +582,7 @@ mod tests { ) .unwrap(); assert!( - out.contains("def _lazy_module(name):"), + out.contains("def _lazy_module("), "missing _lazy_module helper in:\n{out}" ); assert!( @@ -864,7 +725,7 @@ mod tests { fn polyfill_relative_submodule() { check_polyfill_body( "from . import x\n", - "x = _lazy_module(_by_iu.resolve_name(\".x\", __package__))\n", + "x = _lazy_module(\".x\", __package__)\n", ); } @@ -872,7 +733,7 @@ mod tests { fn polyfill_relative_attr() { check_polyfill_body( "from .pkg import x\n", - "x = _lazy_attr(_by_iu.resolve_name(\".pkg\", __package__), \"x\")\n", + "x = _lazy_attr(\".pkg\", \"x\", __package__)\n", ); } @@ -880,7 +741,7 @@ mod tests { fn polyfill_relative_double_dot() { check_polyfill_body( "from .. import x\n", - "x = _lazy_module(_by_iu.resolve_name(\"..x\", __package__))\n", + "x = _lazy_module(\"..x\", __package__)\n", ); } @@ -1042,4 +903,33 @@ mod tests { .unwrap(); assert_eq!(py, "import os\n"); } + + /// a stub is read by a checker and never executed. an import in one declares + /// the binding it makes, where a deferred one would declare a call result + #[test] + fn a_stub_keeps_its_imports_as_written() { + let source = indoc! {" + import json + from dataclasses import dataclass + from ty_extensions import Intersection + lazy import csv + "}; + for min_version in [PythonVersion::PY310, PythonVersion::from((3, 15))] { + let config = Config { + is_stub: true, + min_version, + ..cfg_315() + }; + assert_eq!( + transpile(source, &config).unwrap(), + indoc! {" + import json + from dataclasses import dataclass + from ty_extensions import Intersection + import csv + "}, + "for {min_version}" + ); + } + } } diff --git a/crates/by_transforms/src/transforms/main_function.rs b/crates/by_transforms/src/transforms/main_function.rs index 7a0b83ec08..da860039d2 100644 --- a/crates/by_transforms/src/transforms/main_function.rs +++ b/crates/by_transforms/src/transforms/main_function.rs @@ -20,100 +20,13 @@ use ruff_python_ast::{self as ast, CmpOp, Expr, ModModule, Parameters, Stmt, Stm use super::ast_driver::{AstPass, PassContext}; use super::source_util::{is_synthetic_decorator, python_string_literal}; -/// Parses `sys.argv` into `main`'s parameters. Driven by a spec of -/// `(name, converter, kind, required)` tuples emitted from the signature, so -/// the helper itself never introspects the function. -/// -/// each parameter accepts both spellings — a positional slot and a `--name` -/// option — which argparse cannot express with a single argument, so they are -/// registered as two arguments over internal `p` / `o` destinations and -/// merged afterwards. `bool` is a flag pair (`--name` / `--no-name`) and takes -/// no positional slot -/// -/// `_extra` is the converter of `main`'s leading `*rest`, which asks for -/// whatever the interface does not claim. everything declared after it is -/// keyword-only, so the extras are the only positional arguments and bind to -/// `*rest`. `None` means there is no such parameter -const MAIN_ARGS_RUNTIME: &str = r#"def _by_main_args(_fn, _params, _extra=None): - import argparse - - _parser = argparse.ArgumentParser(description=_fn.__doc__) - for _i, (_name, _type, _kind, _required, _choices) in enumerate(_params): - _flags = [f"--{_name.replace('_', '-')}"] - if "_" in _name: - _flags.append(f"--{_name}") - if _type is None: - _parser.add_argument(*_flags, dest=f"o{_i}", action="store_true", default=None) - _parser.add_argument( - *[f"--no-{_flag[2:]}" for _flag in _flags], - dest=f"o{_i}", - action="store_false", - default=None, - ) - continue - if _kind != "keyword": - _parser.add_argument( - f"p{_i}", - metavar=_name, - nargs="?", - type=_type, - default=None, - choices=_choices, - ) - _parser.add_argument( - *_flags, - dest=f"o{_i}", - metavar=_name.upper(), - type=_type, - default=None, - choices=_choices, - ) - if _extra is None: - _parsed = vars(_parser.parse_args()) - _rest = [] - else: - _namespace, _rest = _parser.parse_known_args() - _parsed = vars(_namespace) - # `parse_known_args` hands back what it did not recognise as it was - # written, so the vararg's own annotation is what converts it - _rest = [_extra(_value) for _value in _rest] - _args = [] - _kwargs = {} - _omitted = None - for _i, (_name, _type, _kind, _required, _choices) in enumerate(_params): - _value = _parsed.get(f"o{_i}") - _positional = _parsed.get(f"p{_i}") - if _value is not None and _positional is not None: - _parser.error(f"argument {_name}: given both positionally and as an option") - if _value is None: - _value = _positional - if _value is None: - if _required: - _parser.error(f"the following arguments are required: {_name}") - if _kind == "positional": - _omitted = _name - continue - if _kind == "positional": - if _omitted is not None: - _parser.error(f"argument {_name}: cannot be given without {_omitted}") - _args.append(_value) - else: - _kwargs[_name] = _value - for _value in _rest: - if _omitted is not None: - _parser.error(f"argument {_value}: cannot be given without {_omitted}") - _args.append(_value) - return _args, _kwargs -"#; - pub(crate) struct MainFunction<'src> { source: &'src str, - is_stub: bool, } impl<'src> MainFunction<'src> { - pub(crate) fn new(source: &'src str, is_stub: bool) -> Self { - Self { source, is_stub } + pub(crate) fn new(source: &'src str) -> Self { + Self { source } } /// true when `main` carries the synthetic `private` modifier, which the @@ -128,11 +41,13 @@ impl<'src> MainFunction<'src> { } impl AstPass for MainFunction<'_> { + // the guard runs `main` when the module is executed as a script, which a + // stub never is + fn runtime_only(&self) -> bool { + true + } + fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { - // stubs declare types only; they are never executed as scripts - if self.is_stub { - return; - } let Some(main) = last_top_level_main(&module.body) else { return; }; @@ -152,7 +67,7 @@ impl AstPass for MainFunction<'_> { let call = if params.is_empty() && extra.is_none() { "main()".to_owned() } else { - ctx.required_imports.push(MAIN_ARGS_RUNTIME.to_owned()); + ctx.runtime.insert(crate::runtime::MAIN_ARGS); ctx.epilogue .push(" _by_args, _by_kwargs = _by_main_args(main, [".to_owned()); for param in ¶ms { @@ -873,4 +788,15 @@ mod tests { pass "}); } + + /// a stub is never run as a script, so it declares `main` and nothing calls it + #[test] + fn a_stub_gets_no_entry_point() { + let source = "def main(name: str) -> None: ...\n"; + let config = Config { + is_stub: true, + ..Config::test_default() + }; + assert_eq!(transpile(source, &config).unwrap(), source); + } } diff --git a/crates/by_transforms/src/transforms/match_polyfill.rs b/crates/by_transforms/src/transforms/match_polyfill.rs index 1dca86dcbb..5dc0ee51e9 100644 --- a/crates/by_transforms/src/transforms/match_polyfill.rs +++ b/crates/by_transforms/src/transforms/match_polyfill.rs @@ -68,6 +68,7 @@ use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::source_util::{preamble_offset, temporary_name}; +use crate::Config; /// the version that understands `match` natively; at or above it nothing here runs const MATCH_VERSION: PythonVersion = PythonVersion::PY310; @@ -77,11 +78,11 @@ const MATCH_VERSION: PythonVersion = PythonVersion::PY310; const LOWERING_VERSION: PythonVersion = PythonVersion::PY38; /// the sentinel a helper returns for "this sub-pattern did not match" -const MISS: &str = "_by_match_miss"; +const MISS: &str = crate::runtime::MATCH_MISS.name(); /// Rewrite every `match` statement in `source` for a target that predates them. -pub(crate) fn lower(source: String, min_version: PythonVersion) -> String { - if !(LOWERING_VERSION..MATCH_VERSION).contains(&min_version) { +pub(crate) fn lower(source: String, config: &Config) -> String { + if !(LOWERING_VERSION..MATCH_VERSION).contains(&config.min_version) { return source; } @@ -106,7 +107,7 @@ pub(crate) fn lower(source: String, min_version: PythonVersion) -> String { .iter() .any(|(_, replacement)| replacement.contains(MISS)); let body = apply(&source, edits); - let preamble = preamble(&needs, sentinel); + let preamble = crate::runtime_preamble(config, &helpers(&needs, sentinel)); let at = preamble_offset(&body); format!("{}{preamble}{}", &body[..at], &body[at..]) } @@ -492,93 +493,33 @@ fn disjoin(parts: &[String]) -> String { /// lacks an attribute or a key falls through to the next case — while a subject /// whose class is malformed (a `__match_args__` that is not a tuple of names) /// still raises the `TypeError` python raises for it. -fn preamble(needs: &Needs, sentinel: bool) -> String { - let mut out = String::new(); +/// The runtime helpers a lowered `match` calls, by the name it calls them +/// under. What each needs in turn — the sentinel a lookup returns, the sequence +/// types it tests against — is settled in [`crate::runtime`]. +fn helpers(needs: &Needs, sentinel: bool) -> Vec { + let mut helpers = Vec::new(); + // the chain names the sentinel itself where a pattern compares against it, + // rather than only reaching it through a helper that returns it if sentinel { - out.push_str("_by_match_miss = object()\n"); - } - - if needs.sequence || needs.mapping { - out.push_str("import collections.abc as _by_match_abc\n"); + helpers.push(crate::runtime::MATCH_MISS); } if needs.sequence { - // python decides "is a sequence" by a type flag rather than by an ABC, - // and sets it on a handful of builtins that register no ABC of their - // own. str, bytes and bytearray carry the flag's opposite: they are - // sequences everywhere else, and never match a sequence pattern - out.push_str("import array as _by_match_array\n"); - out.push_str( - "_by_match_seq_types = (list, tuple, range, memoryview, _by_match_array.array, \ - _by_match_abc.Sequence)\n", - ); - out.push_str("def _by_match_seq(subject):\n"); - out.push_str( - " return isinstance(subject, _by_match_seq_types) and not isinstance(subject, \ - (str, bytes, bytearray))\n", - ); + helpers.push(crate::runtime::MATCH_SEQ); } if needs.mapping { - out.push_str("def _by_match_map(subject):\n"); - out.push_str(" return isinstance(subject, _by_match_abc.Mapping)\n"); - out.push_str("def _by_match_key(subject, key):\n"); - out.push_str(" try:\n"); - out.push_str(" return subject[key]\n"); - out.push_str(" except KeyError:\n"); - out.push_str(" return _by_match_miss\n"); + helpers.push(crate::runtime::MATCH_MAP); + helpers.push(crate::runtime::MATCH_KEY); } if needs.mapping_rest { - out.push_str("def _by_match_rest(subject, matched):\n"); - out.push_str( - " return {key: value for key, value in subject.items() if key not in matched}\n", - ); + helpers.push(crate::runtime::MATCH_REST); } if needs.class_positional { - // a handful of builtins take one positional sub-pattern that matches - // the subject itself, in place of reading `__match_args__` - out.push_str( - "_by_match_self = (bool, bytearray, bytes, dict, float, frozenset, int, list, set, \ - str, tuple)\n", - ); - out.push_str("def _by_match_args(cls, subject, count):\n"); - out.push_str(" if cls in _by_match_self:\n"); - out.push_str(" if count > 1:\n"); - out.push_str( - " raise TypeError(f\"{cls.__name__}() accepts 1 positional sub-pattern \ - ({count} given)\")\n", - ); - out.push_str(" return (subject,)\n"); - out.push_str(" args = getattr(cls, \"__match_args__\", ())\n"); - out.push_str(" if not isinstance(args, tuple):\n"); - out.push_str( - " raise TypeError(f\"{cls.__name__}.__match_args__ must be a tuple \ - (got {type(args).__name__})\")\n", - ); - out.push_str(" if count > len(args):\n"); - out.push_str( - " raise TypeError(f\"{cls.__name__}() accepts {len(args)} positional \ - sub-patterns ({count} given)\")\n", - ); - out.push_str(" values = []\n"); - out.push_str(" for name in args[:count]:\n"); - out.push_str(" if not isinstance(name, str):\n"); - out.push_str( - " raise TypeError(f\"__match_args__ elements must be strings \ - (got {type(name).__name__})\")\n", - ); - out.push_str(" try:\n"); - out.push_str(" values.append(getattr(subject, name))\n"); - out.push_str(" except AttributeError:\n"); - out.push_str(" return _by_match_miss\n"); - out.push_str(" return tuple(values)\n"); + helpers.push(crate::runtime::MATCH_ARGS); } if needs.class_keyword { - out.push_str("def _by_match_attr(subject, name):\n"); - out.push_str(" try:\n"); - out.push_str(" return getattr(subject, name)\n"); - out.push_str(" except AttributeError:\n"); - out.push_str(" return _by_match_miss\n"); + helpers.push(crate::runtime::MATCH_ATTR); } - out + helpers } #[cfg(test)] diff --git a/crates/by_transforms/src/transforms/mod.rs b/crates/by_transforms/src/transforms/mod.rs index 0f9bbba945..a3ece30213 100644 --- a/crates/by_transforms/src/transforms/mod.rs +++ b/crates/by_transforms/src/transforms/mod.rs @@ -59,7 +59,6 @@ pub(crate) mod optional_type; pub(crate) mod overload; pub(crate) mod parametric_is; pub(crate) mod postfix_await; -pub(crate) mod private_method; pub(crate) mod propagate; pub(crate) mod properties; pub(crate) mod protocol_type; @@ -93,4 +92,4 @@ pub(crate) mod typing_redirect; pub(crate) mod unique_loop_bindings; pub(crate) mod unpack; pub(crate) mod use_site_variance; -pub(crate) mod wrapped_runtime; +pub(crate) mod visibility_rename; diff --git a/crates/by_transforms/src/transforms/modifiers.rs b/crates/by_transforms/src/transforms/modifiers.rs index 64fd2754ea..9475285aeb 100644 --- a/crates/by_transforms/src/transforms/modifiers.rs +++ b/crates/by_transforms/src/transforms/modifiers.rs @@ -24,17 +24,31 @@ //! `private` → modifier deleted; symbol renamed with `_` prefix and excluded from `__all__` //! `private type X = V` → `type _X = V` (the modifier is a node flag, not a synthetic decorator) -use std::collections::HashMap; - use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::helpers::is_immutable_scalar_default; -use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{Expr, Stmt, StmtAnnAssign, StmtClassDef, StmtFunctionDef, StmtTypeAlias}; -use ruff_python_stdlib::basedpython::private_mangles; +use ruff_python_ast::helpers::{ + DeclarationMarker, DeclarationMarkerKind, MemberVisibility, declaration_marker_visibility, + is_classvar_annot_marker_id, is_classvar_marker_id, is_final_marker_id, is_let_marker_id, +}; +use ruff_python_ast::visitor::{Visitor, walk_stmt}; +use ruff_python_ast::{ + self as ast, Expr, Stmt, StmtAnnAssign, StmtClassDef, StmtFunctionDef, StmtTypeAlias, +}; +use ruff_python_stdlib::basedpython::visibility_rename; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{AstPass, PassContext}; +/// basedpython: the visibility a modifier keyword records, read off the +/// synthetic decorator the parser leaves in its place +fn visibility_of(modifier: &str) -> MemberVisibility { + match modifier { + "private" => MemberVisibility::Private, + "protected" => MemberVisibility::Protected, + _ => MemberVisibility::Public, + } +} + /// whether a dataclass field's default is already a field specifier — /// `field(...)` / `dataclasses.field(...)` — which carries its own factory fn is_field_specifier(default: &Expr) -> bool { @@ -111,6 +125,9 @@ pub(crate) struct Modifiers<'src> { /// a sealed subclass (the runtime tuple assignment lives at module scope /// and cannot reference a function-local name). func_depth: u32, + /// whether the innermost enclosing scope is a class body — where a + /// visibility keyword names a member — rather than a function body + class_body: bool, } impl<'src> Modifiers<'src> { @@ -133,6 +150,7 @@ impl<'src> Modifiers<'src> { class_bases: Vec::new(), class_depth: 0, func_depth: 0, + class_body: false, } } @@ -297,17 +315,18 @@ impl<'src> Modifiers<'src> { self.exports.push(class.name.as_str().to_owned()); } } - "private" => { + "private" | "protected" => { self.edits .push(Fix::safe_edit(Edit::range_deletion(dec.range()))); - if self.class_depth == 0 { + if self.at_module_level() { self.private_renames.push(class.name.as_str().to_owned()); self.rename_with_underscore(class.name.range()); - } else { - // `private` on a nested class member uses Python - // name-mangling (`__name`) so it's hidden from - // subclass scope - self.rename_with_dunder(class.name.range()); + } else if self.class_body { + // a nested class is a member like any other: its + // visibility is spelled in its name, which for `private` + // is the `__name` python name-mangles out of subclass + // scope + self.rename_with_visibility(&class.name, visibility_of(name)); } } _ => {} @@ -367,22 +386,21 @@ impl<'src> Modifiers<'src> { self.exports.push(func.name.as_str().to_owned()); } } - "private" => { + "private" | "protected" => { self.edits .push(Fix::safe_edit(Edit::range_deletion(dec.range()))); - if self.class_depth == 0 { + if self.at_module_level() { self.private_renames.push(func.name.as_str().to_owned()); self.rename_with_underscore(func.name.range()); - } else if private_mangles(func.name.as_str()) { - // a `private` method is name-mangled to `__name`. a - // dunder is left alone: python calls it by its exact - // name, so renaming would change what the method *is* - // rather than who can reach it — and python's own - // mangling rule skips a name with two trailing - // underscores anyway. the one dunder where `private` - // says something, `__init__`, is checked by ty at the + } else if self.class_body { + // a `private` method is name-mangled to `__name`, a + // `protected` one renamed to `_name`. a dunder is left + // alone: python calls it by its exact name, so renaming + // would change what the method *is* rather than who can + // reach it. the one dunder where the keyword says + // something, `__init__`, is checked by ty at the // construction site instead - self.rename_with_dunder(func.name.range()); + self.rename_with_visibility(&func.name, visibility_of(name)); } } _ => {} @@ -399,16 +417,20 @@ impl<'src> Modifiers<'src> { ))); } - /// Replace the identifier at `range` with a double-underscore-prefixed - /// copy. Used for `private` class members so Python's name-mangling - /// applies and the symbol is hidden from subclass scope + /// Replace `name` with the spelling `visibility` gives it. A no-op for a name + /// the rename would not hide — see [`visibility_rename`]. /// - /// Only call this for a name [`private_mangles`] accepts. - fn rename_with_dunder(&mut self, range: TextRange) { - let original = self.src(range).to_owned(); + /// The decision is made on the name the definition *has*, not on the source + /// the range covers: an `init(…)` shorthand is named `__init__` while its + /// range covers the `init` keyword, and renaming that would leave the class + /// with no constructor. + fn rename_with_visibility(&mut self, name: &ast::Identifier, visibility: MemberVisibility) { + let Some(renamed) = visibility_rename(name.as_str(), visibility.name_prefix()) else { + return; + }; self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("__{original}"), - range, + renamed, + name.range(), ))); } @@ -457,7 +479,21 @@ impl<'src> Modifiers<'src> { if !node.target.is_name_expr() { return; } - let name = self.src(node.target.range()).to_owned(); + let written = self.src(node.target.range()).to_owned(); + // a declaration's visibility is spelled in its name, so every branch + // below emits the renamed one + let visibility = declaration_marker_visibility(node.annotation.as_ref()); + let name = if self.class_body { + visibility_rename(&written, visibility.name_prefix()).unwrap_or(written) + } else if visibility == MemberVisibility::Private && self.at_module_level() { + // a module-level `private` variable is renamed the way a module-level + // `private def` is, and for the same reason. its references are + // renamed by `visibility_rename`, which asks ty where each one resolves + self.private_renames.push(written.clone()); + module_private_name(&written) + } else { + written + }; // every rewrite below replaces what the statement said ahead of the value, // which starts past any decorators written above it — those belong to the // decorated-binding lowering, which erases them and wraps the value @@ -468,11 +504,15 @@ impl<'src> Modifiers<'src> { // the modifier prefix; the rest of the statement stays exactly as written, // with or without an initializer if let Expr::Subscript(s) = node.annotation.as_ref() - && matches!(s.value.as_ref(), Expr::Name(n) if matches!(n.id.as_str(), "__abstract_annot__" | "__visibility_annot__" | "__private_annot__" | "__modifier_annot__")) + && matches!(s.value.as_ref(), Expr::Name(n) if DeclarationMarker::from_id(n.id.as_str()).is_some_and(|marker| marker.kind == DeclarationMarkerKind::Annot)) { - let erase_range = TextRange::new(stmt_start, node.target.range().start()); - self.edits - .push(Fix::safe_edit(Edit::range_deletion(erase_range))); + // one edit spanning the prefix *and* the name: a `private` member is + // emitted under a different name, and a deletion alone would leave + // the written one behind + self.edits.push(Fix::safe_edit(Edit::range_replacement( + name, + TextRange::new(stmt_start, node.target.range().end()), + ))); return; } @@ -480,7 +520,7 @@ impl<'src> Modifiers<'src> { // valueless typed `let x: T` / `final x: T` → `x: Final[T]` — a // read-only declaration with no initializer, `Final` in every scope if let Expr::Subscript(s) = node.annotation.as_ref() - && matches!(s.value.as_ref(), Expr::Name(n) if n.id.as_str() == "__classvar_annot__") + && matches!(s.value.as_ref(), Expr::Name(n) if is_classvar_annot_marker_id(n.id.as_str())) { // valueless `class var a: T` → `a: ClassVar[T]` let slice = s.slice.as_ref(); @@ -495,7 +535,7 @@ impl<'src> Modifiers<'src> { slice.range().end(), ))); } else if let Expr::Subscript(s) = node.annotation.as_ref() - && matches!(s.value.as_ref(), Expr::Name(n) if matches!(n.id.as_str(), "__let__" | "__final__")) + && matches!(s.value.as_ref(), Expr::Name(n) if is_let_marker_id(n.id.as_str()) || is_final_marker_id(n.id.as_str())) { let slice = s.slice.as_ref(); // start at the statement, not the marker, so any modifier prefix @@ -510,7 +550,7 @@ impl<'src> Modifiers<'src> { "]".to_owned(), slice.range().end(), ))); - } else if matches!(node.annotation.as_ref(), Expr::Name(n) if n.id.as_str() == "__let__") + } else if matches!(node.annotation.as_ref(), Expr::Name(n) if is_let_marker_id(n.id.as_str())) { // valueless untyped `let x` → `x: Final` — an uninitialized // read-only declaration @@ -529,20 +569,22 @@ impl<'src> Modifiers<'src> { let value_range = self.value_range(node, value); let prefix_range = TextRange::new(stmt_start, value_range.start()); match ann.id.as_str() { - "__let__" => { + id if is_let_marker_id(id) => { self.needs_final_annotation = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( format!("{name}: Final = "), prefix_range, ))); } - "__modifier_assign__" => { + id if DeclarationMarker::from_id(id) + .is_some_and(|marker| marker.kind == DeclarationMarkerKind::Assign) => + { self.edits.push(Fix::safe_edit(Edit::range_replacement( format!("{name} = "), prefix_range, ))); } - "__classvar__" => { + id if is_classvar_marker_id(id) => { self.needs_classvar = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( format!("{name}: ClassVar = "), @@ -560,7 +602,7 @@ impl<'src> Modifiers<'src> { _ => {} } } - Expr::Subscript(s) if matches!(s.value.as_ref(), Expr::Name(n) if n.id.as_str() == "__classvar_annot__") => + Expr::Subscript(s) if matches!(s.value.as_ref(), Expr::Name(n) if is_classvar_annot_marker_id(n.id.as_str())) => { // `class var a: T [= v]` → `a: ClassVar[T] [= v]` let slice = s.slice.as_ref(); @@ -577,14 +619,14 @@ impl<'src> Modifiers<'src> { post_range, ))); } - Expr::Subscript(s) if matches!(s.value.as_ref(), Expr::Name(n) if matches!(n.id.as_str(), "__let__" | "__final__")) => + Expr::Subscript(s) if matches!(s.value.as_ref(), Expr::Name(n) if is_let_marker_id(n.id.as_str()) || is_final_marker_id(n.id.as_str())) => { // typed: `let a: T = v` / `final a: T = v` — annotation is // Subscript(__let__|__final__, T). callable transform visits only // the slice independently, so emit bracket edits around the slice // range; they don't overlap with callable's edit let is_final = - matches!(s.value.as_ref(), Expr::Name(n) if n.id.as_str() == "__final__"); + matches!(s.value.as_ref(), Expr::Name(n) if is_final_marker_id(n.id.as_str())); let slice = s.slice.as_ref(); // start at the statement, not the marker, so any modifier prefix // ahead of `let` (e.g. `override let a: T = v`) is erased too @@ -654,7 +696,9 @@ impl<'ast> Visitor<'ast> for Modifiers<'_> { // Walk the class body with `class_depth` incremented so nested // declarations are not treated as module-level for visibility purposes. self.class_depth += 1; + let enclosing = std::mem::replace(&mut self.class_body, true); walk_stmt(self, stmt); + self.class_body = enclosing; self.class_depth -= 1; return; } @@ -663,7 +707,9 @@ impl<'ast> Visitor<'ast> for Modifiers<'_> { // Walk the body with `func_depth` incremented so a class defined // inside the function is not treated as a module-level subclass. self.func_depth += 1; + let enclosing = std::mem::replace(&mut self.class_body, false); walk_stmt(self, stmt); + self.class_body = enclosing; self.func_depth -= 1; return; } @@ -679,7 +725,6 @@ impl<'ast> Visitor<'ast> for Modifiers<'_> { } } -/// renames all `Name` expression nodes that match a `private`-renamed symbol /// The name a module-level `private` symbol is emitted under: one leading /// underscore, which is python's own mark for "not part of the interface". /// @@ -687,57 +732,21 @@ impl<'ast> Visitor<'ast> for Modifiers<'_> { /// `__name`, and python name-mangles every `__name` it reads inside a class /// body — so `private def _has_room` would become `__has_room` at module level /// and be looked up as `_Diagnostics__has_room` from a method that calls it. -fn module_private_name(name: &str) -> String { +pub(crate) fn module_private_name(name: &str) -> String { if name.starts_with('_') { return name.to_owned(); } format!("_{name}") } -pub(crate) struct NameRenamer { - renames: HashMap, - edits: Vec, -} - -impl NameRenamer { - fn new(private_names: &[String]) -> Self { - let renames = private_names - .iter() - .map(|n| (n.clone(), module_private_name(n))) - .collect(); - Self { - renames, - edits: Vec::new(), - } - } -} - -impl<'ast> Visitor<'ast> for NameRenamer { - fn visit_stmt(&mut self, stmt: &'ast Stmt) { - walk_stmt(self, stmt); - } - - fn visit_expr(&mut self, expr: &'ast Expr) { - if let Expr::Name(n) = expr { - if let Some(new_name) = self.renames.get(n.id.as_str()) { - self.edits.push(Fix::safe_edit(Edit::range_replacement( - new_name.clone(), - expr.range(), - ))); - return; - } - } - walk_expr(self, expr); - } -} - pub(crate) struct ModifiersPass<'src> { source: &'src str, + is_stub: bool, } impl<'src> ModifiersPass<'src> { - pub(crate) fn new(source: &'src str) -> Self { - Self { source } + pub(crate) fn new(source: &'src str, is_stub: bool) -> Self { + Self { source, is_stub } } } @@ -749,7 +758,13 @@ impl AstPass for ModifiersPass<'_> { } let exports = std::mem::take(&mut inner.exports); let private_renames = std::mem::take(&mut inner.private_renames); - let sealed_classes = std::mem::take(&mut inner.sealed_classes); + // the members tuple is assigned as the module runs, and a stub never + // runs. it declares the classes, which is all a checker reads of them + let sealed_classes = if self.is_stub { + Vec::new() + } else { + std::mem::take(&mut inner.sealed_classes) + }; let class_bases = std::mem::take(&mut inner.class_bases); // typing import grouping mirrors lib.rs's preamble logic @@ -798,22 +813,9 @@ impl AstPass for ModifiersPass<'_> { } } - // 2nd-pass NameRenamer rewrites call sites referencing renamed - // module-level symbols. Runs over the same AST as inner above — - // exports/private_renames already collected - if !private_renames.is_empty() { - let mut renamer = NameRenamer::new(&private_renames); - for stmt in &module.body { - renamer.visit_stmt(stmt); - } - for fix in renamer.edits { - for edit in fix.edits() { - let range = edit.range(); - let repl = edit.content().unwrap_or_default().to_owned(); - ctx.text_edits.push((range, repl)); - } - } - } + // the references to a renamed module-level symbol are rewritten by + // `visibility_rename`, which asks ty which binding each one resolves to — + // a parameter or class attribute that shares the name is not the symbol // `private` renames the symbol, so a name it claims is not the name the // module ends up with — exporting it would put a name in `__all__` that @@ -1373,7 +1375,7 @@ mod tests { fn var_decl_with_modifier_chain() { // a visibility modifier ahead of `var` is stripped with it, matching the // bare-assignment modifier forms - check("private var a = 1\n", "a = 1\n"); + check("private var a = 1\n", "_a = 1\n"); } #[test] @@ -1838,9 +1840,9 @@ mod tests { #[test] fn private_annot_in_class() { - // `private` on a class member carries a meaning ty reads (the member is - // invisible to a widened view of the class), so it parses to its own marker. - // the lowering is still a bare prefix erasure — no rename, no runtime artefact + // `private` on a class member renames it to the `__name` python + // name-mangles, whatever else the modifier chain says and whether or not + // the declaration binds a value check( indoc! {" class Foo: @@ -1850,9 +1852,9 @@ mod tests { "}, indoc! {" class Foo: - t: int - u: int = 1 - v: str = \"v\" + __t: int + __u: int = 1 + __v: str = \"v\" "}, ); } @@ -1905,6 +1907,97 @@ mod tests { ); } + #[test] + fn protected_method_inside_class() { + // `protected` renames with a single underscore, which python does not + // mangle — a subclass reaches the member under the same name + check( + indoc! {" + class Outer: + protected def helper(self): ... + "}, + indoc! {" + class Outer: + def _helper(self): ... + "}, + ); + } + + #[test] + fn a_visibility_keyword_renames_an_attribute() { + check( + indoc! {" + class Outer: + private count: int + protected step: int = 2 + "}, + indoc! {" + class Outer: + __count: int + _step: int = 2 + "}, + ); + } + + #[test] + fn final_composes_with_a_visibility_keyword() { + // `final` and the visibility keyword each carry something the other does + // not — the `Final` qualifier and the rename — so neither is dropped + check( + indoc! {" + class Outer: + final private limit: int = 3 + "}, + indoc! {" + from typing import Final + class Outer: + __limit: Final[int] = 3 + "}, + ); + } + + #[test] + fn a_visibility_keyword_renames_a_let_binding() { + check( + indoc! {" + class Outer: + private let fixed: int = 5 + protected let bound = 6 + "}, + indoc! {" + from typing import Final + class Outer: + __fixed: int = 5 + _bound: Final = 6 + "}, + ); + } + + #[test] + fn a_visibility_keyword_renames_an_untyped_attribute() { + // the untyped form binds a member exactly as the annotated one does, so + // the keyword has to reach it there too + check( + indoc! {" + class Outer: + private count = 0 + protected step = 2 + "}, + indoc! {" + class Outer: + __count = 0 + _step = 2 + "}, + ); + } + + #[test] + fn a_module_level_private_variable_is_renamed() { + // renamed the way a module-level `private def` is: one leading underscore, + // python's own mark for "not part of the interface" + check("private count: int = 1\n", "_count: int = 1\n"); + } + #[test] fn export_skipped_inside_class() { // `export` on a class member must not pollute the module-level `__all__`. @@ -1999,4 +2092,27 @@ mod tests { "}, ); } + + /// the members tuple is assigned as the module runs, and a stub never runs + #[test] + fn a_stub_assigns_no_sealed_members() { + let out = transpile( + indoc! {" + sealed class A + class B(A) + "}, + &Config { + is_stub: true, + ..Config::test_default() + }, + ) + .unwrap(); + assert_eq!( + out, + indoc! {" + class A: ... + class B(A): ... + "} + ); + } } diff --git a/crates/by_transforms/src/transforms/mutable_defaults.rs b/crates/by_transforms/src/transforms/mutable_defaults.rs index ccd776af42..1ad8f0d494 100644 --- a/crates/by_transforms/src/transforms/mutable_defaults.rs +++ b/crates/by_transforms/src/transforms/mutable_defaults.rs @@ -96,6 +96,10 @@ struct MutableDefaults<'src> { /// functions whose body starts with parser-synthesized statements, so a /// guard has no source position to anchor to unanchored: Vec, + is_stub: bool, + /// `(function, parameter)` for each parameter a stub cannot declare. see + /// [`ParameterGuards::undeclarable`] + undeclarable: Vec<(String, String)>, } /// replace a default with the sentinel. a template rather than plain text so @@ -134,12 +138,26 @@ pub(crate) struct ParameterGuards { /// relocate nothing; they add text at the end of a parameter pub(crate) written: Vec<(TextRange, Vec)>, pub(crate) guards: Vec, + /// in a stub, the parameters python cannot declare where they stand: a + /// required one after a defaulted one. python rejects the shape, and the + /// sentinel default a module gets for it would declare the parameter + /// optional — only the guard in the body says otherwise, and a stub has none + pub(crate) undeclarable: Vec, } -pub(crate) fn parameter_guards(f: &StmtFunctionDef, types: &dyn TypeInfo) -> ParameterGuards { +/// The parameter guards `f` calls for. A stub is never run, so it gets only what +/// its signature declares: an inherited default, and none of the guards or the +/// sentinels they re-evaluate — the defaults it writes are never shared between +/// calls. +pub(crate) fn parameter_guards( + f: &StmtFunctionDef, + types: &dyn TypeInfo, + is_stub: bool, +) -> ParameterGuards { let mut sentinels = Vec::new(); let mut written = Vec::new(); let mut guards = Vec::new(); + let mut undeclarable = Vec::new(); let params = f.parameters.as_ref(); // positional parameters: swap non-scalar defaults for the sentinel, and give // basedpython's required-after-defaulted parameters a sentinel default plus @@ -150,7 +168,7 @@ pub(crate) fn parameter_guards(f: &StmtFunctionDef, types: &dyn TypeInfo) -> Par match pw.default.as_deref() { Some(d) => { seen_default = true; - if !is_immutable_scalar_default(d) && !body_cannot_evaluate(d) { + if !is_stub && !is_immutable_scalar_default(d) && !body_cannot_evaluate(d) { sentinels.push(sentinel_edit(d.range())); guards.push(Guard::Reevaluate { name: pw.parameter.name.id.to_string(), @@ -164,6 +182,9 @@ pub(crate) fn parameter_guards(f: &StmtFunctionDef, types: &dyn TypeInfo) -> Par seen_default = true; written.push(written_default(pw, &value)); } + None if seen_default && is_stub => { + undeclarable.push(pw.parameter.name.id.to_string()); + } None if seen_default => { written.push(written_default(pw, "_MISSING")); guards.push(Guard::Required { @@ -176,7 +197,7 @@ pub(crate) fn parameter_guards(f: &StmtFunctionDef, types: &dyn TypeInfo) -> Par } for pw in ¶ms.kwonlyargs { match pw.default.as_deref() { - Some(d) if !is_immutable_scalar_default(d) && !body_cannot_evaluate(d) => { + Some(d) if !is_stub && !is_immutable_scalar_default(d) && !body_cannot_evaluate(d) => { sentinels.push(sentinel_edit(d.range())); guards.push(Guard::Reevaluate { name: pw.parameter.name.id.to_string(), @@ -195,9 +216,19 @@ pub(crate) fn parameter_guards(f: &StmtFunctionDef, types: &dyn TypeInfo) -> Par sentinels, written, guards, + undeclarable, } } +/// the error for a parameter a stub cannot declare. see +/// [`ParameterGuards::undeclarable`] +pub(crate) fn undeclarable_error(function: &str, parameter: &str) -> String { + format!( + "a stub cannot declare parameter `{parameter}` of `{function}`: it is required but \ + follows a defaulted parameter, and python has no spelling for that in a signature" + ) +} + /// Whether the *callee's* body could evaluate `default` at all. /// /// Re-evaluating a default there is the point of the guard, and it is what lets @@ -257,7 +288,13 @@ impl MutableDefaults<'_> { sentinels, written, guards, - } = parameter_guards(f, self.types); + undeclarable, + } = parameter_guards(f, self.types, self.is_stub); + self.undeclarable.extend( + undeclarable + .into_iter() + .map(|parameter| (f.name.to_string(), parameter)), + ); self.relocating.extend(sentinels); self.edits.extend(written); if guards.is_empty() { @@ -288,11 +325,12 @@ impl<'ast> Visitor<'ast> for MutableDefaults<'_> { pub(crate) struct MutableDefaultsPass<'src> { source: &'src str, + is_stub: bool, } impl<'src> MutableDefaultsPass<'src> { - pub(crate) fn new(source: &'src str) -> Self { - Self { source } + pub(crate) fn new(source: &'src str, is_stub: bool) -> Self { + Self { source, is_stub } } } @@ -306,10 +344,16 @@ impl TypeAwarePass for MutableDefaultsPass<'_> { guards: Vec::new(), used: false, unanchored: Vec::new(), + is_stub: self.is_stub, + undeclarable: Vec::new(), }; for stmt in stmts { inner.visit_stmt(stmt); } + if let Some((function, parameter)) = inner.undeclarable.first() { + ctx.errors.push(undeclarable_error(function, parameter)); + return; + } if let Some(name) = inner.unanchored.first() { // the `init(…)` shorthand is the one construct that generates its // own body, and it emits its own guards; anything else reaching here @@ -986,4 +1030,59 @@ mod tests { "}, ); } + + fn stub() -> crate::Config { + crate::Config { + is_stub: true, + ..crate::Config::test_default() + } + } + + /// a stub is never run, so a default it declares is never shared between + /// calls. it stays as written, with no guard + #[test] + fn a_stub_keeps_a_mutable_default() { + let source = "def f(xs: list[int] = [], *, ys: list[int] = []) -> None: ...\n"; + assert_eq!(transpile(source, &stub()).unwrap(), source); + } + + /// an inherited default is part of what an override declares, which a stub + /// is for + #[test] + fn a_stub_writes_the_default_an_override_inherits() { + let out = transpile( + indoc! {" + class A: + def f(self, a: int = 1) -> None: ... + + class B(A): + def f(self, a: int) -> None: ... + "}, + &stub(), + ) + .unwrap(); + assert_eq!( + out, + indoc! {" + class A: + def f(self, a: int = 1) -> None: ... + + class B(A): + def f(self, a: int = 1) -> None: ... + "} + ); + } + + /// python rejects a required parameter after a defaulted one. a module gets + /// a sentinel default and a guard in the body that raises, and a stub has no + /// body for the guard, so the default alone would declare the parameter + /// optional + #[test] + fn a_stub_cannot_declare_a_required_parameter_after_a_default() { + let error = transpile("def f(x: int = 1, y: int) -> None: ...\n", &stub()).unwrap_err(); + assert!( + error.contains("a stub cannot declare parameter `y` of `f`"), + "got: {error}" + ); + } } diff --git a/crates/by_transforms/src/transforms/optional_type.rs b/crates/by_transforms/src/transforms/optional_type.rs index e0e20bcc15..34c4aee359 100644 --- a/crates/by_transforms/src/transforms/optional_type.rs +++ b/crates/by_transforms/src/transforms/optional_type.rs @@ -18,7 +18,7 @@ //! directly wrapping another optional keeps the outer layer as the runtime //! `Optional[...]` wrapper (`int??` ⇒ `Optional[int | None]`) so its distinct //! outer-`None` state is not collapsed into the inner one. the wrapper's runtime -//! class (see [`wrapped_runtime`](super::wrapped_runtime)) is injected when emitted. +//! class (see [`crate::runtime`]) is injected when emitted. //! //! The result form `T ? E` (`ExprBinOp` with `Operator::Result`) and the //! postfix `^` / `!` operators are intentionally left for a later pass — their @@ -29,7 +29,6 @@ use ruff_python_ast::{Expr, PythonVersion, Stmt, UnaryOp}; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{PassContext, TypeAwarePass}; -use super::wrapped_runtime::OPTIONAL_RUNTIME; use crate::type_info::TypeInfo; /// Walks the source AST and emits narrow text edits that lower each optional in @@ -227,7 +226,7 @@ impl TypeAwarePass for OptionalTypePass<'_> { lower.visit_stmt(stmt); } if lower.needs_runtime { - ctx.required_imports.push(OPTIONAL_RUNTIME.to_owned()); + ctx.runtime.insert(crate::runtime::OPTIONAL); } if lower.needs_union { ctx.required_imports @@ -282,7 +281,9 @@ class Optional: "var b: dict[str, str]? = None\n", "b: dict[str, str] | None = None\n", ); - check("private var c: int? = None\n", "c: int | None = None\n"); + // a module-level `private` variable is renamed; the optional lowering + // keeps the name the declaration is emitted under + check("private var c: int? = None\n", "_c: int | None = None\n"); } #[test] diff --git a/crates/by_transforms/src/transforms/parametric_is.rs b/crates/by_transforms/src/transforms/parametric_is.rs index 77be606b25..81c7b719ca 100644 --- a/crates/by_transforms/src/transforms/parametric_is.rs +++ b/crates/by_transforms/src/transforms/parametric_is.rs @@ -84,326 +84,6 @@ pub(crate) fn variance_tuple(variances: &[u8]) -> String { } } -pub(crate) const PARAMETRIC_IS_RUNTIME: &str = "\ -def _by_type_param_defaults(args): - # a class records its generic bases *unsubstituted* — `class L[T = Never] - # (list[T])` stores `list[T]`, never `list[Never]` — so a type parameter - # left at its pep 696 default resolves to that default rather than staying a - # bare TypeVar that matches nothing - resolved = [] - substituted = False - for arg in args: - has_default = getattr(arg, \"has_default\", None) - if has_default is not None and has_default(): - resolved.append(arg.__default__) - substituted = True - else: - resolved.append(arg) - return tuple(resolved) if substituted else args - -def _by_alias(value): - # a reified generic class specializes to a *subclass*, which records the - # alias it stands for; anything else already is what it says it is. read - # from the class's own dict, so an ordinary subclass of a specialization is - # not mistaken for one - if isinstance(value, type): - return value.__dict__.get(\"__orig_class__\", value) - return value - -def _by_subst(annotation, mapping): - # replace type parameters with the arguments bound to them, rebuilding - # nested aliases (`list[dict[str, T]]` with `T = int` → `list[dict[str, int]]`) - annotation = _by_alias(annotation) - try: - if annotation in mapping: - return mapping[annotation] - except TypeError: - pass - args = getattr(annotation, \"__args__\", ()) - if not args: - return annotation - replaced = tuple(_by_subst(arg, mapping) for arg in args) - if replaced == args: - return annotation - origin = getattr(annotation, \"__origin__\", None) - if origin is None: - return annotation - try: - return origin[replaced] - except TypeError: - return annotation - -def _by_specialize(alias, origin, depth=0): - # the arguments with which `alias` satisfies `origin`, resolved *down the - # declared base chain* rather than assumed to line up positionally. a base - # that fixes or reorders its arguments is then followed faithfully: - # `class Odd[T](list[int])` is a `list[int]` whatever `T` is, and - # `class Swap[A, B](dict[B, A])` specializes `dict` in the other order - if depth > 16: - return None - alias = _by_alias(alias) - klass = getattr(alias, \"__origin__\", alias) - if not isinstance(klass, type): - return None - args = getattr(alias, \"__args__\", ()) - params = getattr(klass, \"__type_params__\", ()) - if not args: - defaulted = _by_type_param_defaults(params) - if defaulted is not params: - args = defaulted - if klass is origin: - return args or None - mapping = {} - for param, arg in zip(params, args): - try: - mapping[param] = arg - except TypeError: - pass - bases = klass.__dict__.get(\"__orig_bases__\") - if bases is None: - # a class inheriting only plain classes records no `__orig_bases__` - bases = getattr(klass, \"__bases__\", ()) - for base in bases: - found = _by_specialize(_by_subst(base, mapping) if mapping else base, origin, depth + 1) - if found is not None: - return found - # the declared bases don't reach `origin`: a builtin registered as a *virtual* - # subclass of an abc (`list` for `Sequence`) has no base to walk. its - # arguments do line up positionally once membership is established. this runs - # only after resolution has failed, so it applies to the already-resolved base - # (`list[int]`), never to a subclass that fixes or reorders arguments - if args and isinstance(origin, type): - try: - if issubclass(klass, origin): - return args - except TypeError: - pass - return None - -def _by_generic_args(value, origin): - # an explicit `A[int]()` records its specialization on the instance; - # otherwise the class itself is the starting point and any pep 696 defaults - # stand in for the arguments it was constructed with - reified = getattr(value, \"__orig_class__\", None) - found = _by_specialize(reified if reified is not None else type(value), origin) - return [found] if found is not None else [] - -def _parametric_is(value, alias, variances): - alias = _by_alias(getattr(alias, \"__value__\", alias)) - origin = getattr(alias, \"__origin__\", alias) - if not isinstance(value, origin): - return False - target_args = getattr(alias, \"__args__\", ()) - if len(target_args) != len(variances): - return False - for reified_args in _by_generic_args(value, origin): - if len(reified_args) != len(target_args): - continue - for r, t, v in zip(reified_args, target_args, variances): - if v == 3 or r == t: - continue - if v == 1 and _parametric_is_sub(r, t): - continue - if v == 2 and _parametric_is_sub(t, r): - continue - break - else: - return True - return False - -def _parametric_is_sub(a, b): - if a is b or b is object: - return True - a_origin = getattr(a, \"__origin__\", a) - b_origin = getattr(b, \"__origin__\", b) - if isinstance(a_origin, type) and isinstance(b_origin, type) and not getattr(b, \"__args__\", ()): - try: - return issubclass(a_origin, b_origin) - except TypeError: - return False - return a == b - -def _parametric_is_lenient(value, alias, variances): - # the checked-cast form: a value that records no reification has no - # arguments to check, so the base class test is the whole guarantee. this is - # what keeps `[1, 2] cast list[int]` legal while still rejecting a value - # whose recorded arguments contradict the target - alias = _by_alias(getattr(alias, \"__value__\", alias)) - origin = getattr(alias, \"__origin__\", alias) - if not isinstance(value, origin): - return False - if not _by_generic_args(value, origin): - return True - return _parametric_is(value, alias, variances) -"; - -/// runtime residue for a parametric test against a *protocol* target -/// (`value is A[int]`). a protocol's instances never record which -/// specialization they satisfy, so `__orig_class__` can't answer it — but -/// basedpython reifies annotations, so the value's class is checked -/// structurally: each protocol member's reified annotation must match the -/// member's specialized type. `members` is a list of kind-tagged tuples: -/// -/// - `("attr", name, expected_type, variance)` — a data member, checked against -/// the value class's annotation for `name` -/// - `("method", name, [(type, variance), …], return_or_None)` — a method -/// member, whose parameters (contravariant) and return (covariant) are checked -/// against the value method's reified parameter/return annotations; a -/// parameter with no annotation but a default falls back to `type(default)` -/// -/// `variance` matches [`ArgVariance`]'s codes (0 invariant → equality, 1 -/// covariant → subtype, 2 contravariant → supertype, 3 bivariant → any). -/// annotations are read with `typing.get_type_hints` (resolving string -/// annotations and inherited members), falling back to a raw `__mro__` walk -const PROTOCOL_IS_RUNTIME: &str = "\ -_by_proto_missing = object() - -def _by_member_annotation(klass, name): - try: - import typing - hints = typing.get_type_hints(klass) - except Exception: - hints = None - if hints is not None and name in hints: - return hints[name] - for base in klass.__mro__: - annotations = base.__dict__.get(\"__annotations__\", {}) - if name in annotations: - return annotations[name] - return _by_proto_missing - -def _by_lit(*values): - # rebuild `typing.Literal[…]` for a literal type argument (`A[True]` - # specializes `T` to `Literal[True]`). spelled as a call so the member list - # needs no import of its own — this helper ships with the check - import typing - return typing.Literal[values] - -def _by_literal_args(t): - import typing - return typing.get_args(t) if typing.get_origin(t) is typing.Literal else None - -def _by_proto_sub(a, b): - if a is b or b is object: - return True - a_values = _by_literal_args(a) - b_values = _by_literal_args(b) - if a_values is not None: - # `Literal[True]` is a subtype of another literal that lists all its - # values, and of any class every value is an instance of - if b_values is not None: - return all(value in b_values for value in a_values) - return isinstance(b, type) and all(isinstance(value, b) for value in a_values) - if b_values is not None: - # a whole class is never a subtype of a narrower literal - return False - a_origin = getattr(a, \"__origin__\", a) - b_origin = getattr(b, \"__origin__\", b) - if isinstance(a_origin, type) and isinstance(b_origin, type) and not getattr(b, \"__args__\", ()): - try: - return issubclass(a_origin, b_origin) - except TypeError: - return False - return a == b - -def _by_variance_ok(actual, expected, variance): - # 0 invariant (equality), 1 covariant (actual <: expected), - # 2 contravariant (expected <: actual), 3 bivariant (any) - if variance == 3 or actual == expected: - return True - if variance == 1 and _by_proto_sub(actual, expected): - return True - if variance == 2 and _by_proto_sub(expected, actual): - return True - return False - -def _by_method_matches(klass, name, params, ret): - method = getattr(klass, name, None) - if not callable(method): - return False - import inspect, typing - try: - signature = inspect.signature(method) - hints = typing.get_type_hints(method) - except Exception: - return False - positional = [ - p for p in signature.parameters.values() - if p.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) - ] - # drop the receiver (`self` / `cls`) an unbound method still carries - positional = positional[1:] - if len(positional) < len(params): - return False - # extra positional parameters the protocol doesn't supply must be optional, - # else a caller matching the protocol would fail to provide them - for p in positional[len(params):]: - if p.default is inspect.Parameter.empty: - return False - # likewise any required keyword-only parameter would break a protocol call - for p in signature.parameters.values(): - if p.kind == inspect.Parameter.KEYWORD_ONLY and p.default is inspect.Parameter.empty: - return False - for (expected, variance), p in zip(params, positional): - if p.name in hints: - actual = hints[p.name] - elif p.default is not inspect.Parameter.empty: - # a reified default gives the parameter's inferred type at runtime - actual = type(p.default) - else: - return False - if not _by_variance_ok(actual, expected, variance): - return False - if ret is not None: - expected, variance = ret - if \"return\" not in hints or not _by_variance_ok(hints[\"return\"], expected, variance): - return False - return True - -def _by_protocol_is(value, members): - klass = type(value) - for member in members: - kind = member[0] - if kind == \"attr\": - _, name, expected, variance = member - actual = _by_member_annotation(klass, name) - if actual is _by_proto_missing: - # the member is *there*, it just carries no annotation any - # runtime can read — python records nothing for a `self.a: int` - # written inside `__init__`. answering `False` would contradict - # the checker, which accepts that class as satisfying the - # protocol, so refuse to answer rather than answer wrongly - if hasattr(value, name): - raise TypeError( - \"cannot check `\" + klass.__qualname__ + \".\" + name - + \"` against a parameterized protocol: its type is declared \" - + \"inside a method, and only a class-level annotation \" - + \"survives to runtime. declare it in the class body\" - ) - return False - if not _by_variance_ok(actual, expected, variance): - return False - else: - _, name, params, ret = member - if not _by_method_matches(klass, name, params, ret): - return False - return True -"; - -/// matches a value against a template literal type — a pattern such as -/// `f"a{int}b"`, whose type is the set of strings it can produce. -/// -/// the regular expression comes from the checker, which builds it from the same -/// reading of the pattern's holes that decides the static answer, so the test -/// accepts exactly the strings the type contains. a non-`str` value is not one -/// of them -const PATTERN_IS_RUNTIME: &str = "\ -import re as _by_re - -def _by_pattern_is(value, pattern): - return isinstance(value, str) and _by_re.fullmatch(pattern, value) is not None -"; - /// the runtime variance code `_by_variance_ok` expects fn variance_code(variance: ArgVariance) -> u8 { match variance { @@ -924,13 +604,23 @@ pub(crate) enum PredicateRuntime { } impl PredicateRuntime { - /// the definitions this helper needs in the emitted module - pub(crate) fn source(self) -> &'static str { + /// the runtime helpers the emitted predicate calls by name. what each of + /// those in turn needs is settled in [`crate::runtime`], off the calls in + /// their own bodies + pub(crate) fn helpers(self) -> &'static [crate::runtime::Helper] { match self { - Self::Parametric => PARAMETRIC_IS_RUNTIME, - Self::Protocol => PROTOCOL_IS_RUNTIME, - Self::Conformance => super::conformance::WITNESS_RUNTIME, - Self::Pattern => PATTERN_IS_RUNTIME, + // both strictnesses are spelled at the use site — which one a probe + // gets depends on how much the checker could already settle — so + // the pair travels together + Self::Parametric => &[ + crate::runtime::PARAMETRIC_IS, + crate::runtime::PARAMETRIC_IS_LENIENT, + ], + // `_by_lit` is spelled into the member list itself, not just called + // from inside the check, so it travels with the protocol runtime + Self::Protocol => &[crate::runtime::PROTOCOL_IS, crate::runtime::LITERAL], + Self::Conformance => super::conformance::WITNESS_HELPERS, + Self::Pattern => &[crate::runtime::PATTERN_IS], } } @@ -1018,7 +708,7 @@ impl TypeAwarePass for ParametricIsPass<'_> { // reified-generic requirement; a user-generic probe (`A[int]`) works // on any target for runtime in inner.runtimes { - ctx.required_imports.push(runtime.source().to_owned()); + ctx.runtime.extend(runtime.helpers()); } ctx.template_edits.extend(inner.edits); } diff --git a/crates/by_transforms/src/transforms/private_method.rs b/crates/by_transforms/src/transforms/private_method.rs deleted file mode 100644 index 6239e4c56b..0000000000 --- a/crates/by_transforms/src/transforms/private_method.rs +++ /dev/null @@ -1,143 +0,0 @@ -//! call sites of a `private` method (basedpython) -//! -//! `private def helper()` in a class body lowers to `def __helper()`, which -//! python name-mangles to `_A__helper` while class `A`'s body is executed. The -//! call sites keep the name the source wrote, so they have to be pointed at the -//! same attribute: -//! -//! ```by -//! class A: -//! private def helper(self) -> int: -//! return 1 -//! -//! def use(self) -> int: -//! return self.helper() -//! ``` -//! -//! → -//! -//! ```python -//! class A: -//! def __helper(self) -> int: -//! return 1 -//! -//! def use(self) -> int: -//! return self._A__helper() -//! ``` -//! -//! the mangled name is written out rather than left to python: python mangles -//! lexically, so `self.__helper` would mean `_B__helper` in a subclass's body -//! and `__helper` outside a class altogether, while `_A__helper` names the same -//! attribute from every one of those places - -use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{Expr, Stmt}; -use ruff_text_size::Ranged; - -use super::ast_driver::{PassContext, TypeAwarePass}; -use crate::type_info::TypeInfo; - -pub(crate) struct PrivateMethodPass; - -impl TypeAwarePass for PrivateMethodPass { - fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let mut renamer = Renamer { - types, - edits: Vec::new(), - }; - for stmt in stmts { - renamer.visit_stmt(stmt); - } - ctx.text_edits.extend(renamer.edits); - } -} - -struct Renamer<'a> { - types: &'a dyn TypeInfo, - edits: Vec<(ruff_text_size::TextRange, String)>, -} - -impl<'ast> Visitor<'ast> for Renamer<'_> { - fn visit_stmt(&mut self, stmt: &'ast Stmt) { - walk_stmt(self, stmt); - } - - fn visit_expr(&mut self, expr: &'ast Expr) { - if let Expr::Attribute(attribute) = expr - && let Some(mangled) = self.types.private_method_name(attribute) - { - self.edits.push((attribute.attr.range(), mangled)); - } - walk_expr(self, expr); - } -} - -#[cfg(test)] -mod tests { - use crate::{Config, transpile}; - use indoc::indoc; - - fn out(input: &str) -> String { - transpile(input, &Config::test_default()).unwrap() - } - - #[test] - fn a_call_reaches_the_mangled_definition() { - let out = out(indoc! {" - class A: - private def helper(self) -> int: - return 1 - - def use(self) -> int: - return self.helper() - "}); - assert!(out.contains("def __helper(self) -> int:"), "got:\n{out}"); - assert!(out.contains("return self._A__helper()"), "got:\n{out}"); - } - - #[test] - fn a_call_from_a_subclass_reaches_the_declaring_class() { - // python would mangle `self.__helper` written here to `_B__helper`, - // which names nothing — the declaring class is what the name records - let out = out(indoc! {" - class A: - private def helper(self) -> int: - return 1 - - class B(A): - def use(self) -> int: - return self.helper() - "}); - assert!(out.contains("return self._A__helper()"), "got:\n{out}"); - } - - #[test] - fn an_ordinary_method_is_untouched() { - let out = out(indoc! {" - class A: - def helper(self) -> int: - return 1 - - def use(self) -> int: - return self.helper() - "}); - assert!(out.contains("return self.helper()"), "got:\n{out}"); - } - - #[test] - fn a_same_named_method_on_another_class_is_untouched() { - let out = out(indoc! {" - class A: - private def helper(self) -> int: - return 1 - - class B: - def helper(self) -> int: - return 2 - - def use(self) -> int: - return self.helper() - "}); - assert!(out.contains("return self.helper()"), "got:\n{out}"); - } -} diff --git a/crates/by_transforms/src/transforms/properties.rs b/crates/by_transforms/src/transforms/properties.rs index 4eafb62afe..73385c001a 100644 --- a/crates/by_transforms/src/transforms/properties.rs +++ b/crates/by_transforms/src/transforms/properties.rs @@ -43,9 +43,13 @@ use std::collections::HashMap; use std::fmt::Write; +use ruff_python_ast::helpers::{MemberVisibility, property_backing_name}; use ruff_python_ast::token::{Tokens, parenthesized_range}; use ruff_python_ast::visitor::{Visitor, walk_stmt}; -use ruff_python_ast::{AnyNodeRef, Expr, ExprRef, Stmt, StmtClassDef, StmtFunctionDef}; +use ruff_python_ast::{ + AnyNodeRef, Expr, ExprRef, PropertyConstruct, Stmt, StmtClassDef, StmtFunctionDef, +}; +use ruff_python_stdlib::basedpython::visibility_rename; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass, render_stmt}; @@ -145,49 +149,6 @@ impl<'src> PropertiesPass<'src> { } } -/// The runtime half of a `static let` property. Python dropped `classmethod` -/// chaining onto `property` in 3.13, and a read-only class-level property needs -/// nothing else a metaclass would offer, so a plain non-data descriptor is the -/// whole implementation. Mirrors `_by_static_property` in `ty_extensions._internal`, -/// which is ty's type-only view of the same thing. -const STATIC_PROPERTY_HELPER: &str = "\ -class _by_static_property: - def __init__(self, fget): - self._fget = fget - def __get__(self, instance, owner=None): - return self._fget(owner if owner is not None else type(instance)) -"; - -/// A property accessor block's marker, as recorded by the parser on the getter. -struct PropertyMarker { - /// The whole `var`/`let` declaration plus its accessor suite — the span the - /// lowering replaces. - construct: TextRange, - /// `static let`: a class-level property, lowered to [`STATIC_PROPERTY_HELPER`] - /// rather than to `property`. - is_static: bool, -} - -/// The property marker on `func`, or `None` for any other function. -fn property_marker(func: &StmtFunctionDef) -> Option { - func.decorator_list - .iter() - .find_map(|dec| match &dec.expression { - Expr::Name(name) => match name.id.as_str() { - "__property__" => Some(PropertyMarker { - construct: dec.range(), - is_static: false, - }), - "__static_property__" => Some(PropertyMarker { - construct: dec.range(), - is_static: true, - }), - _ => None, - }, - _ => None, - }) -} - /// Whether `func` is the `@.setter` half of property `prop`. fn is_setter_of(func: &StmtFunctionDef, prop: &str) -> bool { func.decorator_list.iter().any(|dec| match &dec.expression { @@ -366,7 +327,6 @@ impl PropertiesPass<'_> { fn accessor_body( &self, func: &StmtFunctionDef, - is_getter: bool, body_indent: &str, backing_name: &str, ctx: &mut PassContext, @@ -403,27 +363,39 @@ impl PropertiesPass<'_> { // is also passed through whole, parentheses included, which is what lets // the continuation lines keep their source column: inside parentheses they // are continuations rather than a block, so their depth means nothing - let value = self.value_range(first); // a `get() = ` accessor's `return` exists only in the AST, so the // parser gives it the expression's own range; a `return` the author wrote // starts at the keyword, several characters earlier. that is an exact fact // about which shape this is, where the line position is only a proxy for it - // — and the proxy is wrong for a value held off the accessor's line by a - // backslash, which came out as a body with no `return` at all + // — and the proxy is wrong both for a value held off the accessor's line by a + // backslash, which came out as a body with no `return` at all, and for a + // suite written on the accessor's line, whose `get(): pass` came out as + // `return pass` let synthesized_return = matches!( first, Stmt::Return(ret) if ret.value.as_ref().is_some_and(|v| v.range() == first.range()) ); - let inline = synthesized_return - || line_start(self.source, value.start()) - == line_start(self.source, func.range().start()); - if inline { - let mut frags = vec![Fragment::Lit(body_indent.to_owned())]; - if is_getter { - frags.push(Fragment::Lit("return ".to_owned())); + if synthesized_return { + return vec![ + Fragment::Lit(format!("{body_indent}return ")), + Fragment::Src(self.value_range(first)), + ]; + } + // a suite on the accessor's own line — `get(): pass`, `set(v): a = v; field = a`, + // and `set(v) = `, whose expression stands as the statement — can only + // hold simple statements, so each one becomes a body line of its own + if line_start(self.source, first.range().start()) + == line_start(self.source, func.range().start()) + { + let mut frags = Vec::new(); + for (idx, stmt) in func.body.iter().enumerate() { + if idx > 0 { + frags.push(Fragment::Lit("\n".to_owned())); + } + frags.push(Fragment::Lit(body_indent.to_owned())); + frags.push(Fragment::Src(self.value_range(stmt))); } - frags.push(Fragment::Src(value)); frags } else { // the source body sits at the accessor suite's indent; the emitted @@ -511,27 +483,39 @@ impl PropertiesPass<'_> { // construct's template, because the driver absorbs a zero-width insertion // sharing a template's start rather than emitting it alongside let mut pending: Vec<(TextRange, Vec)> = Vec::new(); - // `(public name, emitted name)` for each `private` property in this class - let mut renames: Vec<(String, String)> = Vec::new(); for member in &class.body { let Stmt::FunctionDef(getter) = member else { continue; }; - let Some(PropertyMarker { - construct, + // `static let` lowers to the runtime's `_by_static_property` rather than to `property` + let Some(PropertyConstruct { + range: construct, is_static, - }) = property_marker(getter) + }) = getter.property_construct() else { continue; }; let prop = getter.name.as_str(); - // the getter's name node keeps the *public* name's source range while its - // `id` carries the emitted name (they differ for a `private` property), and - // storage is `__` — a dunder so python's name mangling hides it - let public = &self.source - [usize::from(getter.name.range().start())..usize::from(getter.name.range().end())]; - let backing_name = format!("__{public}"); + // modifier keywords written ahead of the declaration compose with the + // property. the getter's name node keeps the declaration name's real + // source range, so the prefix is the span before it + let prefix = &self.source + [usize::from(construct.start())..usize::from(getter.name.range().start())]; + let modifiers: Vec<&str> = prefix.split_whitespace().collect(); + let has = |word: &str| modifiers.contains(&word); + // a visibility keyword renames the property the way it renames any + // member, and decides the name of its storage + let visibility = if has("private") { + MemberVisibility::Private + } else if has("protected") { + MemberVisibility::Protected + } else { + MemberVisibility::Public + }; + let emitted = visibility_rename(prop, visibility.name_prefix()) + .unwrap_or_else(|| prop.to_owned()); + let backing_name = property_backing_name(prop, visibility); // the setter and backing field the parser synthesised for *this* // construct carry ranges inside its span, which keeps a same-named @@ -579,25 +563,11 @@ impl PropertiesPass<'_> { let indent = line_indent(self.source, construct.start()).to_owned(); let body_indent = format!("{indent} "); - // modifier keywords written ahead of the declaration compose with the - // property. the getter's name node keeps the declaration name's real - // source range, so the prefix is the span before it. these decorators - // sit *under* `@property` / `@.setter` so they apply to the - // accessor function itself, which is what type checkers expect - let prefix = &self.source - [usize::from(construct.start())..usize::from(getter.name.range().start())]; - let modifiers: Vec<&str> = prefix.split_whitespace().collect(); - let has = |word: &str| modifiers.contains(&word); + // these decorators sit *under* `@property` / `@.setter` so they + // apply to the accessor function itself, which is what type checkers + // expect let is_abstract = has("abstract"); - // a `private` property is emitted one underscore deeper than the name - // the author wrote, so in-class accesses spelled under the public name - // have to be redirected in the output too. the parser has already - // retargeted them in the AST, but the source still says `self.x` - if has("private") { - renames.push((public.to_owned(), prop.to_owned())); - } - let mut accessor_decorators = String::new(); if is_abstract { ctx.required_imports @@ -652,13 +622,13 @@ impl PropertiesPass<'_> { // getter. a `static` property is a descriptor taking the owning class // rather than a `property` taking an instance let (decorator, receiver) = if is_static { - ctx.required_imports.push(STATIC_PROPERTY_HELPER.to_owned()); + ctx.runtime.insert(crate::runtime::STATIC_PROPERTY); ("_by_static_property", "cls") } else { ("property", "self") }; frags.push(Fragment::Lit(format!( - "@{decorator}\n{indent}{accessor_decorators}def {prop}({receiver})" + "@{decorator}\n{indent}{accessor_decorators}def {emitted}({receiver})" ))); if let Some(returns) = &getter.returns { frags.push(Fragment::Lit(" -> ".to_owned())); @@ -669,13 +639,13 @@ impl PropertiesPass<'_> { frags.push(Fragment::Lit(": ...".to_owned())); } else { frags.push(Fragment::Lit(":\n".to_owned())); - frags.extend(self.accessor_body(getter, true, &body_indent, &backing_name, ctx)); + frags.extend(self.accessor_body(getter, &body_indent, &backing_name, ctx)); } // setter if let Some(setter) = setter { frags.push(Fragment::Lit(format!( - "\n{indent}@{prop}.setter\n{indent}{accessor_decorators}def {prop}(self" + "\n{indent}@{emitted}.setter\n{indent}{accessor_decorators}def {emitted}(self" ))); // the value parameter follows the synthetic `self` if let Some(value_param) = setter.parameters.args.get(1) { @@ -689,13 +659,7 @@ impl PropertiesPass<'_> { frags.push(Fragment::Lit(") -> None: ...".to_owned())); } else { frags.push(Fragment::Lit(") -> None:\n".to_owned())); - frags.extend(self.accessor_body( - setter, - false, - &body_indent, - &backing_name, - ctx, - )); + frags.extend(self.accessor_body(setter, &body_indent, &backing_name, ctx)); } } @@ -734,79 +698,6 @@ impl PropertiesPass<'_> { } } ctx.template_edits.extend(pending); - - if !renames.is_empty() { - self.rename_private_accesses(class, &renames, ctx); - } - } - - /// Redirects in-class accesses of a `private` property to the name it is - /// actually emitted under. The parser retargeted them in the AST so ty resolves - /// them; the source still spells the public name, so the output needs an edit - /// per occurrence. An access from outside the class is left alone — the property - /// genuinely is not there, and ty reports it. - fn rename_private_accesses( - &self, - class: &StmtClassDef, - renames: &[(String, String)], - ctx: &mut PassContext, - ) { - for member in &class.body { - let Stmt::FunctionDef(func) = member else { - continue; - }; - let Some(receiver) = func - .parameters - .posonlyargs - .first() - .or_else(|| func.parameters.args.first()) - .map(|param| param.parameter.name.id.as_str()) - else { - continue; - }; - let mut finder = PrivateAccessFinder { - source: self.source, - renames, - receiver, - edits: Vec::new(), - }; - for stmt in &func.body { - finder.visit_stmt(stmt); - } - ctx.text_edits.extend(finder.edits); - } - } -} - -/// Collects the source ranges of in-class accesses that name a `private` property -/// under its public spelling. -struct PrivateAccessFinder<'a> { - source: &'a str, - renames: &'a [(String, String)], - receiver: &'a str, - edits: Vec<(TextRange, String)>, -} - -impl<'ast> Visitor<'ast> for PrivateAccessFinder<'_> { - fn visit_stmt(&mut self, stmt: &'ast Stmt) { - // a nested class has its own `self` - if matches!(stmt, Stmt::ClassDef(_)) { - return; - } - walk_stmt(self, stmt); - } - - fn visit_expr(&mut self, expr: &'ast Expr) { - if let Expr::Attribute(attr) = expr - && matches!(attr.value.as_ref(), Expr::Name(name) if name.id.as_str() == self.receiver) - { - let range = attr.attr.range(); - let written = &self.source[usize::from(range.start())..usize::from(range.end())]; - if let Some((_, emitted)) = self.renames.iter().find(|(public, _)| public == written) { - self.edits.push((range, emitted.clone())); - } - } - ruff_python_ast::visitor::walk_expr(self, expr); } } @@ -985,6 +876,62 @@ mod tests { ); } + /// a suite written on the accessor's line is a body like any other, not the + /// value of a `get() = ` accessor: it has no `return` to add + #[test] + fn a_getter_suite_on_the_accessor_line_is_a_body() { + check( + indoc! {" + class A: + let n: None + get(): pass + + class B: + let v: int = 0 + get(): return field + "}, + indoc! {" + class A: + @property + def n(self) -> None: + pass + + class B: + def __init__(self) -> None: + self.__v: int = 0 + @property + def v(self) -> int: + return self.__v + "}, + ); + } + + /// every statement of a suite on the accessor's line is emitted, each on a + /// line of its own + #[test] + fn a_suite_on_the_accessor_line_keeps_every_statement() { + check( + indoc! {" + class A: + var v: int = 0 + get() = field + set(value): checked = abs(value); field = checked + "}, + indoc! {" + class A: + def __init__(self) -> None: + self.__v: int = 0 + @property + def v(self) -> int: + return self.__v + @v.setter + def v(self, value: int) -> None: + checked = abs(value) + self.__v = checked + "}, + ); + } + /// the design doc's motivating example #[test] fn stored_var_property() { @@ -1189,9 +1136,8 @@ mod tests { ); } - /// `private` shifts the construct one underscore deeper — property `_x`, - /// storage `__x` — and in-class accesses spelled under the public name are - /// redirected to it + /// `private` renames the property to the `__x` python mangles, moves its + /// storage to `__x_field`, and renames the in-class accesses with it #[test] fn private_property_is_renamed() { check( @@ -1205,6 +1151,38 @@ mod tests { def bump(self): self.x = self.x + 1 "}, + indoc! {" + class A: + def __init__(self) -> None: + self.__x_field: int = 0 + @property + def __x(self) -> int: + return self.__x_field + @__x.setter + def __x(self, value: int) -> None: + self.__x_field = value + + def bump(self): + self._A__x = self._A__x + 1 + "}, + ); + } + + /// `protected` renames the property to `_x`, whose storage keeps the + /// ordinary `__x` + #[test] + fn protected_property_is_renamed() { + check( + indoc! {" + class A: + protected var x: int = 0 + get() = field + set(value): + field = value + + def bump(self): + self.x = self.x + 1 + "}, indoc! {" class A: def __init__(self) -> None: @@ -1237,19 +1215,19 @@ mod tests { indoc! {" class A: def __init__(self) -> None: - self.__n: int = 5 + self.__n_field: int = 5 @property - def _n(self) -> int: - return self.__n + def __n(self) -> int: + return self.__n_field def f(self): - return self._n + return self._A__n "}, ); } - /// a same-named attribute on another object is not a property access and must - /// keep its name + /// a same-named attribute on another object is not a property access, so it + /// keeps its name #[test] fn private_rename_only_touches_the_receiver() { check( @@ -1264,16 +1242,16 @@ mod tests { indoc! {" class A: def __init__(self) -> None: - self.__x: int = 0 + self.__x_field: int = 0 @property - def _x(self) -> int: - return self.__x - @_x.setter - def _x(self, value: int) -> None: - self.__x = value + def __x(self) -> int: + return self.__x_field + @__x.setter + def __x(self, value: int) -> None: + self.__x_field = value def f(self, other): - return other.x + self._x + return other.x + self._A__x "}, ); } diff --git a/crates/by_transforms/src/transforms/protocol_type.rs b/crates/by_transforms/src/transforms/protocol_type.rs index a9f11b5784..74250c2391 100644 --- a/crates/by_transforms/src/transforms/protocol_type.rs +++ b/crates/by_transforms/src/transforms/protocol_type.rs @@ -440,8 +440,12 @@ pub(crate) fn cleanup( } }; push_missing(&mut preamble, "from typing import Protocol"); - for line in inner.callable.take_import_lines() { - push_missing(&mut preamble, &line); + let (imports, helpers) = inner.callable.take_requirements(); + for line in imports + .into_iter() + .chain(crate::runtime_entries(config, &helpers)) + { + push_missing(&mut preamble, line.trim_end_matches('\n')); } for defs in [inner.callable.class_defs().to_owned(), inner.class_defs()] { for class_def in defs.split_inclusive("\n\n") { @@ -470,8 +474,9 @@ impl TypeAwarePass for ProtocolTypePass<'_> { .push(defs.trim_end_matches('\n').to_owned()); } } - ctx.required_imports - .extend(inner.callable.take_import_lines()); + let (imports, helpers) = inner.callable.take_requirements(); + ctx.required_imports.extend(imports); + ctx.runtime.extend(helpers); for fix in inner.edits { for edit in fix.edits() { ctx.text_edits diff --git a/crates/by_transforms/src/transforms/raises_clause.rs b/crates/by_transforms/src/transforms/raises_clause.rs index a918a9d3d1..3f334dfe79 100644 --- a/crates/by_transforms/src/transforms/raises_clause.rs +++ b/crates/by_transforms/src/transforms/raises_clause.rs @@ -27,61 +27,6 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{AstPass, PassContext}; use crate::type_info::TypeInfo; -/// Fails when a guarded function raises outside its declared set. -/// -/// The wrapper shape is chosen at decoration time rather than by the transform: -/// a coroutine must be awaited, and a generator or async generator iterated, -/// before the body runs at all — wrapping any of them with a plain call would -/// catch nothing. An async generator is checked first because it answers `False` -/// to both `iscoroutinefunction` and `isgeneratorfunction`. -const RAISES_RUNTIME: &str = r#"def _by_raises(_allowed, _name): - import functools - import inspect - - def _check(_exc): - if not isinstance(_exc, _allowed): - raise AssertionError( - f"{_name} raised {type(_exc).__name__}, which its `raises` clause does not include" - ) from _exc - - def _decorate(_fn): - if inspect.isasyncgenfunction(_fn): - @functools.wraps(_fn) - async def _wrapper(*_args, **_kwargs): - try: - async for _item in _fn(*_args, **_kwargs): - yield _item - except BaseException as _exc: - _check(_exc) - raise - elif inspect.iscoroutinefunction(_fn): - @functools.wraps(_fn) - async def _wrapper(*_args, **_kwargs): - try: - return await _fn(*_args, **_kwargs) - except BaseException as _exc: - _check(_exc) - raise - elif inspect.isgeneratorfunction(_fn): - @functools.wraps(_fn) - def _wrapper(*_args, **_kwargs): - try: - yield from _fn(*_args, **_kwargs) - except BaseException as _exc: - _check(_exc) - raise - else: - @functools.wraps(_fn) - def _wrapper(*_args, **_kwargs): - try: - return _fn(*_args, **_kwargs) - except BaseException as _exc: - _check(_exc) - raise - return _wrapper - - return _decorate"#; - /// Deletes every `raises` clause. pub(crate) struct RaisesStripPass<'src> { source: &'src str, @@ -150,6 +95,11 @@ impl<'src> RaisesGuardPass<'src> { } impl super::ast_driver::TypeAwarePass for RaisesGuardPass<'_> { + // the guard checks what a call raises as it raises it + fn runtime_only(&self) -> bool { + true + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { if !self.enabled { return; @@ -179,7 +129,7 @@ impl super::ast_driver::TypeAwarePass for RaisesGuardPass<'_> { return; } - ctx.required_imports.push(RAISES_RUNTIME.to_owned()); + ctx.runtime.insert(crate::runtime::RAISES); ctx.text_edits.extend(guards); } } @@ -218,6 +168,13 @@ impl<'ast> Visitor<'ast> for GuardCollector<'_> { } /// The decorator insertion guarding `function`, when its clause has a runtime test. +/// +/// It goes directly in front of the `def` keyword, below any decorator the +/// function is written with, so that it wraps the function itself. That position +/// is also where a statement another lowering puts before the function lands +/// — the `TypeVar` definitions the pep 695 polyfill writes, the guard a mutable +/// default moves into an enclosing body — and those lead any insertion made +/// there, so they stay above the decorator rather than between it and its `def`. fn guard_for( source: &str, function: &StmtFunctionDef, @@ -228,16 +185,50 @@ fn guard_for( return None; } - let allowed = types.declared_raises_runtime_target(function)?; - let (offset, indent) = def_line_start(source, function)?; + let target = types.declared_raises_runtime_target(function)?; + let (offset, indent) = def_keyword(source, function)?; let name = function.name.as_str(); + let arguments = match &target.resolved { + Some(resolved) => { + let parameters: Vec<&str> = resolved + .own + .iter() + .chain(&resolved.receiver) + .map(String::as_str) + .collect(); + let lambda = if parameters.is_empty() { + format!("lambda: {}", resolved.expression) + } else { + format!("lambda {}: {}", parameters.join(", "), resolved.expression) + }; + format!( + "{}, \"{name}\", {lambda}, {}, {}", + target.ceiling, + python_names(&resolved.own), + python_names(&resolved.receiver), + ) + } + None => format!("{}, \"{name}\"", target.ceiling), + }; + Some(( TextRange::empty(offset), - format!("{indent}@_by_raises({allowed}, \"{name}\")\n"), + format!("@_by_raises({arguments})\n{indent}"), )) } +/// `names` as a python tuple of strings. The trailing comma is what keeps a +/// single name a tuple. +fn python_names(names: &[String]) -> String { + let names = names + .iter() + .map(|name| format!("\"{name}\", ")) + .collect::>() + .concat(); + format!("({names})") +} + /// A body that is exactly `...` declares a signature and runs nothing. fn is_stub_body(function: &StmtFunctionDef) -> bool { match function.body.as_slice() { @@ -248,10 +239,7 @@ fn is_stub_body(function: &StmtFunctionDef) -> bool { } /// The offset of the `def` keyword and the indentation of its line. -/// -/// The statement's own range starts at the first decorator, so the guard — which -/// must be the innermost wrapper — is placed by finding the `def` itself. -fn def_line_start<'src>( +fn def_keyword<'src>( source: &'src str, function: &StmtFunctionDef, ) -> Option<(TextSize, &'src str)> { @@ -273,13 +261,14 @@ fn def_line_start<'src>( return None; } - Some((TextSize::try_from(line).ok()?, indent)) + Some((TextSize::try_from(keyword).ok()?, indent)) } #[cfg(test)] mod tests { use crate::{Config, transpile}; use indoc::indoc; + use ruff_python_ast::PythonVersion; fn check(input: &str, expected: &str) { assert_eq!( @@ -295,6 +284,15 @@ mod tests { } } + /// [`guarded`] for a target that keeps pep 695 type parameters, so a generic + /// `def` reaches the guard as written rather than through the polyfill. + fn guarded_generic() -> Config { + Config { + min_version: PythonVersion::PY312, + ..guarded() + } + } + #[test] fn clause_stripped() { check( @@ -399,6 +397,115 @@ mod tests { ); } + #[test] + fn guard_of_a_generic_clause_tests_the_parameter_bound() { + // which exception `T` is was chosen by the caller, and the guard runs + // inside the callee — but every `T` is an `OSError`, so testing that + // still catches a function raising outside its clause + let out = transpile( + "def f[T: OSError](error: T) raises T:\n raise error\n", + &guarded_generic(), + ) + .unwrap(); + assert!( + out.contains("@_by_raises(OSError, \"f\")"), + "guard missing:\n{out}" + ); + } + + #[test] + fn a_reified_clause_carries_a_resolver_beside_its_ceiling() { + // `T` reifies, so the guard is handed the names it can read at the call + // and a lambda building the test from them. the ceiling stays as what it + // falls back to when nothing answers + let out = transpile( + "def f[reified T: OSError](error: T) raises T | ValueError:\n raise error\n", + &guarded_generic(), + ) + .unwrap(); + assert!( + out.contains( + "@_by_raises((OSError, ValueError), \"f\", \ + lambda T: (T, ValueError), (\"T\", ), ())" + ), + "resolver missing:\n{out}" + ); + } + + #[test] + fn a_type_parameter_with_no_exception_ceiling_is_not_guarded() { + // `object` says nothing an `isinstance` could test, so there is no guard + // to write rather than one that passes everything + let out = transpile( + "def f[T](error: T) raises T:\n raise error\n", + &guarded_generic(), + ) + .unwrap(); + assert!(!out.contains("_by_raises"), "unexpected guard:\n{out}"); + } + + #[test] + fn the_pep695_polyfill_writes_its_typevars_above_the_guard() { + // below 3.12 a generic `def` grows `_T = TypeVar(...)` before it. that is + // a statement, so it has to land above the whole decorated definition — + // between a decorator and its `def` is not python at all + let out = transpile( + "def f[T](value: T) raises TypeError:\n raise TypeError\n", + &guarded(), + ) + .unwrap(); + assert!( + out.contains("_T = TypeVar(\"_T\")\n@_by_raises(TypeError, \"f\")\ndef f("), + "wrong order:\n{out}" + ); + } + + #[test] + fn a_reified_class_parameter_is_read_off_the_receiver() { + // a direct method with a receiver reads its class's argument from the + // instance, so the guard names it separately from the function's own + let out = transpile( + "class R[reified T: OSError]:\n def m(self, error: T) raises T:\n raise error\n", + &guarded_generic(), + ) + .unwrap(); + assert!( + out.contains("@_by_raises(OSError, \"m\", lambda T: T, (), (\"T\", ))"), + "receiver resolver missing:\n{out}" + ); + } + + #[test] + fn a_nested_function_does_not_read_a_class_parameter_off_its_arguments() { + // `inner` is not a method: its first argument is not a receiver, so the + // class's parameter is tested at its ceiling + let out = transpile( + "class R[reified T: OSError]:\n def m(self):\n def inner(error: T) raises T:\n raise error\n return inner\n", + &guarded_generic(), + ) + .unwrap(); + assert!( + out.contains("@_by_raises(OSError, \"inner\")"), + "nested function should test the ceiling:\n{out}" + ); + } + + #[test] + fn a_mutable_default_moves_its_guard_above_a_guarded_def() { + // the default's guard is a statement placed before the first statement of + // the body, which here is a guarded `def`: it has to land above the + // decorator, not between the decorator and its `def` + let out = transpile( + "def outer(x: list[int] = []):\n def inner() raises TypeError:\n raise TypeError\n return inner\n", + &guarded(), + ) + .unwrap(); + assert!( + out.contains(" @_by_raises(TypeError, \"inner\")\n def inner():"), + "guard should sit directly above its def:\n{out}" + ); + } + #[test] fn guard_that_would_be_dropped_is_an_error() { // `typeof` makes an AST pass re-render the whole statement, discarding an @@ -498,4 +605,18 @@ mod tests { .unwrap(); assert!(!out.contains("_by_raises"), "unexpected guard:\n{out}"); } + + /// the guard checks what a call raises as it raises it, and nothing calls + /// into a stub + #[test] + fn a_stub_gets_no_guard() { + let config = Config { + is_stub: true, + ..guarded() + }; + assert_eq!( + transpile("def f() -> int raises ValueError:\n return 1\n", &config).unwrap(), + "def f() -> int:\n return 1\n" + ); + } } diff --git a/crates/by_transforms/src/transforms/reified_class.rs b/crates/by_transforms/src/transforms/reified_class.rs index fe2ae49431..97ac91b667 100644 --- a/crates/by_transforms/src/transforms/reified_class.rs +++ b/crates/by_transforms/src/transforms/reified_class.rs @@ -58,119 +58,6 @@ use super::reified_generic::REIFIED_MARKER; use super::source_util::{PrologueStatement, body_prologue, line_indent, line_start}; use crate::type_info::TypeInfo; -/// the `generic_class` decorator, injected into the preamble when any class -/// reifies. -/// -/// it replaces the class's `__class_getitem__`, so `A[int]` no longer builds a -/// `typing` alias but a memoized subclass of `A` carrying the type arguments — -/// which is what makes them readable from `__new__` and `__init__` onwards, -/// where an `__orig_class__` stamp applied after construction is not yet there. -/// being a real subclass also keeps `isinstance(a, A)` and `class B(A[int])` -/// working, neither of which survives an alias standing in for a class; the -/// specialization declares an empty `__slots__` so a slotted class stays slotted, -/// and `__init_subclass__` is held back for it, since it is the same class with -/// its arguments fixed rather than a subclass the program wrote. -/// -/// each specialization composes what it binds with what its bases already bound -/// and resolves the chain, so `class B[U](A[U])` specialized as `B[int]` answers -/// `T` with `int` and not with `U`. `__orig_class__` is carried as a class -/// attribute, which is where the alias would have put it, so every reader of a -/// runtime specialization — `_parametric_is` included — sees the same thing it -/// saw before. -/// -/// `_type_argument` answers one read. it takes the receiver rather than the -/// class so a `classmethod` can pass `cls` and everything else `self`, and it -/// raises rather than returning the `TypeVar` object the parameter would -/// otherwise still name — whether because nothing specialized the class or -/// because a base's argument was never filled in -const GENERIC_CLASS_RUNTIME: &str = "\ -def generic_class(cls): - cls.__class_getitem__ = classmethod(_specialize) - return cls - - -def _specialize(cls, item): - args = item if isinstance(item, tuple) else (item,) - if \"__by_type_arguments__\" in cls.__dict__: - raise TypeError(f\"{cls.__name__} is already specialized\") - cache = cls.__dict__.get(\"__by_specializations__\") - if cache is None: - cache = {} - cls.__by_specializations__ = cache - try: - made = cache.get(args) - except TypeError: - raise TypeError( - f\"a type argument to {cls.__name__} is not hashable, so the \" - f\"specialization it names cannot be built\" - ) from None - if made is not None: - return made - params = cls.__type_params__ - bound = {} - for base in reversed(cls.__mro__): - bound.update(base.__dict__.get(\"__by_type_arguments__\") or {}) - bound.update(_bind_type_params(params, args, {}, cls.__name__)) - for param in params: - if param.__name__ not in bound: - raise TypeError( - f\"too few type arguments for {cls.__name__}: \" - f\"no argument for {param.__name__!r}\" - ) - for name, value in bound.items(): - seen = {name} - while isinstance(value, (TypeVar, TypeVarTuple)) and value.__name__ in bound: - if value.__name__ in seen: - break - seen.add(value.__name__) - value = bound[value.__name__] - bound[name] = value - namespace = { - \"__by_type_arguments__\": bound, - \"__orig_class__\": GenericAlias(cls, args), - # the specialization declares nothing of its own, so a slotted class - # stays slotted instead of gaining a `__dict__` here - \"__slots__\": (), - } - # a specialization is the same class with its arguments fixed, not a - # subclass the program wrote, so the hook that greets a subclass must not - # run for it: it would be handed neither the class keywords the definition - # was given nor a class anybody declared - saved = cls.__dict__.get(\"__init_subclass__\", _by_absent) - cls.__init_subclass__ = classmethod(lambda cls, **kwargs: None) - try: - made = type(cls)(cls.__name__, (cls,), namespace) - except TypeError as exc: - # a metaclass that takes class-creation keywords cannot be given them - # again: nothing records what the definition was written with - raise TypeError( - f\"cannot build a specialization of {cls.__name__}: {exc}\" - ) from exc - finally: - if saved is _by_absent: - del cls.__init_subclass__ - else: - cls.__init_subclass__ = saved - made.__module__ = cls.__module__ - made.__qualname__ = cls.__qualname__ - cache[args] = made - return made - - -def _type_argument(owner, name): - cls = owner if isinstance(owner, type) else type(owner) - bound = getattr(cls, \"__by_type_arguments__\", None) - value = _by_absent if bound is None else bound.get(name, _by_absent) - # a value still standing as a type parameter is a base's argument that - # nothing filled in, which means the instance came from the bare class - if value is _by_absent or isinstance(value, (TypeVar, TypeVarTuple)): - raise TypeError( - f\"{cls.__name__} has no type argument for {name!r}: it was not \" - f\"constructed from a specialization\" - ) - return value -"; - /// one reified type parameter bound from the receiver at the top of a method struct TypeArgumentBinding { name: String, @@ -300,27 +187,26 @@ impl<'ast> Visitor<'ast> for ReifiedClass<'_> { pub(crate) struct ReifiedClassPass<'src> { source: &'src str, min_version: PythonVersion, - is_stub: bool, } impl<'src> ReifiedClassPass<'src> { - pub(crate) fn new(source: &'src str, min_version: PythonVersion, is_stub: bool) -> Self { + pub(crate) fn new(source: &'src str, min_version: PythonVersion) -> Self { Self { source, min_version, - is_stub, } } } impl TypeAwarePass for ReifiedClassPass<'_> { + // a stub describes a runtime that lives elsewhere; there is no + // specialization to build here, and the decorator would name a + // polyfill the stub never carries + fn runtime_only(&self) -> bool { + true + } + fn run(&self, stmts: &[Stmt], _types: &dyn TypeInfo, ctx: &mut PassContext) { - // a stub describes a runtime that lives elsewhere; there is no - // specialization to build here, and the decorator would name a - // polyfill the stub never carries - if self.is_stub { - return; - } let mut inner = ReifiedClass::new(self.source, self.min_version); for stmt in stmts { inner.visit_stmt(stmt); @@ -374,15 +260,8 @@ impl TypeAwarePass for ReifiedClassPass<'_> { return; } if inner.used { - ctx.required_imports - .push("from types import GenericAlias".to_owned()); - ctx.required_imports - .push("_by_absent = object()".to_owned()); - ctx.required_imports - .push("from typing import ParamSpec, TypeVar, TypeVarTuple".to_owned()); - ctx.required_imports - .push(super::reified_generic::BIND_TYPE_PARAMS_RUNTIME.to_owned()); - ctx.required_imports.push(GENERIC_CLASS_RUNTIME.to_owned()); + ctx.runtime.insert(crate::runtime::GENERIC_CLASS); + ctx.runtime.insert(crate::runtime::TYPE_ARGUMENT); } ctx.text_edits.extend(inner.edits); ctx.statement_inserts.extend(inner.prologues); diff --git a/crates/by_transforms/src/transforms/reified_generic.rs b/crates/by_transforms/src/transforms/reified_generic.rs index e965deadd6..bb15e42279 100644 --- a/crates/by_transforms/src/transforms/reified_generic.rs +++ b/crates/by_transforms/src/transforms/reified_generic.rs @@ -57,141 +57,6 @@ use super::ast_driver::{PassContext, TypeAwarePass}; use super::source_util::{line_indent, line_start}; use crate::type_info::TypeInfo; -/// the `generic` wrapper, injected into the preamble when any function reifies. -/// -/// `f[int]` produces a specialized `generic` carrying `args=(int,)`; calling it -/// rebuilds the function with a closure whose type-parameter cells hold the -/// type arguments, keyed by `co_freevars` name so unrelated cells (captured -/// locals, `__class__`) survive. parameter defaults, kwonly defaults and the -/// qualname carry over to the rebuilt function. -/// -/// the supplied arguments are mapped onto the parameters by -/// [`BIND_TYPE_PARAMS_RUNTIME`], so `f()` works when every reified parameter -/// carries a pep 696 default; a slot that binding leaves empty and the body -/// reads raises `TypeError` at the call. the wrapper is also a descriptor: -/// `__get__` captures the receiver so a reified *method* (`obj.m[int]()`) binds -/// `self` like an ordinary method. attribute access falls through to the -/// wrapped function, keeping introspection (`f.__name__`, `f.__doc__`) working -const GENERIC_RUNTIME: &str = "\ -class generic: - def __init__(self, fn, args=None, instance=None, fields=None): - self.fn = fn - self.args = args - self.instance = instance - self.fields = fields - - def __repr__(self): - return f\"\" - - def __getattr__(self, name): - if name == \"fn\": - raise AttributeError(name) - return getattr(self.fn, name) - - def __get__(self, obj, objtype=None): - if obj is None: - return self - return generic(self.fn, self.args, obj, self.fields) - - def __getitem__(self, *items, **fields): - if self.args is not None or self.fields is not None: - raise TypeError(\"type arguments already specified\") - if len(items) == 1 and isinstance(items[0], tuple): - items = items[0] - # reject a bad arity here, not at the call - _bind_type_params(self.fn.__type_params__, items, fields, self.fn.__name__) - return generic(self.fn, items, self.instance, fields) - - def __call__(self, *args, **kwargs): - fn = self.fn - code = fn.__code__ - values = _bind_type_params( - fn.__type_params__, self.args or (), self.fields or {}, fn.__name__ - ) - for param in fn.__type_params__: - name = param.__name__ - if name not in values and name in code.co_freevars: - # a synthesized parameter stands for an erased union the user - # never spelled, so naming it would leak the lowering - if name.startswith(\"__by_erased\"): - raise TypeError( - f\"{fn.__name__}() cannot tell which specialization it was \" - f\"given: the argument's type arguments are erased at \" - f\"runtime, and the call site did not record them\" - ) - raise TypeError(f\"{fn.__name__}() missing a type argument for {name!r}\") - closure = tuple( - CellType(values[name]) if name in values else cell - for name, cell in zip(code.co_freevars, fn.__closure__ or ()) - ) - temp_fn = FunctionType(code, fn.__globals__, fn.__name__, fn.__defaults__, closure) - temp_fn.__kwdefaults__ = fn.__kwdefaults__ - temp_fn.__qualname__ = fn.__qualname__ - if self.instance is not None: - return temp_fn(self.instance, *args, **kwargs) - return temp_fn(*args, **kwargs) -"; - -/// binds supplied type arguments onto a type-parameter list, shared by the -/// function wrapper and the class specializer. -/// -/// a `TypeVarTuple` takes, as a tuple, the whole run of positional arguments -/// the fixed parameters around it don't claim, so `[int, str, bool]` on -/// `[T, *Args]` binds `T = int` and `Args = (str, bool)`; a keyword-variadic -/// `**Kwargs` sits outside the positional slots entirely and binds the mapping -/// of the keyword fields (`f[foo=int]` → `Kwargs = {'foo': int}`, spelled -/// `f.__getitem__(foo=int)` in the lowered python, since subscripts take no -/// keywords). an omitted slot is filled from its pep 696 default, read off the -/// parameter list itself; an unfilled `TypeVarTuple` or `**Kwargs` binds empty, -/// and any other slot is simply left out for the caller to answer for. -/// over-specializing a parameter list with no variadic raises -pub(crate) const BIND_TYPE_PARAMS_RUNTIME: &str = "\ -def _bind_type_params(params, supplied, fields, owner): - pack = next((p for p in params if isinstance(p, ParamSpec)), None) - if pack is None and fields: - raise TypeError( - f\"{owner} has no keyword-variadic type parameter for \" - f\"{', '.join(fields)}\" - ) - slots = [p for p in params if p is not pack] - variadic = next( - (i for i, p in enumerate(slots) if isinstance(p, TypeVarTuple)), None - ) - if variadic is None: - if len(supplied) > len(slots): - raise TypeError( - f\"too many type arguments for {owner}: \" - f\"expected {len(slots)}, got {len(supplied)}\" - ) - bound = dict(zip((p.__name__ for p in slots), supplied)) - else: - trailing = slots[variadic + 1:] - packed = tuple(supplied[variadic:len(supplied) - len(trailing)]) - bound = dict(zip((p.__name__ for p in slots[:variadic]), supplied)) - if packed: - bound[slots[variadic].__name__] = packed - bound.update( - zip( - (p.__name__ for p in trailing), - supplied[variadic + len(packed):], - ) - ) - if fields: - bound[pack.__name__] = dict(fields) - for param in params: - name = param.__name__ - if name in bound: - continue - has_default = getattr(param, \"has_default\", None) - if has_default is not None and has_default(): - bound[name] = param.__default__ - elif isinstance(param, TypeVarTuple): - bound[name] = () - elif param is pack: - bound[name] = {} - return bound -"; - /// marker comment appended to the synthesized `@generic` decorator line. the /// reverse transpiler keys on it to re-sugar the wrapper back to a bare `def`; /// a hand-written `@generic` (without the marker) is left untouched. this @@ -314,6 +179,12 @@ impl<'src> ReifiedGenericPass<'src> { } impl TypeAwarePass for ReifiedGenericPass<'_> { + // reification hands a function its type arguments as values when it is + // called, and a call through the stub's declaration is one it never sees + fn runtime_only(&self) -> bool { + true + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut inner = ReifiedGeneric::new(self.source, self.min_version); for stmt in stmts { @@ -357,13 +228,7 @@ impl TypeAwarePass for ReifiedGenericPass<'_> { return; } if inner.used { - ctx.required_imports - .push("from types import CellType, FunctionType".to_owned()); - ctx.required_imports - .push("from typing import ParamSpec, TypeVarTuple".to_owned()); - ctx.required_imports - .push(BIND_TYPE_PARAMS_RUNTIME.to_owned()); - ctx.required_imports.push(GENERIC_RUNTIME.to_owned()); + ctx.runtime.insert(crate::runtime::GENERIC); } ctx.text_edits.extend(inner.edits); } @@ -377,132 +242,33 @@ mod tests { use ruff_python_ast::PythonVersion; fn check_at(input: &str, expected: &str, version: PythonVersion) { + assert_eq!(out_at(input, version), expected); + } + + fn out_at(input: &str, version: PythonVersion) -> String { let config = Config { min_version: version, ..Config::test_default() }; - assert_eq!(transpile(input, &config).unwrap(), expected); + transpile(input, &config).unwrap() } #[test] fn value_position_use_wraps_with_generic() { + // reading `T` in the body is what makes the function reified. the + // wrapper's own source is `_by_runtime.py`'s to specify, so the expected + // preamble is read from there rather than repeated here + let mut preamble = crate::runtime::inline([crate::runtime::GENERIC]).join("\n"); + preamble.push('\n'); check_at( indoc! {" def f[T](t: object): return isinstance(t, T) f[int](1) "}, - indoc! {" - from types import CellType, FunctionType - from typing import ParamSpec, TypeVarTuple - class generic: - def __init__(self, fn, args=None, instance=None, fields=None): - self.fn = fn - self.args = args - self.instance = instance - self.fields = fields - - def __repr__(self): - return f\"\" - - def __getattr__(self, name): - if name == \"fn\": - raise AttributeError(name) - return getattr(self.fn, name) - - def __get__(self, obj, objtype=None): - if obj is None: - return self - return generic(self.fn, self.args, obj, self.fields) - - def __getitem__(self, *items, **fields): - if self.args is not None or self.fields is not None: - raise TypeError(\"type arguments already specified\") - if len(items) == 1 and isinstance(items[0], tuple): - items = items[0] - # reject a bad arity here, not at the call - _bind_type_params(self.fn.__type_params__, items, fields, self.fn.__name__) - return generic(self.fn, items, self.instance, fields) - - def __call__(self, *args, **kwargs): - fn = self.fn - code = fn.__code__ - values = _bind_type_params( - fn.__type_params__, self.args or (), self.fields or {}, fn.__name__ - ) - for param in fn.__type_params__: - name = param.__name__ - if name not in values and name in code.co_freevars: - # a synthesized parameter stands for an erased union the user - # never spelled, so naming it would leak the lowering - if name.startswith(\"__by_erased\"): - raise TypeError( - f\"{fn.__name__}() cannot tell which specialization it was \" - f\"given: the argument's type arguments are erased at \" - f\"runtime, and the call site did not record them\" - ) - raise TypeError(f\"{fn.__name__}() missing a type argument for {name!r}\") - closure = tuple( - CellType(values[name]) if name in values else cell - for name, cell in zip(code.co_freevars, fn.__closure__ or ()) - ) - temp_fn = FunctionType(code, fn.__globals__, fn.__name__, fn.__defaults__, closure) - temp_fn.__kwdefaults__ = fn.__kwdefaults__ - temp_fn.__qualname__ = fn.__qualname__ - if self.instance is not None: - return temp_fn(self.instance, *args, **kwargs) - return temp_fn(*args, **kwargs) - - def _bind_type_params(params, supplied, fields, owner): - pack = next((p for p in params if isinstance(p, ParamSpec)), None) - if pack is None and fields: - raise TypeError( - f\"{owner} has no keyword-variadic type parameter for \" - f\"{', '.join(fields)}\" - ) - slots = [p for p in params if p is not pack] - variadic = next( - (i for i, p in enumerate(slots) if isinstance(p, TypeVarTuple)), None - ) - if variadic is None: - if len(supplied) > len(slots): - raise TypeError( - f\"too many type arguments for {owner}: \" - f\"expected {len(slots)}, got {len(supplied)}\" - ) - bound = dict(zip((p.__name__ for p in slots), supplied)) - else: - trailing = slots[variadic + 1:] - packed = tuple(supplied[variadic:len(supplied) - len(trailing)]) - bound = dict(zip((p.__name__ for p in slots[:variadic]), supplied)) - if packed: - bound[slots[variadic].__name__] = packed - bound.update( - zip( - (p.__name__ for p in trailing), - supplied[variadic + len(packed):], - ) - ) - if fields: - bound[pack.__name__] = dict(fields) - for param in params: - name = param.__name__ - if name in bound: - continue - has_default = getattr(param, \"has_default\", None) - if has_default is not None and has_default(): - bound[name] = param.__default__ - elif isinstance(param, TypeVarTuple): - bound[name] = () - elif param is pack: - bound[name] = {} - return bound - - @generic # basedpython: reified - def f[T](t: object): - return isinstance(t, T) - f[int](1) - "}, + &format!( + "{preamble}@generic # basedpython: reified\ndef f[T](t: object):\n return isinstance(t, T)\nf[int](1)\n" + ), PythonVersion::PY312, ); } @@ -974,4 +740,20 @@ mod tests { "an erased type parameter has no cell to forward: {out}" ); } + + /// a stub declares the function. the wrapper that hands it its type + /// arguments belongs to the implementation, and in a stub it would stand a + /// runtime class where the signature should be + #[test] + fn a_stub_declares_a_reified_function_unwrapped() { + let config = Config { + is_stub: true, + min_version: PythonVersion::PY312, + ..Config::test_default() + }; + assert_eq!( + transpile("def make[reified T]() -> T: ...\n", &config).unwrap(), + "def make[T]() -> T: ...\n" + ); + } } diff --git a/crates/by_transforms/src/transforms/runtime_union.rs b/crates/by_transforms/src/transforms/runtime_union.rs index a2e052cde1..3a13037598 100644 --- a/crates/by_transforms/src/transforms/runtime_union.rs +++ b/crates/by_transforms/src/transforms/runtime_union.rs @@ -48,6 +48,12 @@ impl RuntimeUnionPass { } impl TypeAwarePass for RuntimeUnionPass { + // only a union the runtime evaluates needs the older spelling. a checker + // reads `X | Y` in a stub whatever version the stub is for + fn runtime_only(&self) -> bool { + true + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { if self.min_version >= MIN_VERSION { return; @@ -340,4 +346,19 @@ mod tests { assert!(out.contains("Alias = int | str"), "got:\n{out}"); assert!(out.contains("isinstance(x, int | str)"), "got:\n{out}"); } + + /// nothing in a stub is evaluated, so a union in one keeps the spelling a + /// checker reads whatever the target + #[test] + fn a_stub_keeps_its_unions() { + let config = Config { + is_stub: true, + min_version: PythonVersion::PY39, + ..Config::test_default() + }; + assert_eq!( + transpile("Alias = int | str\n", &config).unwrap(), + "Alias = int | str\n" + ); + } } diff --git a/crates/by_transforms/src/transforms/some_ctor.rs b/crates/by_transforms/src/transforms/some_ctor.rs index b56afd01ea..cc40a078b5 100644 --- a/crates/by_transforms/src/transforms/some_ctor.rs +++ b/crates/by_transforms/src/transforms/some_ctor.rs @@ -1,7 +1,7 @@ //! Runtime lowering for the `Some(...)` optional constructor. //! //! `Some` is the present-case constructor for a wrapped optional. It lowers to -//! the runtime `Optional` wrapper class (see [`wrapped_runtime`]), so `Some(x)` +//! the runtime `Optional` wrapper class (see [`crate::runtime`]), so `Some(x)` //! becomes `Optional(x)`. The class is injected as a polyfill when any `Some` //! reference is rewritten. //! @@ -13,7 +13,6 @@ use ruff_python_ast::{Expr, ExprContext, Stmt}; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{PassContext, TypeAwarePass}; -use super::wrapped_runtime::OPTIONAL_RUNTIME; use crate::type_info::TypeInfo; struct SomeCtor { @@ -62,7 +61,7 @@ impl TypeAwarePass for SomeCtorPass { inner.visit_stmt(stmt); } if inner.used { - ctx.required_imports.push(OPTIONAL_RUNTIME.to_owned()); + ctx.runtime.insert(crate::runtime::OPTIONAL); } ctx.text_edits.extend(inner.edits); } diff --git a/crates/by_transforms/src/transforms/soundness.rs b/crates/by_transforms/src/transforms/soundness.rs index d4b0362977..ab9c101ae7 100644 --- a/crates/by_transforms/src/transforms/soundness.rs +++ b/crates/by_transforms/src/transforms/soundness.rs @@ -69,67 +69,18 @@ use ruff_python_ast::{Comprehension, Expr, ExprCall, Parameter, Stmt, StmtFuncti use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; -use super::parametric_is::{PARAMETRIC_IS_RUNTIME, variance_tuple}; +use super::parametric_is::variance_tuple; use super::source_util::{line_indent, line_start}; use crate::Config; use crate::config::SoundnessPositions; use crate::type_info::{SoundnessCheck, TypeInfo}; -const CHECK_HELPER: &str = "\ -def _soundness_check(_v, _t): - if not isinstance(_v, _t): - raise TypeError( - f\"type soundness violation: expected {getattr(_t, '__name__', _t)}, \" - f\"got {type(_v).__name__}\" - ) - return _v -"; - -const ITER_HELPER: &str = "\ -def _soundness_iter(_it, _t): - for _x in _it: - yield _soundness_check(_x, _t) -"; - -const AITER_HELPER: &str = "\ -async def _soundness_aiter(_it, _t): - async for _x in _it: - yield _soundness_check(_x, _t) -"; - // deep check for a user-generic-specialized target: validates the base class // always, and the reified type arguments when the value carries them // (`__orig_class__`, stamped by `A[int](…)`). a value with no reification // passes the argument check — its parameters aren't available to check, // leaving the base `isinstance` as the guarantee. reuses `_parametric_is` // (and its `_parametric_is_sub`) from `PARAMETRIC_IS_RUNTIME` -const PARAMETRIC_HELPER: &str = "\ -def _soundness_parametric(_v, _alias, _variances): - _alias = _by_alias(_alias) - _origin = getattr(_alias, \"__origin__\", _alias) - if not isinstance(_v, _origin): - raise TypeError( - f\"type soundness violation: expected {getattr(_origin, '__name__', _origin)}, \" - f\"got {type(_v).__name__}\" - ) - if getattr(_v, \"__orig_class__\", None) is not None and not _parametric_is(_v, _alias, _variances): - raise TypeError( - f\"type soundness violation: expected {_alias}, got {_v.__orig_class__}\" - ) - return _v -"; - -const ITER_P_HELPER: &str = "\ -def _soundness_iter_p(_it, _alias, _variances): - for _x in _it: - yield _soundness_parametric(_x, _alias, _variances) -"; - -const AITER_P_HELPER: &str = "\ -async def _soundness_aiter_p(_it, _alias, _variances): - async for _x in _it: - yield _soundness_parametric(_x, _alias, _variances) -"; /// which parameter an argument binds to, for the `arguments` gate enum ArgSlot<'a> { @@ -568,18 +519,18 @@ pub(crate) struct SoundnessPass<'src> { impl<'src> SoundnessPass<'src> { pub(crate) fn new(source: &'src str, config: &Config) -> Self { Self { - // stubs never execute, so checks would only be noise there - positions: if config.is_stub { - SoundnessPositions::none() - } else { - config.soundness - }, + positions: config.soundness, source, } } } impl TypeAwarePass for SoundnessPass<'_> { + // every check validates a value as the program produces it + fn runtime_only(&self) -> bool { + true + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { if !self.positions.any() { return; @@ -596,25 +547,24 @@ impl TypeAwarePass for SoundnessPass<'_> { if inner.edits.is_empty() && inner.guards.is_empty() { return; } - ctx.required_imports.push(CHECK_HELPER.to_owned()); + ctx.runtime.insert(crate::runtime::SOUNDNESS_CHECK); if inner.used_iter { - ctx.required_imports.push(ITER_HELPER.to_owned()); + ctx.runtime.insert(crate::runtime::SOUNDNESS_ITER); } if inner.used_aiter { - ctx.required_imports.push(AITER_HELPER.to_owned()); + ctx.runtime.insert(crate::runtime::SOUNDNESS_AITER); } // a deep parametric check reuses the `_parametric_is` probe (which // brings its own `_parametric_is_sub`); function names resolve at call // time, so the def order among these preamble helpers is irrelevant if inner.used_parametric { - ctx.required_imports.push(PARAMETRIC_IS_RUNTIME.to_owned()); - ctx.required_imports.push(PARAMETRIC_HELPER.to_owned()); + ctx.runtime.insert(crate::runtime::SOUNDNESS_PARAMETRIC); } if inner.used_iter_p { - ctx.required_imports.push(ITER_P_HELPER.to_owned()); + ctx.runtime.insert(crate::runtime::SOUNDNESS_ITER_P); } if inner.used_aiter_p { - ctx.required_imports.push(AITER_P_HELPER.to_owned()); + ctx.runtime.insert(crate::runtime::SOUNDNESS_AITER_P); } ctx.template_edits.extend(inner.edits); ctx.statement_inserts.extend(inner.guards); @@ -1322,4 +1272,16 @@ mod tests { let out = check_with("def f(s: str): ...\n", crate::SoundnessPositions::all()); assert!(out.contains("_soundness_check(s, str)"), "got:\n{out}"); } + + /// a stub is never run, so it produces no value to check + #[test] + fn a_stub_gets_no_checks() { + let source = "def f(x: int) -> int:\n return x\n"; + let config = Config { + is_stub: true, + soundness: crate::SoundnessPositions::all(), + ..Config::test_default() + }; + assert_eq!(transpile(source, &config).unwrap(), source); + } } diff --git a/crates/by_transforms/src/transforms/source_util.rs b/crates/by_transforms/src/transforms/source_util.rs index 971abd685d..8b52ed99df 100644 --- a/crates/by_transforms/src/transforms/source_util.rs +++ b/crates/by_transforms/src/transforms/source_util.rs @@ -575,7 +575,7 @@ fn body_range(stmt: &Stmt, header_end: TextSize) -> Option { /// The offset past everything a `def`'s header can span, so a statement before /// it is one the parser synthesized from a parameter rather than a body. -fn header_end(f: &StmtFunctionDef) -> TextSize { +pub(crate) fn header_end(f: &StmtFunctionDef) -> TextSize { f.parameters.range().end().max( f.returns .as_ref() diff --git a/crates/by_transforms/src/transforms/statement_expression.rs b/crates/by_transforms/src/transforms/statement_expression.rs index ac840f88a6..c70c183db5 100644 --- a/crates/by_transforms/src/transforms/statement_expression.rs +++ b/crates/by_transforms/src/transforms/statement_expression.rs @@ -888,18 +888,20 @@ mod tests { /// and leak the keyword into the output #[test] fn a_declarations_lowering_survives_the_moved_prefix() { - for (declaration, lowered) in [ - ("let a", "a: Final"), - ("var a", "a"), - ("let a: int", "a: Final[int]"), - ("final a: int", "a: Final[int]"), - ("private a: int", "a: int"), + // a module-level `private` variable is emitted under `_a`, and so is the + // `print` that reads it + for (declaration, lowered, read) in [ + ("let a", "a: Final", "a"), + ("var a", "a", "a"), + ("let a: int", "a: Final[int]", "a"), + ("final a: int", "a: Final[int]", "a"), + ("private a: int", "_a: int", "_a"), ] { let out = check(&format!( "b: int? = None\n{declaration} = b ?? raise ValueError()\nprint(a)\n" )); assert!( - out.contains(&format!("{lowered} = __by_stmt_expr_0__\nprint(a)")), + out.contains(&format!("{lowered} = __by_stmt_expr_0__\nprint({read})")), "`{declaration}`, got:\n{out}" ); assert!(!out.contains(declaration), "`{declaration}`, got:\n{out}"); diff --git a/crates/by_transforms/src/transforms/string_tag.rs b/crates/by_transforms/src/transforms/string_tag.rs index 46e11c5576..1c8d952b28 100644 --- a/crates/by_transforms/src/transforms/string_tag.rs +++ b/crates/by_transforms/src/transforms/string_tag.rs @@ -36,52 +36,6 @@ use crate::Config; use super::ast_driver::{AstPass, Fragment, PassContext}; -/// PEP 750 `Template` / `Interpolation` polyfill for runtimes before 3.14. -/// -/// matches the `string.templatelib` shape a tag relies on: `Template.strings` -/// is the literal segments (always one more than the interpolations), -/// `Template.interpolations` is the replacement fields, and `Template.values` -/// is their evaluated values. iterating a `Template` yields the segments and -/// interpolations interleaved in source order, the same as the stdlib type -const TEMPLATE_RUNTIME: &str = "\ -class _Interpolation: - def __init__(self, value, expression, conversion=None, format_spec=\"\"): - self.value = value - self.expression = expression - self.conversion = conversion - self.format_spec = format_spec - - -class _Template: - def __init__(self, *args): - strings = [] - interpolations = [] - if not args or isinstance(args[-1], _Interpolation): - args = (*args, \"\") - pending = \"\" - for arg in args: - if isinstance(arg, _Interpolation): - strings.append(pending) - pending = \"\" - interpolations.append(arg) - else: - pending += arg - strings.append(pending) - self.strings = tuple(strings) - self.interpolations = tuple(interpolations) - - @property - def values(self): - return tuple(i.value for i in self.interpolations) - - def __iter__(self): - for index, string in enumerate(self.strings): - if string: - yield string - if index < len(self.interpolations): - yield self.interpolations[index] -"; - pub(crate) struct StringTagPass<'src> { source: &'src str, config: Config, @@ -109,7 +63,8 @@ impl AstPass for StringTagPass<'_> { ctx.text_edits.extend(state.text_edits); ctx.template_edits.extend(state.template_edits); if state.used_polyfill { - ctx.required_imports.push(TEMPLATE_RUNTIME.to_owned()); + ctx.runtime.insert(crate::runtime::TEMPLATE); + ctx.runtime.insert(crate::runtime::INTERPOLATION); } } } @@ -292,14 +247,21 @@ mod tests { } /// transpile at the default 3.10 target, where the polyfill is injected. - /// the polyfill class is prepended and separated from the body by a blank - /// line, the same as other injected runtime classes + /// the polyfill classes are prepended, each separated from what follows by + /// a blank line, the same as other injected runtime definitions. what they + /// are is read back from the runtime rather than repeated here, so this + /// test says where the polyfill goes and `_by_runtime.py` says what it is fn polyfilled(input: &str, expected_body: &str) { let out = transpile(input, &Config::test_default()).unwrap(); - let body = out - .strip_prefix(super::TEMPLATE_RUNTIME) - .and_then(|rest| rest.strip_prefix('\n')) - .unwrap_or_else(|| panic!("template polyfill not prepended; got:\n{out}")); + let mut body = out.as_str(); + for definition in + crate::runtime::inline([crate::runtime::TEMPLATE, crate::runtime::INTERPOLATION]) + { + body = body + .strip_prefix(definition.as_str()) + .and_then(|rest| rest.strip_prefix('\n')) + .unwrap_or_else(|| panic!("template polyfill not prepended; got:\n{out}")); + } assert_eq!(body, expected_body); } diff --git a/crates/by_transforms/src/transforms/trailing_lambda.rs b/crates/by_transforms/src/transforms/trailing_lambda.rs index 33394a84b9..036dde0fcb 100644 --- a/crates/by_transforms/src/transforms/trailing_lambda.rs +++ b/crates/by_transforms/src/transforms/trailing_lambda.rs @@ -711,19 +711,21 @@ mod tests { /// swallows the newline its block already consumed #[test] fn block_as_a_declaration_value() { - for (declaration, lowered) in [ - ("let result", "result: Final"), - ("let result: str", "result: Final[str]"), - ("final result: str", "result: Final[str]"), - ("var result", "result"), - ("private result: str", "result: str"), + // a module-level `private` variable is emitted under `_result`, and so is + // the `print` that reads it + for (declaration, lowered, read) in [ + ("let result", "result: Final", "result"), + ("let result: str", "result: Final[str]", "result"), + ("final result: str", "result: Final[str]", "result"), + ("var result", "result", "result"), + ("private result: str", "_result: str", "_result"), ] { let out = check(&format!( "def f(a: (int) -> None) -> str:\n a(1)\n return \"done\"\n\n{declaration} = f:\n print(it)\nprint(result)\n" )); assert!( out.contains(&format!( - "def _trailing_lambda_0(it=None):\n print(it)\n{lowered} = f(a=_trailing_lambda_0)\nprint(result)" + "def _trailing_lambda_0(it=None):\n print(it)\n{lowered} = f(a=_trailing_lambda_0)\nprint({read})" )), "`{declaration}`, got:\n{out}" ); diff --git a/crates/by_transforms/src/transforms/type_reification.rs b/crates/by_transforms/src/transforms/type_reification.rs index 81d9183470..9e123ab8eb 100644 --- a/crates/by_transforms/src/transforms/type_reification.rs +++ b/crates/by_transforms/src/transforms/type_reification.rs @@ -176,21 +176,23 @@ impl<'ast> Visitor<'ast> for Reifier<'_> { pub(crate) struct TypeReificationPass { min_version: PythonVersion, - is_stub: bool, } impl TypeReificationPass { - pub(crate) fn new(min_version: PythonVersion, is_stub: bool) -> Self { - Self { - min_version, - is_stub, - } + pub(crate) fn new(min_version: PythonVersion) -> Self { + Self { min_version } } } impl TypeAwarePass for TypeReificationPass { + // the specialization is spelled out so the value a call constructs + // carries it as the program runs + fn runtime_only(&self) -> bool { + true + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - if self.min_version < PythonVersion::PY39 || self.is_stub { + if self.min_version < PythonVersion::PY39 { return; } let mut reifier = Reifier { diff --git a/crates/by_transforms/src/transforms/typed_dict_literal.rs b/crates/by_transforms/src/transforms/typed_dict_literal.rs index bb205d2929..ce917c4e18 100644 --- a/crates/by_transforms/src/transforms/typed_dict_literal.rs +++ b/crates/by_transforms/src/transforms/typed_dict_literal.rs @@ -457,8 +457,7 @@ impl AstPass for TypedDictLiteralPass<'_> { .push("from typing import Literal".to_owned()); } if inner.needs_optional_runtime { - ctx.required_imports - .push(super::wrapped_runtime::OPTIONAL_RUNTIME.to_owned()); + ctx.runtime.insert(crate::runtime::OPTIONAL); } if inner.needs_import { // synthesized classes use `closed=True` / `extra_items=T` (PEP 728), diff --git a/crates/by_transforms/src/transforms/typing_redirect.rs b/crates/by_transforms/src/transforms/typing_redirect.rs index 135a47207b..eac2d6811c 100644 --- a/crates/by_transforms/src/transforms/typing_redirect.rs +++ b/crates/by_transforms/src/transforms/typing_redirect.rs @@ -195,4 +195,20 @@ mod tests { "from typing import Self\n", ); } + + /// a stub is never imported, so a name it takes from `typing_extensions` is + /// the checker's to resolve and nothing the built package has to install + #[test] + fn a_stub_needs_nothing_installed() { + let source = "from typing import Self\n"; + let (_, needed) = crate::transpile_with_report(source, &Config::test_default()).unwrap(); + assert_eq!(needed.specifiers(), ["typing_extensions>=4.12"]); + let stub = Config { + is_stub: true, + ..Config::test_default() + }; + let (out, needed) = crate::transpile_with_report(source, &stub).unwrap(); + assert_eq!(out, "from typing_extensions import Self\n"); + assert!(needed.specifiers().is_empty()); + } } diff --git a/crates/by_transforms/src/transforms/unique_loop_bindings.rs b/crates/by_transforms/src/transforms/unique_loop_bindings.rs index 7dafa22d7e..338445c7e9 100644 --- a/crates/by_transforms/src/transforms/unique_loop_bindings.rs +++ b/crates/by_transforms/src/transforms/unique_loop_bindings.rs @@ -54,38 +54,6 @@ use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::source_util::{line_indent, line_start}; use crate::type_info::{CaptureKind, TypeInfo}; -/// rebuilds a function with fresh cells for the loop bindings it captured, so -/// the iteration that defined it keeps its own values. cells the call does not -/// name — outer locals, `__class__`, reified type parameters — are carried -/// over, as are the attributes `FunctionType` does not copy -const LOOP_BIND_RUNTIME: &str = "\ -def _by_loop_bind(**_by_values): - def _by_rebind(_by_fn): - _by_code = _by_fn.__code__ - _by_bound = FunctionType( - _by_code, - _by_fn.__globals__, - _by_fn.__name__, - _by_fn.__defaults__, - tuple( - CellType(_by_values[_by_name]) if _by_name in _by_values else _by_cell - for _by_name, _by_cell in zip(_by_code.co_freevars, _by_fn.__closure__ or ()) - ), - ) - _by_bound.__kwdefaults__ = _by_fn.__kwdefaults__ - _by_bound.__qualname__ = _by_fn.__qualname__ - _by_bound.__doc__ = _by_fn.__doc__ - _by_bound.__dict__.update(_by_fn.__dict__) - if hasattr(_by_fn, \"__annotate__\"): - _by_bound.__annotate__ = _by_fn.__annotate__ - else: - _by_bound.__annotations__ = _by_fn.__annotations__ - if hasattr(_by_fn, \"__type_params__\"): - _by_bound.__type_params__ = _by_fn.__type_params__ - return _by_bound - return _by_rebind -"; - /// which of a loop's bindings a lowering can bind by value #[derive(Clone, Copy, PartialEq, Eq)] enum Reach { @@ -572,6 +540,11 @@ impl<'src> UniqueLoopBindingsPass<'src> { } impl TypeAwarePass for UniqueLoopBindingsPass<'_> { + // what a closure captured only matters once it is called + fn runtime_only(&self) -> bool { + true + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { if !self.enabled { return; @@ -581,9 +554,7 @@ impl TypeAwarePass for UniqueLoopBindingsPass<'_> { inner.visit_stmt(stmt); } if inner.used_runtime { - ctx.required_imports - .push("from types import CellType, FunctionType".to_owned()); - ctx.required_imports.push(LOOP_BIND_RUNTIME.to_owned()); + ctx.runtime.insert(crate::runtime::LOOP_BIND); } ctx.text_edits.extend(inner.decorators); ctx.template_edits.extend(inner.wraps); @@ -603,14 +574,14 @@ mod tests { ); } - /// the decorated form emits the rebind runtime ahead of the body; the tests - /// below assert the body only + /// the decorated form emits the rebind runtime; the tests below assert + /// everything else. exactly the runtime's text is removed, so whatever else + /// the preamble holds is still asserted fn check_body(input: &str, expected: &str) { let out = transpile(input, &Config::test_default()).unwrap(); - let body = out - .split_once(" return _by_rebind\n") - .map(|(_, body)| body.trim_start_matches('\n').to_owned()) - .unwrap_or(out); + let mut runtime = crate::runtime::inline([crate::runtime::LOOP_BIND]).join("\n"); + runtime.push('\n'); + let body = out.replacen(&runtime, "", 1); assert_eq!(body, crate::python_passthrough::lazify_expected(expected)); } @@ -678,6 +649,7 @@ mod tests { return value "}, indoc! {" + _MISSING = object() def build(): for i in items: @_by_loop_bind(i=i) @@ -848,6 +820,7 @@ mod tests { return cb "}, indoc! {" + _MISSING = object() def build(): for i in items: @_by_loop_bind(i=i) @@ -1107,4 +1080,20 @@ mod tests { }; assert_eq!(transpile(source, &config).unwrap(), source); } + + /// what a closure captured only matters once it is called, and nothing in a + /// stub is + #[test] + fn a_stub_binds_nothing() { + let source = indoc! {" + fns = [] + for i in [1, 2, 3]: + fns.append(lambda: print(i)) + "}; + let config = Config { + is_stub: true, + ..Config::test_default() + }; + assert_eq!(transpile(source, &config).unwrap(), source); + } } diff --git a/crates/by_transforms/src/transforms/visibility_rename.rs b/crates/by_transforms/src/transforms/visibility_rename.rs new file mode 100644 index 0000000000..a2f165b912 --- /dev/null +++ b/crates/by_transforms/src/transforms/visibility_rename.rs @@ -0,0 +1,677 @@ +//! access sites of a member declared with a visibility keyword (basedpython) +//! +//! a visibility keyword renames the member it is written on: `private def +//! helper()` in a class body lowers to `def __helper()`, which python +//! name-mangles to `_A__helper` while class `A`'s body is executed, and +//! `protected def helper()` lowers to `def _helper()`. Either way the access +//! sites keep the name the source wrote, so they have to be pointed at the same +//! attribute: +//! +//! ```by +//! class A: +//! private def helper(self) -> int: +//! return 1 +//! +//! def use(self) -> int: +//! return self.helper() +//! ``` +//! +//! → +//! +//! ```python +//! class A: +//! def __helper(self) -> int: +//! return 1 +//! +//! def use(self) -> int: +//! return self._A__helper() +//! ``` +//! +//! the mangled name is written out rather than left to python: python mangles +//! lexically, so `self.__helper` would mean `_B__helper` in a subclass's body +//! and `__helper` outside a class altogether, while `_A__helper` names the same +//! attribute from every one of those places + +use std::collections::HashSet; + +use ruff_python_ast::helpers::{MemberVisibility, declaration_marker_visibility}; +use ruff_python_ast::statement_visitor::{self, StatementVisitor}; +use ruff_python_ast::visitor::{Visitor, walk_except_handler, walk_expr, walk_pattern, walk_stmt}; +use ruff_python_ast::{self as ast, Expr, ExprContext, Stmt}; +use ruff_text_size::{Ranged, TextRange, TextSize}; + +use super::ast_driver::{PassContext, TypeAwarePass}; +use super::modifiers::module_private_name; +use super::source_util::header_end; +use crate::type_info::TypeInfo; + +pub(crate) struct VisibilityRenamePass; + +impl TypeAwarePass for VisibilityRenamePass { + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { + let module_private: HashSet = types.private_module_symbols().into_iter().collect(); + let mut declarations = RenamedDeclarations::default(); + declarations.visit_body(stmts); + let mut renamer = Renamer { + types, + edits: Vec::new(), + header_end: TextSize::new(0), + module_private, + declarations: declarations.targets, + scope: LexicalScope::Module, + globals: HashSet::new(), + }; + for stmt in stmts { + renamer.visit_stmt(stmt); + } + ctx.text_edits.extend(renamer.edits); + } +} + +/// the names a declaration's own lowering renames — every annotated target whose +/// marker carries a visibility keyword, at any depth. the `modifiers` pass writes +/// those along with the keyword prefix it erases, so they are not references for +/// this pass to rename a second time. a type alias's name is not among them: +/// `modifiers` only erases its keyword, and the name is renamed here like every +/// other reference to it +#[derive(Default)] +struct RenamedDeclarations { + targets: HashSet, +} + +impl<'a> StatementVisitor<'a> for RenamedDeclarations { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + if let Stmt::AnnAssign(assign) = stmt + && declaration_marker_visibility(&assign.annotation) != MemberVisibility::Public + && let Expr::Name(name) = assign.target.as_ref() + { + self.targets.insert(name.range); + } + statement_visitor::walk_stmt(self, stmt); + } +} + +/// the kind of scope a statement is directly in +#[derive(Clone, Copy, PartialEq, Eq)] +enum LexicalScope { + Module, + Class, + Function, +} + +/// the names a function body declares `global` — python's `global` holds for the +/// whole body, wherever in it the statement is written, and not for a nested +/// function or class +fn declared_globals(body: &[Stmt]) -> HashSet { + #[derive(Default)] + struct Globals(HashSet); + impl<'a> StatementVisitor<'a> for Globals { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + match stmt { + Stmt::Global(global) => { + self.0 + .extend(global.names.iter().map(|name| name.as_str().to_owned())); + } + Stmt::FunctionDef(_) | Stmt::ClassDef(_) => {} + _ => statement_visitor::walk_stmt(self, stmt), + } + } + } + let mut globals = Globals::default(); + globals.visit_body(body); + globals.0 +} + +/// whether a `def` or `class` carries the synthetic decorator a visibility +/// keyword parses to — its name is then renamed by the `modifiers` pass +fn has_visibility_modifier(decorators: &[ast::Decorator]) -> bool { + decorators.iter().any(|decorator| { + matches!( + &decorator.expression, + Expr::Name(name) + if name.ctx == ExprContext::Invalid + && matches!(name.id.as_str(), "private" | "protected") + ) + }) +} + +struct Renamer<'a> { + types: &'a dyn TypeInfo, + edits: Vec<(ruff_text_size::TextRange, String)>, + /// the offset past the enclosing `def`'s header, or zero outside one + header_end: TextSize, + /// the module-level names this file declares `private` + module_private: HashSet, + /// see [`RenamedDeclarations`] + declarations: HashSet, + /// the kind of scope the statement being visited is directly in + scope: LexicalScope, + /// the names the enclosing function declares `global` + globals: HashSet, +} + +impl Renamer<'_> { + /// `__slots__` and `__match_args__` name members by string, so a string that + /// names a member a visibility keyword renames is renamed too. python mangles a + /// `__slots__` entry as it does the class body's own names, so it takes the + /// class-body spelling; `__match_args__` is read with `getattr`, which does not, + /// so it takes the spelling that reaches the attribute from anywhere + fn rename_member_name_lists(&mut self, class: &ast::StmtClassDef) { + for stmt in &class.body { + let (targets, value) = match stmt { + Stmt::Assign(assign) => (assign.targets.as_slice(), assign.value.as_ref()), + Stmt::AnnAssign(assign) => match &assign.value { + Some(value) => (std::slice::from_ref(assign.target.as_ref()), value.as_ref()), + None => continue, + }, + _ => continue, + }; + let Some(list) = targets.iter().find_map(|target| match target { + Expr::Name(name) if matches!(name.id.as_str(), "__slots__" | "__match_args__") => { + Some(name.id.as_str()) + } + _ => None, + }) else { + continue; + }; + let elements = match value { + Expr::Tuple(tuple) => tuple.elts.as_slice(), + Expr::List(list) => list.elts.as_slice(), + Expr::Set(set) => set.elts.as_slice(), + Expr::StringLiteral(_) => std::slice::from_ref(value), + _ => continue, + }; + for element in elements { + let Expr::StringLiteral(string) = element else { + continue; + }; + let [part] = string.value.as_slice() else { + continue; + }; + let Some((in_body, anywhere)) = self + .types + .class_member_spellings(class, part.value.as_ref()) + else { + continue; + }; + let renamed = if list == "__slots__" { + in_body + } else { + anywhere + }; + self.edits.push((part.content_range(), renamed)); + } + } + } + + /// whether a binding of `name` written here binds the module's private + /// symbol — at module level, or in a function that declares it `global` + fn binds_module(&self, name: &str) -> bool { + self.module_private.contains(name) + && match self.scope { + LexicalScope::Module => true, + LexicalScope::Function => self.globals.contains(name), + LexicalScope::Class => false, + } + } + + /// renames an identifier that binds a module-level private symbol + fn rename_binding(&mut self, name: &ast::Identifier) { + if self.binds_module(name.as_str()) { + self.edits + .push((name.range, module_private_name(name.as_str()))); + } + } + + /// an import that rebinds a module-level private symbol binds its + /// underscored name instead. an alias is renamed; an import written without + /// one gains one, since the name after `import` is what is being imported + fn rename_import_binding(&mut self, alias: &ast::Alias, plain_import: bool) { + let name = alias.name.as_str(); + let bound = match &alias.asname { + Some(asname) => asname.as_str(), + None if plain_import => name.split('.').next().unwrap_or(name), + None => name, + }; + if !self.binds_module(bound) { + return; + } + let renamed = module_private_name(bound); + match &alias.asname { + Some(asname) => self.edits.push((asname.range, renamed)), + // `import a.b` binds its top-level package, which no alias keeps: the + // checker reports the rebinding instead + None if plain_import && name.contains('.') => {} + None => self.edits.push(( + TextRange::empty(alias.name.range.end()), + format!(" as {renamed}"), + )), + } + } +} + +impl<'ast> Visitor<'ast> for Renamer<'_> { + fn visit_except_handler(&mut self, handler: &'ast ast::ExceptHandler) { + let ast::ExceptHandler::ExceptHandler(handler_node) = handler; + if let Some(name) = &handler_node.name { + self.rename_binding(name); + } + walk_except_handler(self, handler); + } + + fn visit_pattern(&mut self, pattern: &'ast ast::Pattern) { + match pattern { + ast::Pattern::MatchAs(ast::PatternMatchAs { + name: Some(name), .. + }) + | ast::Pattern::MatchStar(ast::PatternMatchStar { + name: Some(name), .. + }) + | ast::Pattern::MatchMapping(ast::PatternMatchMapping { + rest: Some(name), .. + }) => self.rename_binding(name), + // `case A(x=...)` reads `x` off the subject, so it names the attribute + // an access would + ast::Pattern::MatchClass(class_pattern) => { + for keyword in &class_pattern.arguments.keywords { + if let Some(renamed) = self + .types + .class_pattern_keyword_name(class_pattern, keyword.attr.as_str()) + { + self.edits.push((keyword.attr.range, renamed)); + } + } + } + _ => {} + } + walk_pattern(self, pattern); + } + + fn visit_stmt(&mut self, stmt: &'ast Stmt) { + // a statement the parser synthesized from an `init(…)` parameter carries + // that parameter's range, so it sits inside the signature rather than the + // body. an edit keyed on one would rename the parameter; the `init_method` + // pass writes those lines itself, visibility spelled out and all + if stmt.range().start() < self.header_end { + return; + } + // `global count` names the module's binding by definition, so it follows + // the rename wherever it is written + if let Stmt::Global(global) = stmt { + for name in &global.names { + if self.module_private.contains(name.as_str()) { + self.edits + .push((name.range, module_private_name(name.as_str()))); + } + } + } + match stmt { + Stmt::FunctionDef(function) => { + if !has_visibility_modifier(&function.decorator_list) { + self.rename_binding(&function.name); + } + let enclosing = ( + std::mem::replace(&mut self.header_end, header_end(function)), + std::mem::replace(&mut self.scope, LexicalScope::Function), + std::mem::replace(&mut self.globals, declared_globals(&function.body)), + ); + walk_stmt(self, stmt); + (self.header_end, self.scope, self.globals) = enclosing; + return; + } + Stmt::ClassDef(class) => { + if !has_visibility_modifier(&class.decorator_list) { + self.rename_binding(&class.name); + } + self.rename_member_name_lists(class); + let enclosing = std::mem::replace(&mut self.scope, LexicalScope::Class); + walk_stmt(self, stmt); + self.scope = enclosing; + return; + } + Stmt::Import(import) => { + for alias in &import.names { + self.rename_import_binding(alias, true); + } + } + Stmt::ImportFrom(import) => { + for alias in &import.names { + if alias.name.as_str() != "*" { + self.rename_import_binding(alias, false); + } + } + } + _ => {} + } + walk_stmt(self, stmt); + } + + fn visit_expr(&mut self, expr: &'ast Expr) { + // a reference to a module-level private symbol is renamed only when it is + // one — a parameter, a local or a class attribute that shares the name is a + // different binding, and renaming it would split it from its uses — and a + // bare name in a class body when it is one of the class's own restricted + // members. ty is asked where each resolves + if let Expr::Name(name) = expr + && name.ctx != ExprContext::Invalid + // a name the parser synthesized has no source to rename + && !name.range.is_empty() + && !self.declarations.contains(&name.range) + { + if self.module_private.contains(name.id.as_str()) + && self.types.resolves_to_module_scope(name) == Some(true) + { + self.edits + .push((name.range, module_private_name(name.id.as_str()))); + } else if let Some(renamed) = self.types.class_body_member_name(name) { + self.edits.push((name.range, renamed)); + } + } + if let Expr::Attribute(attribute) = expr + && let Some(mangled) = self.types.restricted_member_name(attribute) + { + self.edits.push((attribute.attr.range(), mangled)); + } + walk_expr(self, expr); + } +} + +#[cfg(test)] +mod tests { + use crate::{Config, transpile}; + use indoc::indoc; + + fn out(input: &str) -> String { + transpile(input, &Config::test_default()).unwrap() + } + + #[test] + fn a_call_reaches_the_mangled_definition() { + let out = out(indoc! {" + class A: + private def helper(self) -> int: + return 1 + + def use(self) -> int: + return self.helper() + "}); + assert!(out.contains("def __helper(self) -> int:"), "got:\n{out}"); + assert!(out.contains("return self._A__helper()"), "got:\n{out}"); + } + + #[test] + fn a_call_from_a_nested_scope_reaches_the_declaring_class() { + // the comprehension is its own scope, and python mangles lexically — + // the declaring class is what the written-out name records + let out = out(indoc! {" + class A: + private def helper(self) -> int: + return 1 + + def use(self) -> list[int]: + return [self.helper() for _ in range(2)] + "}); + assert!(out.contains("[self._A__helper() for _"), "got:\n{out}"); + } + + #[test] + fn a_protected_member_keeps_one_name_across_the_hierarchy() { + // `protected` renames to `_helper`, which python does not mangle, so a + // subclass reaches the same attribute under the same name + let out = out(indoc! {" + class A: + protected def helper(self) -> int: + return 1 + + class B(A): + def use(self) -> int: + return self.helper() + "}); + assert!(out.contains("def _helper(self) -> int:"), "got:\n{out}"); + assert!(out.contains("return self._helper()"), "got:\n{out}"); + } + + #[test] + fn a_private_attribute_is_reached_by_its_mangled_name() { + let out = out(indoc! {" + class A: + private count: int = 0 + + def bump(self) -> int: + self.count = self.count + 1 + return self.count + "}); + assert!(out.contains("__count: int = 0"), "got:\n{out}"); + assert!( + out.contains("self._A__count = self._A__count + 1"), + "got:\n{out}" + ); + } + + #[test] + fn an_init_parameter_attribute_is_reached_by_the_name_it_was_written_with() { + // the parameter keeps its own name; only the attribute is renamed + let out = out(indoc! {" + class A: + init(private let x: int) + + def get(self) -> int: + return self.x + "}); + assert!(out.contains("def __init__(self, x: int):"), "got:\n{out}"); + assert!(out.contains("self.__x: int = x"), "got:\n{out}"); + assert!(out.contains("return self._A__x"), "got:\n{out}"); + } + + #[test] + fn a_visibility_keyword_composes_with_a_class_variable() { + let out = out(indoc! {" + class Outer: + private class var count: int = 0 + protected class let LIMIT: int = 3 + private class total = 0 + + def read(self) -> int: + return self.count + self.LIMIT + self.total + "}); + assert!(out.contains("__count: ClassVar[int] = 0"), "got:\n{out}"); + assert!(out.contains("_LIMIT: Final[int] = 3"), "got:\n{out}"); + assert!(out.contains("__total: ClassVar = 0"), "got:\n{out}"); + assert!( + out.contains("return self._Outer__count + self._LIMIT + self._Outer__total"), + "got:\n{out}" + ); + } + + #[test] + fn a_class_variable_is_reached_through_the_class_object() { + // python mangles the declaration in the class body, so an access through + // the class itself has to name the same attribute + let out = out(indoc! {" + class Registry: + private class var made: int = 0 + + @classmethod + def total(cls) -> int: + return cls.made + + def bump(self) -> None: + Registry.made += 1 + "}); + assert!(out.contains("return cls._Registry__made"), "got:\n{out}"); + assert!(out.contains("Registry._Registry__made += 1"), "got:\n{out}"); + } + + #[test] + fn a_module_level_private_variable_is_renamed_where_it_is_the_symbol() { + let out = out(indoc! {" + private count: int = 0 + + def bump() -> int: + global count + count = count + 1 + return count + + def shadow(count: int) -> int: + return count + + class Holder: + count = 5 + + def get(self) -> int: + return count + "}); + assert!(out.contains("_count: int = 0"), "got:\n{out}"); + assert!(out.contains("global _count"), "got:\n{out}"); + assert!(out.contains("_count = _count + 1"), "got:\n{out}"); + // a parameter and a class attribute that share the name are bindings of + // their own, and are left alone with every use of them + assert!( + out.contains("def shadow(count: int) -> int:\n return count\n"), + "got:\n{out}" + ); + assert!(out.contains("class Holder:\n count"), "got:\n{out}"); + // a method reads past its class body to the module + assert!(out.contains(" return _count"), "got:\n{out}"); + } + + #[test] + fn a_member_is_renamed_wherever_the_class_body_declares_it() { + // a declaration inside a nested block, a decorated method and a nested + // class are members like any other, and a bare name in the class body + // reads the declaration under the spelling python mangles there + let out = out(indoc! {" + class Shape: + if True: + private sides: int = 3 + private scale: int = 2 + doubled = scale * 2 + + private class Inner: + pass + + @staticmethod + private def make() -> int: + return 4 + + def area(self) -> int: + return self.sides * self.scale + self.make() + + def inner(self) -> object: + return Shape.Inner() + "}); + assert!(out.contains(" __sides: int = 3"), "got:\n{out}"); + assert!(out.contains("doubled: int = __scale * 2"), "got:\n{out}"); + assert!(out.contains("class __Inner:"), "got:\n{out}"); + assert!(out.contains(" def __make() -> int:"), "got:\n{out}"); + assert!( + out.contains("self._Shape__sides * self._Shape__scale + self._Shape__make()"), + "got:\n{out}" + ); + assert!(out.contains("return Shape._Shape__Inner()"), "got:\n{out}"); + } + + #[test] + fn a_name_list_names_the_member_the_way_its_reader_looks_it_up() { + // python mangles a `__slots__` entry like a name in the class body, but + // reads a `__match_args__` entry and a class pattern's keyword with + // `getattr`, which mangles nothing + let out = out(indoc! {" + class Slotted: + __slots__ = ('x',) + private x: int + + class Point: + __match_args__ = ('x',) + private x: int + + def unpack(self) -> int: + match self: + case Point(x=v): + return v + return -1 + "}); + assert!(out.contains("__slots__ = ('__x',)"), "got:\n{out}"); + assert!( + out.contains("__match_args__ = ('_Point__x',)"), + "got:\n{out}" + ); + assert!(out.contains("case Point(_Point__x=v):"), "got:\n{out}"); + } + + #[test] + fn a_protected_member_is_reached_through_super_and_a_union() { + let out = out(indoc! {" + class Base: + protected def hook(self) -> int: + return 1 + + class Child(Base): + protected override def hook(self) -> int: + return super().hook() + 1 + + def pick(self, o: Child | Other) -> int: + return o.hook() + + class Other(Base): ... + "}); + assert!(out.contains("return super()._hook() + 1"), "got:\n{out}"); + assert!(out.contains("return o._hook()"), "got:\n{out}"); + } + + #[test] + fn every_binding_of_a_module_level_private_name_is_renamed() { + // an import, an `except` target and a match capture rebind the module's + // symbol, so each has to bind the renamed one + let out = out(indoc! {" + private last: int = 0 + private err: Exception | None = None + private def loads(s: str) -> int: + return 0 + from json import loads + + try: + raise ValueError('boom') + except ValueError as err: + pass + + match 7: + case last: + pass + "}); + assert!(out.contains("def _loads(s: str) -> int:"), "got:\n{out}"); + assert!(out.contains("loads as _loads"), "got:\n{out}"); + assert!(out.contains("except ValueError as _err:"), "got:\n{out}"); + assert!(out.contains("case _last:"), "got:\n{out}"); + } + + #[test] + fn an_ordinary_method_is_untouched() { + let out = out(indoc! {" + class A: + def helper(self) -> int: + return 1 + + def use(self) -> int: + return self.helper() + "}); + assert!(out.contains("return self.helper()"), "got:\n{out}"); + } + + #[test] + fn a_same_named_method_on_another_class_is_untouched() { + let out = out(indoc! {" + class A: + private def helper(self) -> int: + return 1 + + class B: + def helper(self) -> int: + return 2 + + def use(self) -> int: + return self.helper() + "}); + assert!(out.contains("return self.helper()"), "got:\n{out}"); + } +} diff --git a/crates/by_transforms/src/transforms/wrapped_runtime.rs b/crates/by_transforms/src/transforms/wrapped_runtime.rs deleted file mode 100644 index 88e3dacc6e..0000000000 --- a/crates/by_transforms/src/transforms/wrapped_runtime.rs +++ /dev/null @@ -1,63 +0,0 @@ -//! Shared runtime polyfills basedpython injects into the file that needs them. -//! -//! `Optional` is the runtime machine for wrapped optionals: it is both the -//! present-case value wrapper that `Some(x)` lowers to (`Optional(x)`, holding -//! `.value`) and the subscriptable type the `int??` annotation lowers to -//! (`Optional[int | None]`). Passes that emit either form inject this class via -//! [`PassContext::required_imports`](super::ast_driver::PassContext), which -//! dedupes identical entries so the class is defined at most once. -//! -//! `_by_discard` is the adapter a conversion site wraps a callable in when the -//! site asked for one returning `None`. - -use ty_python_semantic::DISCARD_ADAPTER; - -pub(crate) const OPTIONAL_RUNTIME: &str = "\ -class Optional: - def __init__(self, value): - self.value = value - - def __class_getitem__(cls, item): - return cls - - def __repr__(self): - return f\"Some({self.value!r})\" -"; - -/// The adapter a callable is wrapped in where the site declared one returning -/// `None` and the callable returns something else — basedpython's coercion to -/// `None`, which the checker resolves as a conversion route. -/// -/// A bare closure would throw the result away just as well. It would also stop -/// comparing equal to the callable it wraps, and python deregisters callbacks by -/// value all the time — `observers.remove(cb)`, `atexit.unregister(cb)`, -/// `signal.disconnect(cb)`. Delegating `__eq__` and `__hash__` is what keeps a -/// wrapped callback removable; delegating everything else through `__getattr__` -/// is what keeps `cb.__name__` answering for a framework that reads it -pub(crate) fn discard_return_runtime() -> String { - format!( - "\ -class {DISCARD_ADAPTER}: - __slots__ = (\"__wrapped__\",) - - def __init__(self, fn): - self.__wrapped__ = fn - - def __call__(self, *args, **kwargs): - self.__wrapped__(*args, **kwargs) - - def __getattr__(self, name): - if name == \"__wrapped__\": - raise AttributeError(name) - return getattr(self.__wrapped__, name) - - def __eq__(self, other): - if isinstance(other, {DISCARD_ADAPTER}): - other = other.__wrapped__ - return self.__wrapped__ == other - - def __hash__(self): - return hash(self.__wrapped__) -" - ) -} diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index 35da2c5acc..8eb41561cc 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -11,6 +11,7 @@ use ruff_text_size::TextRange; use ty_python_core::scope::ScopeKind; use ty_python_core::{global_scope, place_table, semantic_index}; use ty_python_semantic::types::call_type_forms::CallTypeForm; +use ty_python_semantic::types::exceptions::RaisesRuntimeTarget; use ty_python_semantic::types::{ DisplaySettings, DynamicType, KnownClass, KnownInstanceType, Type, UnpackedKwargs, character, }; @@ -113,14 +114,15 @@ pub(crate) trait TypeInfo { /// `invalid-match-pattern` fn class_pattern_positional_count(&self, cls: &Expr) -> Option; - /// basedpython: the `isinstance` target for `function`'s declared `raises` - /// clause (`(TypeError, ValueError)`, `()` for `raises Never`), or `None` - /// when the clause has no faithful runtime test — a gradual `raises ...`, or - /// a set with no runtime spelling + /// basedpython: the runtime test for `function`'s declared `raises` clause — + /// an `isinstance` target (`(TypeError, ValueError)`, `()` for `raises Never`) + /// and, for a clause naming a reified type parameter, how to build the exact + /// one at the call — or `None` when the clause has no faithful runtime test: + /// a gradual `raises ...`, or a set with no runtime spelling fn declared_raises_runtime_target( &self, function: &ruff_python_ast::StmtFunctionDef, - ) -> Option; + ) -> Option; /// whether `name` resolves to a basedpython return-value marker — /// `ignorable_return_value` or `must_use_return_value`. both are pure @@ -253,7 +255,42 @@ pub(crate) trait TypeInfo { /// basedpython: the mangled name a `private` method is reached by in the /// emitted python, for an attribute access that resolves to one - fn private_method_name(&self, attribute: &ruff_python_ast::ExprAttribute) -> Option; + fn restricted_member_name(&self, attribute: &ruff_python_ast::ExprAttribute) -> Option; + + /// the module-level names this file declares `private` — the set + /// `private-import` reads, and the one whose references the lowering renames + fn private_module_symbols(&self) -> Vec; + + /// whether the name `reference` reads or writes resolves to the module + /// scope: no scope between it and the module binds it, or one declares it + /// `global`. `None` when ty did not index the reference and cannot place it + fn resolves_to_module_scope(&self, reference: &ExprName) -> Option; + + /// whether `name`, read in an annotation, is a forward reference: a name the + /// program binds, but not by the point python evaluates the annotation where + /// it is written, as it does before 3.14 without `from __future__ import + /// annotations`. `None` when ty did not index the name + fn is_forward_reference(&self, name: &ExprName) -> Option; + + /// how a bare name in a class body is emitted, when it names a member the + /// class declares with a visibility keyword — `y = x + 1` after `private x = + /// 1`. `None` for any other name + fn class_body_member_name(&self, name: &ExprName) -> Option; + + /// how `class`'s own member `name`, declared with a visibility keyword, is + /// spelled in the class body and anywhere else. `None` when it is not renamed + fn class_member_spellings( + &self, + class: &ruff_python_ast::StmtClassDef, + name: &str, + ) -> Option<(String, String)>; + + /// the name a class pattern's keyword (`case A(x=...)`) is emitted under + fn class_pattern_keyword_name( + &self, + pattern: &ruff_python_ast::PatternMatchClass, + keyword: &str, + ) -> Option; /// how `name` resolves through the enclosing trailing lambda block's /// receiver: `self` is the receiver itself, any other name is a member of @@ -648,7 +685,7 @@ impl TypeInfo for SemanticModel<'_> { fn declared_raises_runtime_target( &self, function: &ruff_python_ast::StmtFunctionDef, - ) -> Option { + ) -> Option { ty_python_semantic::types::exceptions::declared_raises_runtime_target( self.db(), &self.program_environment(), @@ -677,8 +714,8 @@ impl TypeInfo for SemanticModel<'_> { self.reified_constructor_type_arguments(call) } - fn private_method_name(&self, attribute: &ruff_python_ast::ExprAttribute) -> Option { - SemanticModel::private_method_name(self, attribute) + fn restricted_member_name(&self, attribute: &ruff_python_ast::ExprAttribute) -> Option { + SemanticModel::restricted_member_name(self, attribute) } fn erased_union(&self, annotation: &Expr) -> Option { @@ -909,6 +946,41 @@ impl TypeInfo for SemanticModel<'_> { None } + fn private_module_symbols(&self) -> Vec { + SemanticModel::private_module_symbols(self) + .into_iter() + .map(|name| name.to_string()) + .collect() + } + + fn resolves_to_module_scope(&self, reference: &ExprName) -> Option { + SemanticModel::resolves_to_module_scope(self, reference) + } + + fn is_forward_reference(&self, name: &ExprName) -> Option { + SemanticModel::is_forward_reference(self, name) + } + + fn class_body_member_name(&self, name: &ExprName) -> Option { + SemanticModel::class_body_member_name(self, name) + } + + fn class_member_spellings( + &self, + class: &ruff_python_ast::StmtClassDef, + name: &str, + ) -> Option<(String, String)> { + SemanticModel::class_member_spellings(self, class, name) + } + + fn class_pattern_keyword_name( + &self, + pattern: &ruff_python_ast::PatternMatchClass, + keyword: &str, + ) -> Option { + SemanticModel::class_pattern_keyword_name(self, pattern, keyword) + } + fn shares_a_cell_scope(&self, reference: &ExprName, anchor: &Expr) -> bool { let db = self.db(); let index = semantic_index(db, self.program_file()); @@ -1200,6 +1272,10 @@ impl TypeInfo for SemanticModel<'_> { .map(|argument| { let variable = if argument.is_block_receiver { RECEIVER_PARAMETER.to_string() + } else if argument.is_module_private { + // the binding is a module-level `private` variable, which the + // lowering emits under its underscored name + crate::transforms::modifiers::module_private_name(&argument.variable) } else { argument.variable.to_string() }; diff --git a/crates/by_transforms/tests/anon_named_tuple_runtime.rs b/crates/by_transforms/tests/anon_named_tuple_runtime.rs index 76f204a383..126789e7de 100644 --- a/crates/by_transforms/tests/anon_named_tuple_runtime.rs +++ b/crates/by_transforms/tests/anon_named_tuple_runtime.rs @@ -115,3 +115,19 @@ fn a_plain_tuple_under_an_alias_is_coerced() { fn both_construction_spellings_build_one_class() { run(SPELLING_PROGRAM); } + +/// a wrapped-optional field lowers to the runtime's `Optional`, which a +/// `NamedTuple` evaluates as the class is created — so the hoisted class has to +/// come after the runtime's definitions, not ahead of them +const WRAPPED_OPTIONAL_PROGRAM: &str = r#" +def g() -> (a: int??, b: int): + return (a=None, b=2) + +assert g().b == 2, "b" +print("ok") +"#; + +#[test] +fn a_hoisted_class_sees_the_runtime() { + run(WRAPPED_OPTIONAL_PROGRAM); +} diff --git a/crates/by_transforms/tests/forward_reference_runtime.rs b/crates/by_transforms/tests/forward_reference_runtime.rs new file mode 100644 index 0000000000..64eacaec0f --- /dev/null +++ b/crates/by_transforms/tests/forward_reference_runtime.rs @@ -0,0 +1,97 @@ +//! Runtime test for forward references in annotations python evaluates as the definition runs. +//! +//! basedpython resolves every annotation as deferred, so a signature may name a class defined +//! further down, the class it sits in, or a name imported only for the checker. Before 3.14 +//! python evaluates those annotations as the `def` or the class body runs, and an unquoted +//! forward reference raises `NameError` at import — which asserting on the lowered text cannot +//! see. +//! +//! Needs an interpreter older than 3.14. From 3.14 annotations are deferred natively, so there +//! is nothing to observe, and the test skips rather than pass without checking anything. + +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile}; + +mod common; +use common::python; + +/// every place an annotation runs with the definition, naming something bound after it +const PROGRAM: &str = r#" +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections import OrderedDict + + +def make() -> Later: + return Later() + + +class Node: + parent: Node? + + def child(self, other: Node) -> list[Node]: + return [other] + + def later(self) -> Later?: + return None + + +value: Later + + +def ordered(counts: OrderedDict[str, int]) -> None: ... + + +class Later: ... + + +assert type(make()) == Later, "a function above the class it returns runs" +assert len(Node().child(Node())) == 1, "so does a method naming its own class" +print("ok") +"#; + +/// whether `python` still evaluates annotations as the definition runs +fn evaluates_annotations_eagerly(python: &str) -> bool { + Command::new(python) + .args([ + "-c", + "import sys; raise SystemExit(sys.version_info >= (3, 14))", + ]) + .status() + .is_ok_and(|status| status.success()) +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_forward_reference_does_not_raise_at_import() { + let Some(python) = python() else { + return; + }; + if !evaluates_annotations_eagerly(&python) { + eprintln!("skipping: {python} defers annotations natively, so nothing is evaluated early"); + return; + } + let config = Config { + min_version: PythonVersion::PY310, + ..Config::default() + }; + let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); + let output = Command::new(&python) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "transpiled program failed on {python}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} diff --git a/crates/by_transforms/tests/visibility_runtime.rs b/crates/by_transforms/tests/visibility_runtime.rs index 2899e75047..8fbb651f8d 100644 --- a/crates/by_transforms/tests/visibility_runtime.rs +++ b/crates/by_transforms/tests/visibility_runtime.rs @@ -89,3 +89,294 @@ fn every_exported_name_exists_at_runtime() { ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); } + +/// Every class-member shape a visibility keyword renames, read back from the +/// places allowed to reach it. +const MEMBERS: &str = r#" +class Counter: + protected step: int = 2 + private total: int = 0 + + init(private let label: str, protected let limit: int) + + private def bump(self) -> int: + self.total = self.total + self.step + return self.total + + def run(self) -> str: + while self.total < self.limit: + self.bump() + return f"{self.label}:{self.total}" + + +class Loud(Counter): + def describe(self) -> str: + return f"{self.step}/{self.limit}" +"#; + +/// The emitted names are what python actually stores, so the check is made +/// against `vars()` as well as against the values: a rename that lands on the +/// wrong spelling still reads back correctly from inside the class, and only +/// the stored name gives it away. +const MEMBER_IMPORTER: &str = r#" +import importlib +m = importlib.import_module("emitted") +c = m.Counter("a", 5) +assert c.run() == "a:6", c.run() +assert m.Loud("b", 3).describe() == "2/3" + +stored = vars(c) +assert "_Counter__label" in stored, stored +assert "_limit" in stored, stored +assert "_Counter__total" in stored, stored +assert not hasattr(c, "label") and not hasattr(c, "total"), "a private member is mangled" +assert c._step == 2 and c._Counter__total == 6 +assert not hasattr(c, "bump") and callable(c._Counter__bump) +print("ok") +"#; + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_renamed_member_is_the_same_attribute_from_every_place_that_may_reach_it() { + let Some(python) = python() else { + eprintln!("skipping member visibility runtime test: no `python3` interpreter found"); + return; + }; + + let config = Config { + min_version: PythonVersion::PY313, + ..Config::default() + }; + let transpiled = transpile(MEMBERS, &config).expect("transpile should succeed"); + + let dir = tempfile::tempdir().expect("failed to create a temp dir"); + std::fs::write(dir.path().join("emitted.py"), &transpiled).expect("failed to write the module"); + + let output = Command::new(&python) + .arg("-c") + .arg(MEMBER_IMPORTER) + .current_dir(dir.path()) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "importing the transpiled module failed on {python}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} + +/// A module-level `private` variable and a `private` class variable, reached +/// from every place that may reach each — and a parameter and a class attribute +/// that share the module variable's name, which are other bindings entirely. +const VARIABLES: &str = r#" +private count: int = 0 + +def bump() -> int: + global count + count += 1 + return count + +def shadow(count: int) -> int: + return count * 10 + +class Holder: + count = 5 + + def module_count(self) -> int: + return count + +class Registry: + private class var made: int = 0 + + init(): + Registry.made += 1 + + @classmethod + def total(cls) -> int: + return cls.made +"#; + +const VARIABLE_IMPORTER: &str = r#" +import importlib +m = importlib.import_module("emitted") +assert m.bump() == 1 and m.bump() == 2 +assert m.shadow(3) == 30, "a parameter that shares the name is its own binding" +assert m.Holder.count == 5, "so is a class attribute" +assert m.Holder().module_count() == 2, "a method reads past its class body to the module" +assert hasattr(m, "_count") and not hasattr(m, "count"), "the module variable is renamed" +m.Registry() +m.Registry() +assert m.Registry.total() == 2 +assert "_Registry__made" in vars(m.Registry) and not hasattr(m.Registry, "made") +print("ok") +"#; + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_renamed_variable_is_the_same_binding_wherever_it_is_the_symbol() { + let Some(python) = python() else { + eprintln!("skipping variable visibility runtime test: no `python3` interpreter found"); + return; + }; + + let config = Config { + min_version: PythonVersion::PY313, + ..Config::default() + }; + let transpiled = transpile(VARIABLES, &config).expect("transpile should succeed"); + + let dir = tempfile::tempdir().expect("failed to create a temp dir"); + std::fs::write(dir.path().join("emitted.py"), &transpiled).expect("failed to write the module"); + + let output = Command::new(&python) + .arg("-c") + .arg(VARIABLE_IMPORTER) + .current_dir(dir.path()) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "importing the transpiled module failed on {python}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} + +/// Every place a class body can declare a member, every list of member names, +/// and every statement that rebinds a module-level `private` name — each read +/// back the way python reads it. +const SHAPES: &str = r#" +private last: int = 0 +private err: Exception | None = None +private def loads(s: str) -> int: + return 0 +from json import loads + +try: + raise ValueError("boom") +except ValueError as err: + pass + +match 7: + case last: + pass + +class Shape: + if True: + private sides: int = 3 + private scale: int = 2 + doubled = scale * 2 + + private class Inner: + pass + + @staticmethod + private def make() -> int: + return 4 + + def area(self) -> int: + return self.sides * self.scale + self.make() + + def inner(self) -> object: + return Shape.Inner() + +class Slotted: + __slots__ = ("x",) + private x: int + + def __init__(self, x: int): + self.x = x + + def get(self) -> int: + return self.x + +class Point: + __match_args__ = ("x",) + private x: int + + def __init__(self, x: int): + self.x = x + + def unpack(self) -> int: + match self: + case Point(x=v): + return v + return -1 + +class Base: + protected def hook(self) -> int: + return 1 + +class Child(Base): + protected override def hook(self) -> int: + return super().hook() + 1 + + def pick(self, o: Child | Other) -> int: + return o.hook() + +class Other(Base): ... +"#; + +const SHAPE_IMPORTER: &str = r#" +import importlib +m = importlib.import_module("emitted") +s = m.Shape() +assert s.area() == 10 and m.Shape.doubled == 4 +assert type(s.inner()).__name__ == "__Inner" +assert m._last == 7, "a match capture binds the renamed variable" +assert m._loads("[1]") == [1], "so does an import" +assert not hasattr(m, "loads") and not hasattr(m, "last") and not hasattr(m, "err") +slotted = m.Slotted(5) +assert slotted.get() == 5 and not hasattr(slotted, "__dict__"), "the slot is the member" +assert m.Point(9).unpack() == 9, "a class pattern's keyword reads `__match_args__`'s name" +c = m.Child() +assert c.pick(c) == 2 and c.pick(m.Other()) == 1 +print("ok") +"#; + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_renamed_name_is_read_back_wherever_python_looks_it_up() { + let Some(python) = python() else { + eprintln!("skipping visibility shapes runtime test: no `python3` interpreter found"); + return; + }; + + let config = Config { + min_version: PythonVersion::PY313, + ..Config::default() + }; + let transpiled = transpile(SHAPES, &config).expect("transpile should succeed"); + + let dir = tempfile::tempdir().expect("failed to create a temp dir"); + std::fs::write(dir.path().join("emitted.py"), &transpiled).expect("failed to write the module"); + + let output = Command::new(&python) + .arg("-c") + .arg(SHAPE_IMPORTER) + .current_dir(dir.path()) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "importing the transpiled module failed on {python}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} diff --git a/crates/ruff_linter/src/rules/basedpython/rules/manual_isinstance.rs b/crates/ruff_linter/src/rules/basedpython/rules/manual_isinstance.rs index 1384ab5f6c..ae6f97b362 100644 --- a/crates/ruff_linter/src/rules/basedpython/rules/manual_isinstance.rs +++ b/crates/ruff_linter/src/rules/basedpython/rules/manual_isinstance.rs @@ -34,8 +34,12 @@ use crate::{AlwaysFixableViolation, Edit, Fix}; /// ``` /// /// ## Fix safety -/// This rule's fix is marked as unsafe when the call contains comments, which -/// the rewrite would drop. +/// This rule's fix is always marked as unsafe. `isinstance` runs its check every +/// time, while a type test the value's static type already decides is emitted as +/// its answer: on a parameter annotated `int`, `isinstance(x, int)` still rejects +/// a caller that passes a `str`, but `x is int` is `True`. The two agree only +/// where the annotations hold at runtime, which is exactly what a guard like this +/// is written to check. The rewrite also drops any comments inside the call. /// /// A call whose second argument is a tuple of classes is reported without a fix: /// the keyword accepts one, but a tuple written where a type is expected reads @@ -122,16 +126,10 @@ pub(crate) fn manual_isinstance(checker: &Checker, call: &ast::ExprCall) { } fn report(checker: &Checker, call: TextRange, range: TextRange, replacement: String) { - let applicability = if checker.comment_ranges().intersects(range) { - Applicability::Unsafe - } else { - Applicability::Safe - }; - checker .report_diagnostic(ManualIsinstance, call) .set_fix(Fix::applicable_edit( Edit::range_replacement(replacement, range), - applicability, + Applicability::Unsafe, )); } diff --git a/crates/ruff_linter/src/rules/basedpython/snapshots/ruff_linter__rules__basedpython__tests__BY003_BY003.by.snap b/crates/ruff_linter/src/rules/basedpython/snapshots/ruff_linter__rules__basedpython__tests__BY003_BY003.by.snap index 48849a6801..a931e33ae2 100644 --- a/crates/ruff_linter/src/rules/basedpython/snapshots/ruff_linter__rules__basedpython__tests__BY003_BY003.by.snap +++ b/crates/ruff_linter/src/rules/basedpython/snapshots/ruff_linter__rules__basedpython__tests__BY003_BY003.by.snap @@ -18,6 +18,7 @@ help: Replace with `is` 3 + if x is int: 4 | ... | +note: This is an unsafe fix and may change runtime behavior BY003 [*] `isinstance` call can be written as `is` --> BY003.by:5:12 @@ -35,6 +36,7 @@ help: Replace with `is` 5 + if x is not str: 6 | ... | +note: This is an unsafe fix and may change runtime behavior BY003 [*] `isinstance` call can be written as `is` --> BY003.by:9:9 @@ -51,6 +53,7 @@ help: Replace with `is` 9 + _ = x is t 10 | _ = isinstance(x, x.__class__) | +note: This is an unsafe fix and may change runtime behavior BY003 [*] `isinstance` call can be written as `is` --> BY003.by:10:9 @@ -69,6 +72,7 @@ help: Replace with `is` 10 + _ = x is x.__class__ 11 | | +note: This is an unsafe fix and may change runtime behavior BY003 [*] `isinstance` call can be written as `is` --> BY003.by:13:9 @@ -86,6 +90,7 @@ help: Replace with `is` 13 + _ = x is int and isinstance(x, str) 14 | | +note: This is an unsafe fix and may change runtime behavior BY003 [*] `isinstance` call can be written as `is` --> BY003.by:13:32 @@ -103,6 +108,7 @@ help: Replace with `is` 13 + _ = isinstance(x, int) and x is str 14 | | +note: This is an unsafe fix and may change runtime behavior BY003 [*] `isinstance` call can be written as `is` --> BY003.by:16:11 @@ -119,6 +125,7 @@ help: Replace with `is` 16 + print(x is int) 17 | _ = [isinstance(x, int)] | +note: This is an unsafe fix and may change runtime behavior BY003 [*] `isinstance` call can be written as `is` --> BY003.by:17:10 @@ -137,6 +144,7 @@ help: Replace with `is` 17 + _ = [x is int] 18 | | +note: This is an unsafe fix and may change runtime behavior BY003 [*] `isinstance` call can be written as `is` --> BY003.by:21:9 @@ -155,6 +163,7 @@ help: Replace with `is` 21 + _ = (x is int) == True 22 | _ = isinstance(x, int).__class__ | +note: This is an unsafe fix and may change runtime behavior BY003 [*] `isinstance` call can be written as `is` --> BY003.by:22:9 @@ -172,6 +181,7 @@ help: Replace with `is` 22 + _ = (x is int).__class__ 23 | _ = -isinstance(x, int) | +note: This is an unsafe fix and may change runtime behavior BY003 [*] `isinstance` call can be written as `is` --> BY003.by:23:10 @@ -187,3 +197,4 @@ help: Replace with `is` 23 + _ = -(x is int) 24 | | +note: This is an unsafe fix and may change runtime behavior diff --git a/crates/ruff_linter/src/rules/ruff/helpers.rs b/crates/ruff_linter/src/rules/ruff/helpers.rs index 6481ff3e82..4943296002 100644 --- a/crates/ruff_linter/src/rules/ruff/helpers.rs +++ b/crates/ruff_linter/src/rules/ruff/helpers.rs @@ -63,7 +63,9 @@ pub(super) fn is_class_var_annotation(annotation: &Expr, semantic: &SemanticMode // what the source wrote `ClassVar` with if matches!( map_subscript(annotation), - Expr::Name(name) if matches!(name.id.as_str(), "__classvar__" | "__classvar_annot__") + Expr::Name(name) + if ruff_python_ast::helpers::is_classvar_marker_id(name.id.as_str()) + || ruff_python_ast::helpers::is_classvar_annot_marker_id(name.id.as_str()) ) { return true; } @@ -83,7 +85,7 @@ pub(super) fn is_final_annotation(annotation: &Expr, semantic: &SemanticModel) - // and carry a synthetic marker in annotation position until they do if matches!( map_subscript(annotation), - Expr::Name(name) if name.id.as_str() == "__final__" + Expr::Name(name) if ruff_python_ast::helpers::is_final_marker_id(name.id.as_str()) ) { return true; } diff --git a/crates/ruff_python_ast/src/helpers.rs b/crates/ruff_python_ast/src/helpers.rs index 0d00916a8b..72eba98b4f 100644 --- a/crates/ruff_python_ast/src/helpers.rs +++ b/crates/ruff_python_ast/src/helpers.rs @@ -1368,18 +1368,210 @@ pub fn if_let_keyword_range( .map(|token| token.range) } -/// basedpython: the synthetic markers a declaration modifier (`let`, `var`, -/// `final`, `private`, `abstract`, a visibility keyword) wraps its annotation in. -/// The declared type rides in the subscript slice. -const DECLARATION_ANNOTATION_MARKERS: [&str; 7] = [ - "__let__", - "__final__", - "__modifier_annot__", - "__private_annot__", - "__visibility_annot__", - "__abstract_annot__", - "__classvar_annot__", -]; +/// basedpython: what a declaration's synthetic annotation marker declares, +/// apart from the member's visibility +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub enum DeclarationMarkerKind { + /// `let x [: T]` — read-only, and `Final` outside a class body + Let, + /// `final x: T` and `class let x: T` — `Final` in every scope + Final, + /// `class var x: T` — a class variable whose type is declared + ClassVarAnnot, + /// `class x = v` — a class variable whose type is read off its value + ClassVar, + /// `[modifiers] x: T` — a declaration whose modifiers carry no type meaning + Annot, + /// `[modifiers] x = v` — a declaration that states no type + Assign, +} + +/// basedpython: the synthetic marker a declaration's modifier chain parses to, +/// in annotation position — `let a: T` is `a: __let__[T]`, `private a = v` is +/// `a: __private_assign__`. the declared type, when there is one, rides in the +/// subscript slice +/// +/// [`DeclarationMarker::id`] is the one place a marker's id is spelled: the +/// parser writes an id from a marker, and everything downstream reads one back +/// through [`DeclarationMarker::from_id`], so no list of ids can drift from what +/// the parser actually writes +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub struct DeclarationMarker { + pub kind: DeclarationMarkerKind, + pub visibility: MemberVisibility, +} + +impl DeclarationMarker { + const ALL: [DeclarationMarker; 18] = { + use DeclarationMarkerKind::{Annot, Assign, ClassVar, ClassVarAnnot, Final, Let}; + use MemberVisibility::{Private, Protected, Public}; + let kinds = [Let, Final, ClassVarAnnot, ClassVar, Annot, Assign]; + let visibilities = [Public, Private, Protected]; + let mut all = [DeclarationMarker::new(Let, Public); 18]; + let mut index = 0; + while index < 18 { + all[index] = DeclarationMarker::new(kinds[index / 3], visibilities[index % 3]); + index += 1; + } + all + }; + + pub const fn new(kind: DeclarationMarkerKind, visibility: MemberVisibility) -> Self { + Self { kind, visibility } + } + + /// the id the parser writes for this marker + pub const fn id(self) -> &'static str { + use DeclarationMarkerKind::{Annot, Assign, ClassVar, ClassVarAnnot, Final, Let}; + use MemberVisibility::{Private, Protected, Public}; + match (self.kind, self.visibility) { + (Let, Public) => "__let__", + (Let, Private) => "__private_let__", + (Let, Protected) => "__protected_let__", + (Final, Public) => "__final__", + (Final, Private) => "__final_private__", + (Final, Protected) => "__final_protected__", + (ClassVarAnnot, Public) => "__classvar_annot__", + (ClassVarAnnot, Private) => "__private_classvar_annot__", + (ClassVarAnnot, Protected) => "__protected_classvar_annot__", + (ClassVar, Public) => "__classvar__", + (ClassVar, Private) => "__private_classvar__", + (ClassVar, Protected) => "__protected_classvar__", + (Annot, Public) => "__modifier_annot__", + (Annot, Private) => "__private_annot__", + (Annot, Protected) => "__protected_annot__", + (Assign, Public) => "__modifier_assign__", + (Assign, Private) => "__private_assign__", + (Assign, Protected) => "__protected_assign__", + } + } + + /// the marker `id` names, `None` for an id that is not a declaration marker + pub fn from_id(id: &str) -> Option { + Self::ALL.into_iter().find(|marker| marker.id() == id) + } + + /// the marker `annotation` is, with the declared type when it carries one. + /// only a synthetic name counts — a real annotation that happens to be + /// spelled like a marker is an ordinary name + pub fn of(annotation: &Expr) -> Option<(Self, Option<&Expr>)> { + let (name, slice) = match annotation { + Expr::Subscript(subscript) => { + (subscript.value.as_name_expr()?, Some(&*subscript.slice)) + } + Expr::Name(name) => (name, None), + _ => return None, + }; + if name.ctx != crate::ExprContext::Invalid { + return None; + } + Some((Self::from_id(name.id.as_str())?, slice)) + } +} + +/// basedpython: how far outside the class that declares it a member can be +/// reached. +/// +/// The emitted python spells the visibility in the member's name, because that +/// is the only enforcement the runtime offers: a `private` member becomes +/// `__name`, which python name-mangles to `_Class__name` so a subclass's body +/// names something else entirely, and a `protected` member becomes `_name`, +/// which is python's long-standing convention for "not part of the interface". +/// The type checker enforces both directly, so neither spelling has to be +/// relied on. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Default)] +pub enum MemberVisibility { + /// Part of the class's interface — reachable from anywhere. + #[default] + Public, + /// Reachable from the declaring class's body and from a subclass's body. + Protected, + /// Reachable only from the declaring class's body. + Private, +} + +#[cfg(feature = "get-size")] +impl get_size2::GetSize for MemberVisibility {} + +impl MemberVisibility { + /// The keyword that declares this visibility, for use in a message. + pub fn keyword(self) -> &'static str { + match self { + MemberVisibility::Public => "public", + MemberVisibility::Protected => "protected", + MemberVisibility::Private => "private", + } + } + + /// The prefix the emitted python spells this visibility with. + pub fn name_prefix(self) -> &'static str { + match self { + MemberVisibility::Public => "", + MemberVisibility::Protected => "_", + MemberVisibility::Private => "__", + } + } + + /// Whether `self` reaches strictly fewer places than `other`. + pub fn is_narrower_than(self, other: Self) -> bool { + (self as u8) > (other as u8) + } +} + +/// basedpython: the storage a property accessor block's `field` names. a +/// `private` property is itself emitted as `__name`, so its storage takes a name +/// of its own; any other property's storage is `__name`, which python's name +/// mangling keeps out of reach +pub fn property_backing_name(public: &str, visibility: MemberVisibility) -> String { + match visibility { + MemberVisibility::Private => format!("__{public}_field"), + MemberVisibility::Public | MemberVisibility::Protected => format!("__{public}"), + } +} + +/// basedpython: whether `marker_id` is the marker a `let` declaration carries, +/// in any of the visibilities it can be written with. +pub fn is_let_marker_id(marker_id: &str) -> bool { + DeclarationMarker::from_id(marker_id) + .is_some_and(|marker| marker.kind == DeclarationMarkerKind::Let) +} + +/// basedpython: whether `marker_id` is the marker a `final` declaration carries, +/// in any of the visibilities it can be written with. +pub fn is_final_marker_id(marker_id: &str) -> bool { + DeclarationMarker::from_id(marker_id) + .is_some_and(|marker| marker.kind == DeclarationMarkerKind::Final) +} + +/// basedpython: the visibility a declaration marker records, `Public` for a +/// marker that records none. +pub fn marker_visibility(marker_id: &str) -> MemberVisibility { + DeclarationMarker::from_id(marker_id) + .map_or(MemberVisibility::Public, |marker| marker.visibility) +} + +/// basedpython: the visibility a declaration's synthetic annotation marker +/// records — `private a: T` parses as `a: __private_annot__[T]`, `private a = v` +/// as `a: __private_assign__`. `Public` for an annotation that is no marker. +pub fn declaration_marker_visibility(annotation: &Expr) -> MemberVisibility { + DeclarationMarker::of(annotation) + .map_or(MemberVisibility::Public, |(marker, _)| marker.visibility) +} + +/// basedpython: whether `marker_id` is the marker a class variable declared by +/// value — `class count = 0` — carries, in any visibility it can be written with. +pub fn is_classvar_marker_id(marker_id: &str) -> bool { + DeclarationMarker::from_id(marker_id) + .is_some_and(|marker| marker.kind == DeclarationMarkerKind::ClassVar) +} + +/// basedpython: whether `marker_id` is the marker a class variable declared by +/// type — `class var count: int` — carries, in any visibility it can be written +/// with. +pub fn is_classvar_annot_marker_id(marker_id: &str) -> bool { + DeclarationMarker::from_id(marker_id) + .is_some_and(|marker| marker.kind == DeclarationMarkerKind::ClassVarAnnot) +} /// basedpython: the *declared type* inside a declaration marker, if `annotation` /// is one — `let a: T` parses as `a: __let__[T]`, and `T` is what a reader of the @@ -1393,12 +1585,9 @@ pub fn declaration_annotation_type(annotation: &Expr) -> Option<&Expr> { let Expr::Subscript(subscript) = annotation else { return None; }; - let Expr::Name(marker) = subscript.value.as_ref() else { - return None; - }; - DECLARATION_ANNOTATION_MARKERS - .contains(&marker.id.as_str()) - .then(|| subscript.slice.as_ref()) + let marker = subscript.value.as_name_expr()?; + DeclarationMarker::from_id(marker.id.as_str())?; + Some(&subscript.slice) } /// basedpython: an `implements A, B` declaration, and the `for` clause that says @@ -1507,10 +1696,16 @@ pub fn binding_keyword( _ => return None, }; - if !matches!( - marker.id.as_str(), - "__let__" | "__modifier_annot__" | "__modifier_assign__" - ) { + // `let` and `var` bind; the other declaration forms have no binding keyword + // to find, whatever their visibility + if !DeclarationMarker::from_id(marker.id.as_str()).is_some_and(|marker| { + matches!( + marker.kind, + DeclarationMarkerKind::Let + | DeclarationMarkerKind::Annot + | DeclarationMarkerKind::Assign + ) + }) { return None; } if source.len() < usize::from(marker.range.end()) { @@ -1795,8 +1990,16 @@ pub fn is_top_parameters_form(expr: &Expr) -> bool { /// annotation is `Name(id="__modifier_assign__", ctx=Invalid)` spanning the /// keyword prefix. The marker carries no type, so the statement declares nothing /// and binds exactly like the `a = 1` it lowers to. +/// +/// A visibility keyword in the chain gets a marker of its own, which is *not* one +/// of these: it says who may reach the member, which has to survive as a +/// qualifier on the declaration, and that only the declaration path carries. pub fn is_untyped_declaration_marker(expr: &Expr) -> bool { - matches!(expr, Expr::Name(name) if name.id.as_str() == "__modifier_assign__") + matches!( + expr, + Expr::Name(name) if DeclarationMarker::from_id(name.id.as_str()) + == Some(DeclarationMarker::new(DeclarationMarkerKind::Assign, MemberVisibility::Public)) + ) } /// basedpython: the inference-context marker an *unannotated* `field = ` in a diff --git a/crates/ruff_python_ast/src/nodes.rs b/crates/ruff_python_ast/src/nodes.rs index 9c1e506098..3027095496 100644 --- a/crates/ruff_python_ast/src/nodes.rs +++ b/crates/ruff_python_ast/src/nodes.rs @@ -29,7 +29,51 @@ use crate::{ str::{Quote, TripleQuotes}, }; +/// basedpython: the property construct a getter was synthesized from — see +/// [`StmtFunctionDef::property_construct`] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PropertyConstruct { + /// the `var` / `let` declaration and its accessor suite, the whole span the + /// construct was written as + pub range: TextRange, + /// `static let`: a class-level property rather than an instance one + pub is_static: bool, +} + impl StmtFunctionDef { + /// basedpython: the property construct this function was synthesized from, + /// when it is that construct's getter + /// + /// the parser lowers a `var` / `let` declaration carrying a `get` / `set` / + /// `field` suite into several class-body members — a getter, an optional + /// backing declaration, an optional setter — and marks the getter with a + /// synthetic decorator spanning the whole construct. the other members are + /// ranged inside that span, and the source spells one member, so whatever + /// reads the construct back finds it through the getter and this range + pub fn property_construct(&self) -> Option { + self.decorator_list + .iter() + .find_map(|decorator| match &decorator.expression { + Expr::Name(marker) if marker.is_invalid() => { + let is_static = match marker.id.as_str() { + "__property__" => false, + "__static_property__" => true, + _ => return None, + }; + Some(PropertyConstruct { + range: decorator.range(), + is_static, + }) + } + _ => None, + }) + } + + /// basedpython: the range of [`StmtFunctionDef::property_construct`] + pub fn property_construct_range(&self) -> Option { + self.property_construct().map(|construct| construct.range) + } + /// basedpython: the expression whose signature decides how a trailing /// lambda block is passed. The synthetic decorator carries the called /// expression; for a plain parenthesized call (`f(2):`) the deciding diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/modifiers.by b/crates/ruff_python_formatter/resources/test/fixtures/ruff/modifiers.by index f2fdd52637..d6dbf4e972 100644 --- a/crates/ruff_python_formatter/resources/test/fixtures/ruff/modifiers.by +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/modifiers.by @@ -79,6 +79,25 @@ class I[T]: return self.u +# `protected` — the same shapes, plus the `final` chain where both halves of the +# marker have to survive +class J: + protected step: int = 2 + protected var seen: int + final protected limit: int = 3 + final private cap: int = 4 + private let fixed: int = 5 + protected let bound = 6 + private tally = 0 + private class var registry: int = 0 + protected class let CAP: int = 9 + private class hits = 0 + protected def helper(self) -> int: + return self.step + + init(protected let base: int, private let tag: str) + + # `private type` aliases — the modifier has no decorator slot to ride on, so it # lives on the node itself and must survive the round trip private type Alias = int | str diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/properties.by b/crates/ruff_python_formatter/resources/test/fixtures/ruff/properties.by index 9bd4a5540f..750770d29e 100644 --- a/crates/ruff_python_formatter/resources/test/fixtures/ruff/properties.by +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/properties.by @@ -17,3 +17,12 @@ class Bag: class Loader: late var handle: str + +class Anon: + let a + get() = 1 + + var b = 0 + get() = field + set(value): + field = value diff --git a/crates/ruff_python_formatter/src/statement/stmt_ann_assign.rs b/crates/ruff_python_formatter/src/statement/stmt_ann_assign.rs index 894a2c3757..ede061a87f 100644 --- a/crates/ruff_python_formatter/src/statement/stmt_ann_assign.rs +++ b/crates/ruff_python_formatter/src/statement/stmt_ann_assign.rs @@ -1,4 +1,8 @@ use ruff_formatter::write; +use ruff_python_ast::helpers::{ + DeclarationMarker, DeclarationMarkerKind, is_classvar_marker_id, is_final_marker_id, + is_let_marker_id, +}; use ruff_python_ast::{Expr, StmtAnnAssign}; use ruff_text_size::Ranged; @@ -45,8 +49,8 @@ impl Format> for AssignedValue<'_> { #[allow(clippy::option_option)] fn synthetic_let(ann: &Expr) -> Option> { match ann { - Expr::Name(n) if n.id.as_str() == "__let__" => Some(None), - Expr::Subscript(s) if matches!(s.value.as_ref(), Expr::Name(n) if n.id.as_str() == "__let__") => { + Expr::Name(n) if is_let_marker_id(n.id.as_str()) => Some(None), + Expr::Subscript(s) if matches!(s.value.as_ref(), Expr::Name(n) if is_let_marker_id(n.id.as_str())) => { Some(Some(s.slice.as_ref())) } _ => None, @@ -59,7 +63,7 @@ fn synthetic_final(ann: &Expr) -> Option<&Expr> { let Expr::Subscript(s) = ann else { return None; }; - if !matches!(s.value.as_ref(), Expr::Name(n) if n.id.as_str() == "__final__") { + if !matches!(s.value.as_ref(), Expr::Name(n) if is_final_marker_id(n.id.as_str())) { return None; } Some(s.slice.as_ref()) @@ -99,7 +103,7 @@ fn declaration_prefix<'src>( fn synthetic_marker(ann: &Expr) -> Option<&'static str> { if let Expr::Name(n) = ann { match n.id.as_str() { - "__classvar__" => return Some("class"), + id if is_classvar_marker_id(id) => return Some("class"), "__newtype__" => return Some("newtype"), "__sentinel__" => return Some("sentinel"), _ => {} @@ -123,14 +127,12 @@ fn synthetic_modifier_annot<'ast, 'src>( let Expr::Name(name) = s.value.as_ref() else { return None; }; - if !matches!( - name.id.as_str(), - "__abstract_annot__" - | "__visibility_annot__" - | "__private_annot__" - | "__modifier_annot__" - | "__classvar_annot__" - ) { + if !DeclarationMarker::from_id(name.id.as_str()).is_some_and(|marker| { + matches!( + marker.kind, + DeclarationMarkerKind::Annot | DeclarationMarkerKind::ClassVarAnnot + ) + }) { return None; } let start = u32::from(name.range.start()) as usize; @@ -147,7 +149,9 @@ fn synthetic_modifier_assign<'src>(ann: &Expr, src: &'src str) -> Option<&'src s let Expr::Name(name) = ann else { return None; }; - if name.id.as_str() != "__modifier_assign__" { + if !DeclarationMarker::from_id(name.id.as_str()) + .is_some_and(|marker| marker.kind == DeclarationMarkerKind::Assign) + { return None; } let start = u32::from(name.range.start()) as usize; @@ -228,6 +232,20 @@ impl FormatNodeRule for FormatStmtAnnAssign { } return Ok(()); } + if f.options().is_basedpython() + && let Expr::Name(marker) = annotation.as_ref() + && is_classvar_marker_id(marker.id.as_str()) + && let Some(prefix) = declaration_prefix(item, target, f.context().source()) + { + // `[visibility] class NAME = value` — the keyword prefix is rendered + // verbatim from source, so a visibility keyword ahead of `class` + // survives the round trip + write!(f, [text(prefix), space(), target.format()])?; + if let Some(v) = value { + assigned_value(v, padding).fmt(f)?; + } + return Ok(()); + } if let Some(keyword) = synthetic_marker(annotation) { write!(f, [text(keyword), space(), target.format()])?; if let Some(v) = value { diff --git a/crates/ruff_python_formatter/src/statement/suite.rs b/crates/ruff_python_formatter/src/statement/suite.rs index e600d358a2..fb8998a5cf 100644 --- a/crates/ruff_python_formatter/src/statement/suite.rs +++ b/crates/ruff_python_formatter/src/statement/suite.rs @@ -2,7 +2,7 @@ use ruff_formatter::{ FormatContext, FormatOwnedWithRule, FormatRefWithRule, FormatRuleWithOptions, write, }; use ruff_python_ast::helpers::is_compound_statement; -use ruff_python_ast::{self as ast, Expr, ExprContext, PySourceType, Stmt, Suite}; +use ruff_python_ast::{self as ast, Expr, PySourceType, Stmt, Suite}; use ruff_python_ast::{AnyNodeRef, StmtExpr}; use ruff_python_trivia::{ SimpleTokenKind, SimpleTokenizer, lines_after, lines_after_ignoring_end_of_line_trivia, @@ -960,34 +960,18 @@ impl Format> for SuiteChildStatement<'_> { } } -/// The construct range a basedpython property accessor block was parsed from, if -/// `stmt` is the getter the parser synthesised for one. +/// the construct range a basedpython property accessor block was parsed from, if +/// `stmt` is the getter the parser synthesised for one /// -/// A `var` / `let` declaration carrying a `get` / `set` / `field` suite lowers to -/// several class-body members (a backing field, a getter, a setter), none of which -/// has an AST-faithful surface printer — their `self` parameters and backing -/// attributes are synthetic and zero-width. The getter leads the group and carries -/// a synthetic property marker whose range spans the whole construct, so the -/// suite formatter can emit that source verbatim and swallow the rest of the group, -/// exactly as it does for a `# fmt: skip` region. +/// none of the members the construct lowers to has an AST-faithful surface printer +/// — their `self` parameters and backing attributes are synthetic and zero-width — +/// so the suite formatter emits the construct's source verbatim and swallows the +/// rest of the group, exactly as it does for a `# fmt: skip` region fn property_construct_range(stmt: &Stmt) -> Option { - let Stmt::FunctionDef(function) = stmt else { - return None; - }; - function - .decorator_list - .iter() - .find_map(|decorator| match &decorator.expression { - // `__static_property__` is the `static let` (class-level) variant; both - // are synthesised the same way and neither has a surface printer - Expr::Name(name) - if matches!(name.id.as_str(), "__property__" | "__static_property__") - && name.ctx == ExprContext::Invalid => - { - Some(decorator.range()) - } - _ => None, - }) + match stmt { + Stmt::FunctionDef(function) => function.property_construct_range(), + _ => None, + } } pub(crate) fn skip_range( diff --git a/crates/ruff_python_formatter/tests/snapshots/format@modifiers.by.snap b/crates/ruff_python_formatter/tests/snapshots/format@modifiers.by.snap index 487d2b67e3..35d6c6567f 100644 --- a/crates/ruff_python_formatter/tests/snapshots/format@modifiers.by.snap +++ b/crates/ruff_python_formatter/tests/snapshots/format@modifiers.by.snap @@ -85,6 +85,25 @@ class I[T]: return self.u +# `protected` — the same shapes, plus the `final` chain where both halves of the +# marker have to survive +class J: + protected step: int = 2 + protected var seen: int + final protected limit: int = 3 + final private cap: int = 4 + private let fixed: int = 5 + protected let bound = 6 + private tally = 0 + private class var registry: int = 0 + protected class let CAP: int = 9 + private class hits = 0 + protected def helper(self) -> int: + return self.step + + init(protected let base: int, private let tag: str) + + # `private type` aliases — the modifier has no decorator slot to ride on, so it # lives on the node itself and must survive the round trip private type Alias = int | str @@ -180,6 +199,26 @@ class I[T]: return self.u +# `protected` — the same shapes, plus the `final` chain where both halves of the +# marker have to survive +class J: + protected step: int = 2 + protected var seen: int + final protected limit: int = 3 + final private cap: int = 4 + private let fixed: int = 5 + protected let bound = 6 + private tally = 0 + private class var registry: int = 0 + protected class let CAP: int = 9 + private class hits = 0 + + protected def helper(self) -> int: + return self.step + + init(protected let base: int, private let tag: str) + + # `private type` aliases — the modifier has no decorator slot to ride on, so it # lives on the node itself and must survive the round trip private type Alias = int | str diff --git a/crates/ruff_python_formatter/tests/snapshots/format@properties.by.snap b/crates/ruff_python_formatter/tests/snapshots/format@properties.by.snap index 8ce24da4c0..e9cf00d909 100644 --- a/crates/ruff_python_formatter/tests/snapshots/format@properties.by.snap +++ b/crates/ruff_python_formatter/tests/snapshots/format@properties.by.snap @@ -23,6 +23,15 @@ class Bag: class Loader: late var handle: str + +class Anon: + let a + get() = 1 + + var b = 0 + get() = field + set(value): + field = value ``` ## Output @@ -50,4 +59,14 @@ class Bag: class Loader: late var handle: str + + +class Anon: + let a + get() = 1 + + var b = 0 + get() = field + set(value): + field = value ``` diff --git a/crates/ruff_python_parser/src/parser/mod.rs b/crates/ruff_python_parser/src/parser/mod.rs index 7ba3f9c753..8c562e80be 100644 --- a/crates/ruff_python_parser/src/parser/mod.rs +++ b/crates/ruff_python_parser/src/parser/mod.rs @@ -104,6 +104,10 @@ pub(crate) struct Parser<'src> { /// basedpython: depth of nested class bodies currently being parsed. /// Used to recognise `init(...)` as a method shorthand only inside a class. class_body_depth: u32, + /// basedpython: how many function bodies enclose the statement being parsed, + /// counting only those not re-entered by a class body since. a visibility + /// keyword on a declaration in one is written on a local + function_body_depth: u32, /// basedpython: extra class-body members a single `parse_statement` produced /// but could not return directly. A property accessor block lowers one @@ -189,6 +193,7 @@ impl<'src> Parser<'src> { recursion_depth: 0, current_token_id: TokenId::default(), class_body_depth: 0, + function_body_depth: 0, pending_members: Vec::new(), pending_narrow_props: Vec::new(), expr_consumed_suite: false, diff --git a/crates/ruff_python_parser/src/parser/statement.rs b/crates/ruff_python_parser/src/parser/statement.rs index 037c8f04ba..7a7dab25dc 100644 --- a/crates/ruff_python_parser/src/parser/statement.rs +++ b/crates/ruff_python_parser/src/parser/statement.rs @@ -2,7 +2,10 @@ use std::fmt::{Display, Write}; use thin_vec::ThinVec; -use ruff_python_ast::helpers::{is_compound_statement, written_annotation_type}; +use ruff_python_ast::helpers::{ + DeclarationMarker, DeclarationMarkerKind, MemberVisibility, declaration_annotation_type, + is_compound_statement, property_backing_name, written_annotation_type, +}; use ruff_python_ast::name::Name; use ruff_python_ast::token::TokenKind; use ruff_python_ast::visitor::transformer::{self, Transformer}; @@ -100,6 +103,7 @@ fn is_modifier_kw(text: &str) -> bool { | "export" | "public" | "private" + | "protected" // basedpython: `late var x: T` defers a property's initialisation. // the keyword strips like any other modifier prefix; validity (only on // `var`, never with an initialiser) is checked where the property is lowered @@ -152,6 +156,7 @@ fn definition_modifier_marker(kw: &str) -> Option<&'static str> { "static" => "static", "export" | "public" => "export", "private" => "private", + "protected" => "protected", _ => return None, }) } @@ -199,10 +204,18 @@ fn param_prefix_declares_attribute(prefix: &str) -> bool { .any(|word| matches!(word, "let" | "var")) } -/// True when a parameter's modifier prefix carries the `private` visibility -/// keyword — the synthesised attribute is then name-mangled (`self.__name`). -fn param_prefix_is_private(prefix: &str) -> bool { - prefix.split_whitespace().any(|word| word == "private") +/// The visibility a modifier prefix declares for the member it binds. On a +/// parameter, the parameter itself always keeps the name it was written with; +/// only the attribute it synthesises is affected. +fn prefix_visibility(prefix: &str) -> MemberVisibility { + prefix + .split_whitespace() + .find_map(|word| match word { + "private" => Some(MemberVisibility::Private), + "protected" => Some(MemberVisibility::Protected), + _ => None, + }) + .unwrap_or_default() } /// The range of the `let` keyword inside a parameter's modifier prefix (the @@ -463,26 +476,13 @@ fn synth_backing_attr(backing: &Name, ctx: ExprContext, at: TextSize) -> Expr { } /// The declared type of a property, peeled out of the synthetic `let` / `var` -/// declaration marker: `__let__[T]` / `__modifier_annot__[T]` carry the type in -/// the subscript slice. A `final` or a `private` anywhere in the modifier chain -/// swaps the marker for `__final__` / `__private_annot__`, which carry the type the -/// same way. An untyped declaration (`let x` / `var x = v`, whose marker is a bare -/// `Name`) has no declared type. +/// declaration marker. Whatever the modifier chain swapped the marker for — a +/// `final`, a visibility keyword, both — the type rides in the subscript slice, +/// and [`declaration_annotation_type`] is the one place that knows every marker +/// that can carry it. An untyped declaration (`let x` / `var x = v`, whose marker +/// is a bare `Name`) has no declared type. fn property_decl_type(annotation: &Expr) -> Option { - if let Expr::Subscript(subscript) = annotation - && let Expr::Name(marker) = subscript.value.as_ref() - && matches!( - marker.id.as_str(), - "__let__" - | "__modifier_annot__" - | "__private_annot__" - | "__final__" - | "__classvar_annot__" - ) - { - return Some((*subscript.slice).clone()); - } - None + declaration_annotation_type(annotation).cloned() } /// Calls `report` for every part of `expr` that cannot be assigned to. @@ -872,6 +872,7 @@ impl<'src> Parser<'src> { ); self.bump(TokenKind::Name); let mut alias = self.parse_type_alias_statement(); + self.check_visibility_placement(MemberVisibility::Private, alias.range); alias.is_private = true; alias.range = self.node_range(start); // a type alias is a *simple* statement, so unlike the @@ -944,7 +945,8 @@ impl<'src> Parser<'src> { "`var` declarations are not valid in .py files".to_string(), ); if following == TokenKind::Colon { - let decl = self.parse_modifier_annot_decl(start, "__modifier_annot__"); + let decl = + self.parse_modifier_annot_decl(start, DeclarationMarkerKind::Annot); return Some(mark_context(decl, has_context)); } if following != TokenKind::Equal { @@ -1047,7 +1049,8 @@ impl<'src> Parser<'src> { self.error_if_not_basedpython(format!( "`{kw}` modifier on annotated assignments is not valid in .py files" )); - let decl = self.parse_modifier_annot_decl(start, "__modifier_annot__"); + let decl = + self.parse_modifier_annot_decl(start, DeclarationMarkerKind::Annot); Some(mark_context(decl, has_context)) } _ => None, @@ -1106,14 +1109,17 @@ impl<'src> Parser<'src> { } /// Returns whether the modifier-keyword token at chain position `idx` is a - /// visibility keyword (`private`, `public`, or `export`). + /// visibility keyword (`private`, `protected`, `public`, or `export`). fn is_visibility_modifier_at(&mut self, idx: usize) -> bool { let range = if idx == 0 { self.current_token_range() } else { self.peek_nth(idx - 1).1 }; - matches!(self.src_text(range), "private" | "public" | "export") + matches!( + self.src_text(range), + "private" | "protected" | "public" | "export" + ) } /// Whether the parser is sitting on the `init(...)` constructor shorthand: @@ -1212,16 +1218,34 @@ impl<'src> Parser<'src> { if matches!(self.peek(), TokenKind::Def | TokenKind::Async) { continue; } - // `class var x: T` declares a class *variable*, which is not a - // definition a modifier chain can hang off — and it carries one - // marker, which its own keyword already fills. left alone it - // parses on as a nested class named by the binding keyword - let binding = self.peek_nth(0).1; + // `class var x: T` / `class x = v` declares a class *variable*, + // which is not a definition a modifier chain can hang off, and it + // carries one marker, which its own keyword already fills. a + // visibility keyword is the exception: it says who may reach the + // variable, which composes with any declaration. left alone + // anything else parses on as a nested class named by the binding + // keyword + let (first_kind, binding) = self.peek_nth(0); let (named, name_range) = self.peek_nth(1); - if matches!(self.src_text(binding), "let" | "var") && declares_a_name(named) { + let binding_kw = self.src_text(binding).to_owned(); + let declares_class_var = + matches!(binding_kw.as_str(), "let" | "var") && declares_a_name(named); + let assigns_class_var = declares_a_name(first_kind) && named == TokenKind::Equal; + if declares_class_var || assigns_class_var { + let only_visibility = self + .src_text(TextRange::new(start, self.current_token_range().start())) + .split_whitespace() + .all(|word| matches!(word, "private" | "protected")); + if only_visibility { + return if declares_class_var { + self.parse_class_var_annot_decl(start, &binding_kw) + } else { + self.parse_class_var_decl(start) + }; + } self.add_error( ParseErrorType::OtherError( - "a `class` variable declaration takes no modifier — write it on its own" + "a `class` variable takes no modifier but `private` or `protected`" .to_string(), ), TextRange::new(start, name_range.end()), @@ -1244,6 +1268,25 @@ impl<'src> Parser<'src> { break; } + // basedpython: a visibility keyword needs a class member or a module-level + // declaration to say something about + let visibility_marker = + decorators + .iter() + .find_map(|decorator| match &decorator.expression { + Expr::Name(name) if name.ctx == ExprContext::Invalid => { + match name.id.as_str() { + "private" => Some((MemberVisibility::Private, decorator.range)), + "protected" => Some((MemberVisibility::Protected, decorator.range)), + _ => None, + } + } + _ => None, + }); + if let Some((visibility, range)) = visibility_marker { + self.check_visibility_placement(visibility, range); + } + // the chain is complete, so the definition it modifies is now known: a // modifier that reads on the other kind decides nothing here, and was // being carried into a lowering that had no arm for it and left the @@ -1337,6 +1380,11 @@ impl<'src> Parser<'src> { /// Parses `class a = 1` → produces a synthetic `AnnAssign` that the /// `modifiers` transform rewrites to `a: ClassVar = 1`. fn parse_class_var_decl(&mut self, start: TextSize) -> Stmt { + // a visibility keyword ahead of `class` (`private class count = 0`) was + // consumed by the modifier chain, so it is read back from the source + let visibility = prefix_visibility( + self.src_text(TextRange::new(start, self.current_token_range().start())), + ); // consume "class" self.bump(TokenKind::Class); let name = self.parse_identifier(); @@ -1352,7 +1400,9 @@ impl<'src> Parser<'src> { node_index: AtomicNodeIndex::NONE, }); let annotation = Expr::Name(ast::ExprName { - id: Name::new_static("__classvar__"), + id: Name::new_static( + DeclarationMarker::new(DeclarationMarkerKind::ClassVar, visibility).id(), + ), ctx: ExprContext::Invalid, range: class_range, node_index: AtomicNodeIndex::NONE, @@ -1417,9 +1467,9 @@ impl<'src> Parser<'src> { ); } let marker = if keyword == "let" { - "__final__" + DeclarationMarkerKind::Final } else { - "__classvar_annot__" + DeclarationMarkerKind::ClassVarAnnot }; let stmt = self.parse_modifier_annot_decl(start, marker); @@ -1468,15 +1518,25 @@ impl<'src> Parser<'src> { /// Parses `let x = 5` → produces a synthetic `AnnAssign` that the /// `modifiers` transform rewrites to `x: Final = 5`. fn parse_let_decl(&mut self, start: TextSize) -> Stmt { + // the caller has already consumed the modifier chain, so the source + // between the statement's start and the `let` is what it was written + // with — and a visibility keyword in there renames the member the `let` + // binds, so it rides in the marker the same way it does elsewhere + let visibility = prefix_visibility( + self.src_text(TextRange::new(start, self.current_token_range().start())), + ); self.bump(TokenKind::Name); // consume "let" let name = self.parse_identifier(); + self.check_visibility_placement(visibility, TextRange::new(start, name.range.start())); // the marker spans the whole keyword prefix, from the statement's start — a // modifier ahead of the `let` (`context let a: T`, `private let a = v`) is part of // what was written, and everything that re-emits or highlights the declaration // reads it from this range let let_range = TextRange::new(start, name.range.start()); let let_name = Expr::Name(ast::ExprName { - id: Name::new_static("__let__"), + id: Name::new_static( + DeclarationMarker::new(DeclarationMarkerKind::Let, visibility).id(), + ), ctx: ExprContext::Invalid, range: let_range, node_index: AtomicNodeIndex::NONE, @@ -1547,7 +1607,10 @@ impl<'src> Parser<'src> { let modifier_start = self.current_token_range().start(); // consume modifier keywords until we reach the variable name (the Name // token immediately followed by `=`, or by the end of the statement for - // the initializer-less `var x` the caller has already rejected). + // the initializer-less `var x` the caller has already rejected). a + // visibility keyword in the chain renames the member it binds, so it is + // recorded in the marker exactly as the annotated form records it + let mut visibility = MemberVisibility::Public; loop { let (next_kind, _) = self.peek_nth(0); if matches!( @@ -1556,9 +1619,15 @@ impl<'src> Parser<'src> { ) { break; } + match self.src_text(self.current_token_range()) { + "private" => visibility = MemberVisibility::Private, + "protected" => visibility = MemberVisibility::Protected, + _ => {} + } self.bump(TokenKind::Name); } let name = self.parse_identifier(); + self.check_visibility_placement(visibility, TextRange::new(start, name.range.start())); let value = self .eat(TokenKind::Equal) .then(|| self.parse_declaration_value()); @@ -1569,7 +1638,9 @@ impl<'src> Parser<'src> { node_index: AtomicNodeIndex::NONE, }); let annotation = Expr::Name(ast::ExprName { - id: Name::new_static("__modifier_assign__"), + id: Name::new_static( + DeclarationMarker::new(DeclarationMarkerKind::Assign, visibility).id(), + ), ctx: ExprContext::Invalid, range: TextRange::new(modifier_start, name.range.start()), node_index: AtomicNodeIndex::NONE, @@ -1657,7 +1728,7 @@ impl<'src> Parser<'src> { /// Parses `abstract a: int` → produces a synthetic `AnnAssign` that the /// `modifiers` transform rewrites to `a: int` (strips the `abstract` prefix). fn parse_abstract_annot_decl(&mut self, start: TextSize) -> Stmt { - self.parse_modifier_annot_decl(start, "__abstract_annot__") + self.parse_modifier_annot_decl(start, DeclarationMarkerKind::Annot) } /// Parses `final NAME: T [= v]` into an `AnnAssign` whose annotation is the @@ -1670,7 +1741,9 @@ impl<'src> Parser<'src> { self.bump(TokenKind::Name); // consume "final" let name = self.parse_identifier(); let final_marker = Expr::Name(ast::ExprName { - id: Name::new_static("__final__"), + id: Name::new_static( + DeclarationMarker::new(DeclarationMarkerKind::Final, MemberVisibility::Public).id(), + ), ctx: ExprContext::Invalid, range: final_range, node_index: AtomicNodeIndex::NONE, @@ -1714,15 +1787,37 @@ impl<'src> Parser<'src> { } fn parse_visibility_annot_decl(&mut self, start: TextSize) -> Stmt { - self.parse_modifier_annot_decl(start, "__visibility_annot__") + self.parse_modifier_annot_decl(start, DeclarationMarkerKind::Annot) + } + + /// basedpython: reports a visibility keyword written where it has nothing to + /// say. `private` and `protected` say who may reach a class member, and + /// `private` alone also marks a module-level declaration as the module's own. + /// in a function body, outside any class nested in it, a declaration is a + /// local, which nothing outside the function reaches anyway + fn check_visibility_placement(&mut self, visibility: MemberVisibility, range: TextRange) { + if visibility == MemberVisibility::Public || self.class_body_depth > 0 { + return; + } + let message = if self.function_body_depth > 0 { + format!( + "`{}` is only a modifier on a class member or a module-level declaration", + visibility.keyword() + ) + } else if visibility == MemberVisibility::Protected { + "`protected` is only a modifier on a class member".to_string() + } else { + return; + }; + self.add_error(ParseErrorType::OtherError(message), range); } - /// Parses ` name: T [= v]` and emits an `AnnAssign` whose - /// annotation is `Subscript(Name(synthetic_id), T)` — a synthetic marker - /// spanning the modifier prefix, keeping the declared type `T` in annotation - /// position. The downstream transform deletes that prefix from the source - /// text, leaving `name: T [= v]` behind. - fn parse_modifier_annot_decl(&mut self, start: TextSize, synthetic_id: &'static str) -> Stmt { + /// parses ` name: T [= v]` into an `AnnAssign` whose annotation is + /// `Subscript(Name(), T)`: a synthetic marker of `kind`, spanning the + /// modifier prefix, with the declared type `T` kept in annotation position. + /// the downstream transform deletes that prefix from the source text, leaving + /// `name: T [= v]` behind + fn parse_modifier_annot_decl(&mut self, start: TextSize, kind: DeclarationMarkerKind) -> Stmt { // consume modifier keywords until we reach the variable name (the Name // token immediately followed by `:`), so chains like `final override x: T` // strip in full — not just the first modifier. remember a `final` and a @@ -1730,19 +1825,26 @@ impl<'src> Parser<'src> { // `final`'s `Final` qualifier and `private`'s invisibility to a widened // view of the class must both survive let mut is_final = false; - let mut is_private = false; + // a visibility keyword the modifier chain consumed before handing over — + // `private class var x: T` arrives here past its `class` — is read back + // from the source; the loop below adds whatever is still ahead + let mut visibility = prefix_visibility( + self.src_text(TextRange::new(start, self.current_token_range().start())), + ); loop { if self.peek() == TokenKind::Colon { break; } match self.src_text(self.current_token_range()) { "final" => is_final = true, - "private" => is_private = true, + "private" => visibility = MemberVisibility::Private, + "protected" => visibility = MemberVisibility::Protected, _ => {} } self.bump(TokenKind::Name); } let name = self.parse_identifier(); + self.check_visibility_placement(visibility, TextRange::new(start, name.range.start())); self.bump(TokenKind::Colon); // consume ":" let annotation_expr = self .parse_expression_list(ExpressionContext::yield_or_starred_bitwise_or()) @@ -1762,18 +1864,27 @@ impl<'src> Parser<'src> { // type stays under the marker in annotation position, so `T` is the // declaration — stashing it in `value` instead would make // `override x: T = v` declare nothing and read as `x = v`. - // `final` wins over `private`: a `Final` member is read-only, so it can - // neither be written through a widened view nor lose its qualifier here + // a `final` visibility chain carries both, so neither the `Final` + // qualifier nor the rename the visibility keyword asks for is dropped // the marker spans the whole keyword prefix from the statement's start, // which for `class var x: T` is the `class` — the formatter re-emits the // prefix from this range, so anything left out of it is dropped let marker_range = TextRange::new(start, name.range.start()); let marker = Expr::Name(ast::ExprName { - id: Name::new_static(match (is_final, is_private) { - (true, _) => "__final__", - (false, true) => "__private_annot__", - (false, false) => synthetic_id, - }), + // `class let` hands in `__final__` as its own marker, so it is final + // whether or not a `final` keyword was walked past — without this a + // visibility keyword would silently make it writable + id: Name::new_static( + DeclarationMarker::new( + if is_final || kind == DeclarationMarkerKind::Final { + DeclarationMarkerKind::Final + } else { + kind + }, + visibility, + ) + .id(), + ), ctx: ExprContext::Invalid, range: marker_range, node_index: AtomicNodeIndex::NONE, @@ -5440,13 +5551,12 @@ impl<'src> Parser<'src> { } let name_range = param.name.range; let name_id = param.name.id.clone(); - // a `private` attribute is name-mangled (`self.__name`); the parameter - // itself keeps its declared name, so the value read stays `name_id` - let attr_id = if param_prefix_is_private(prefix) { - Name::new(format!("__{name_id}")) - } else { - name_id.clone() - }; + // the attribute is declared under the name it was written with, whatever + // its visibility: the class's own body reaches a private member by that + // name, and the lowering is what spells out the mangled one. recording + // the visibility is the annotation marker's job, below + let visibility = prefix_visibility(prefix); + let attr_id = name_id.clone(); let self_expr = Expr::Name(ast::ExprName { id: Name::new_static("self"), ctx: ExprContext::Load, @@ -5481,7 +5591,22 @@ impl<'src> Parser<'src> { ); let let_marker = |range| { Expr::Name(ast::ExprName { - id: Name::new_static("__let__"), + id: Name::new_static( + DeclarationMarker::new(DeclarationMarkerKind::Let, visibility).id(), + ), + ctx: ExprContext::Invalid, + range, + node_index: AtomicNodeIndex::NONE, + }) + }; + // a `var` parameter declares a writable attribute, so there is no `let` + // marker for the visibility to ride on — it gets one of its own, which + // wraps the declared type the same way + let visibility_marker = |range| { + Expr::Name(ast::ExprName { + id: Name::new_static( + DeclarationMarker::new(DeclarationMarkerKind::Annot, visibility).id(), + ), ctx: ExprContext::Invalid, range, node_index: AtomicNodeIndex::NONE, @@ -5501,8 +5626,29 @@ impl<'src> Parser<'src> { }))), // `let a` — a bare marker: read-only, with the type left to the value (None, Some(let_range)) => Some(Box::new(let_marker(let_range))), - // `var a: T` — an ordinary, writable declaration + // `var a: T` — an ordinary, writable declaration. a visibility + // keyword on it still has to reach ty, so it rides in a marker of + // its own rather than the `let` one + (Some(ann), None) if visibility != MemberVisibility::Public => { + Some(Box::new(Expr::Subscript(ast::ExprSubscript { + range: TextRange::new(param.range.start(), ann.range().end()), + value: Box::new(visibility_marker(TextRange::new( + param.range.start(), + name_range.start(), + ))), + slice: ann.clone(), + ctx: ExprContext::Load, + node_index: AtomicNodeIndex::NONE, + is_typeof: false, + is_type_decoration: false, + }))) + } (Some(ann), None) => Some(ann.clone()), + // `private var a` — nothing is declared but the visibility, so the + // marker stands alone and the type is read off the value + (None, None) if visibility != MemberVisibility::Public => Some(Box::new( + visibility_marker(TextRange::new(param.range.start(), name_range.start())), + )), // `var a` — no declaration at all (None, None) => None, }; @@ -5621,23 +5767,14 @@ impl<'src> Parser<'src> { ); } - // `private` shifts the whole construct one level of underscore deeper: the - // property becomes `_x` and its storage `__x`. that is self-enforcing — - // the property simply does not exist under its public name, so an access - // from outside the class is an unresolved attribute rather than something - // needing its own check - let is_private = prefix.split_whitespace().any(|word| word == "private"); - let prop_name = if is_private { - Name::new(format!("_{public_name}")) - } else { - public_name.clone() - }; - // storage is an implementation detail, so it gets a dunder name and python's - // name mangling hides it: `self.__a` inside the class body resolves to - // `_A__a`, and there is no `_a` for anything outside to reach. derived from - // the *public* name so a `private` property (already `_x`) gets `__x` rather - // than a third underscore - let backing = Name::new(format!("__{public_name}")); + // a visibility keyword renames the property the way it renames any member: + // the AST keeps the name the author wrote, so ty resolves in-class accesses + // to it and checks the rest, and the lowering spells it `__x` or `_x` + let visibility = prefix_visibility(prefix); + let prop_name = public_name.clone(); + // storage is an implementation detail, so it gets a dunder name python's + // name mangling hides — see `property_backing_name` + let backing = Name::new(property_backing_name(&public_name, visibility)); // modifier keywords ty must see on the accessors themselves (`override` // checked against the base, `final`, `abstract`). they are appended *after* @@ -5656,7 +5793,10 @@ impl<'src> Parser<'src> { }; let word_start = offset + relative; offset = word_start + word.len(); - if !matches!(word, "override" | "final" | "abstract") { + if !matches!( + word, + "override" | "final" | "abstract" | "private" | "protected" + ) { continue; } let (Ok(from), Ok(to)) = ( @@ -5952,6 +6092,19 @@ impl<'src> Parser<'src> { // a backing `self.__x` in a `cls`-receiver getter, or a `@x.setter` on the // descriptor, would each add a second round of errors about the first one let has_backing = !is_static && (references_field || field_decl.is_some()); + // an initialiser is stored in the backing field, so a computed property has + // nowhere to put one. a `static` property has already been told so above + if !is_static + && !has_backing + && let Some(init) = prop_init.as_ref() + { + self.add_error( + ParseErrorType::OtherError( + "a property with no backing `field` takes no initialiser".to_string(), + ), + init.range(), + ); + } // a getter that only reads the field lets the class see storage at its own // type; one with real logic must keep being called, so it stays public-typed @@ -5962,11 +6115,14 @@ impl<'src> Parser<'src> { // in-class accesses are written under the public name, so record what each // one should resolve to. a read may reach storage directly (narrowing); a // write must reach the property so its setter still runs - let read_target = if has_backing && getter_reads_field_only { - backing.clone() - } else { - prop_name.clone() - }; + // a restricted property's in-class reads are renamed along with it, so they + // go through the property rather than being pointed at its storage + let read_target = + if has_backing && getter_reads_field_only && visibility == MemberVisibility::Public { + backing.clone() + } else { + prop_name.clone() + }; if read_target != public_name || prop_name != public_name { self.pending_narrow_props.push(PropertyRetarget { public: public_name, @@ -7544,7 +7700,9 @@ impl<'src> Parser<'src> { // inside the body re-establish their own depth if matches!(parent_clause, Clause::FunctionDef) { let saved = std::mem::take(&mut self.class_body_depth); + self.function_body_depth += 1; let body = self.parse_body_inner(parent_clause); + self.function_body_depth -= 1; self.class_body_depth = saved; body } else if matches!(parent_clause, Clause::Class) { diff --git a/crates/ruff_python_stdlib/src/basedpython.rs b/crates/ruff_python_stdlib/src/basedpython.rs index b8e0b2365b..0a8739df28 100644 --- a/crates/ruff_python_stdlib/src/basedpython.rs +++ b/crates/ruff_python_stdlib/src/basedpython.rs @@ -112,6 +112,22 @@ pub fn private_mangles(name: &str) -> bool { !name.ends_with("__") && name != "_" } +/// the name a class member is emitted under when a visibility keyword spells +/// itself with `prefix` — `__` for `private`, `_` for `protected`, and the empty +/// string for a member that is neither +/// +/// `None` when the member keeps the name it was written with: a name python +/// looks up verbatim ([`private_mangles`]) cannot be hidden by renaming it, and +/// a name that already carries the prefix is already spelled that way. the +/// second case matters most for `protected`, where a further underscore would +/// not make the member more protected but private +pub fn visibility_rename(name: &str, prefix: &str) -> Option { + if prefix.is_empty() || !private_mangles(name) || name.starts_with(prefix) { + return None; + } + Some(format!("{prefix}{name}")) +} + #[cfg(test)] mod tests { use super::{ @@ -155,3 +171,32 @@ mod tests { assert!(!private_mangles("__repr__")); } } + +#[cfg(test)] +mod visibility_tests { + use super::visibility_rename; + #[test] + fn a_visibility_prefix_is_applied_once() { + assert_eq!( + visibility_rename("helper", "__").as_deref(), + Some("__helper") + ); + assert_eq!(visibility_rename("helper", "_").as_deref(), Some("_helper")); + assert_eq!(visibility_rename("helper", ""), None); + } + + /// a `protected` member written `_x` is already spelled the way `protected` + /// spells it; prefixing again would make it `__x`, which python mangles + #[test] + fn a_name_already_carrying_the_prefix_is_left_alone() { + assert_eq!(visibility_rename("_x", "_"), None); + assert_eq!(visibility_rename("__x", "__"), None); + assert_eq!(visibility_rename("_x", "__").as_deref(), Some("___x")); + } + + #[test] + fn a_name_python_looks_up_verbatim_is_left_alone() { + assert_eq!(visibility_rename("__repr__", "__"), None); + assert_eq!(visibility_rename("__repr__", "_"), None); + } +} diff --git a/crates/ty/docs/rules.md b/crates/ty/docs/rules.md index 7684c02551..f51de6864e 100644 --- a/crates/ty/docs/rules.md +++ b/crates/ty/docs/rules.md @@ -8,7 +8,7 @@ Default level: error · Added in 0.0.64 · Related issues · -View source +View source @@ -44,7 +44,7 @@ class Base(ABC): Default level: error · Added in 0.0.13 · Related issues · -View source +View source @@ -90,7 +90,7 @@ class Derived(Base): # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -123,7 +123,7 @@ f(1, b=s1) # ok — explicit Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -159,7 +159,7 @@ report(Celsius()) # error: two conversions apply Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -194,7 +194,7 @@ extension list: Default level: warn · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -258,7 +258,7 @@ class SubProto(BaseProto, Protocol): Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -340,7 +340,7 @@ value = unknown # ty: ignore[unresolved-reference] Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -383,7 +383,7 @@ a4 = True + 1 # ok — a boolean used as a boolean Default level: error · Added in 0.0.16 · Related issues · -View source +View source @@ -435,7 +435,7 @@ Foo.method() # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -463,7 +463,7 @@ Calling a non-callable object will raise a `TypeError` at runtime. Default level: error · Added in 0.0.7 · Related issues · -View source +View source @@ -498,7 +498,7 @@ def f(x: object): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -546,7 +546,7 @@ def main(): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -593,7 +593,7 @@ def Fixed(show: bool): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -626,7 +626,7 @@ a = 1 # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -660,7 +660,7 @@ class C(A, B): ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -706,7 +706,7 @@ def App(done: bool) -> int: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -740,7 +740,7 @@ class B(A): ... # error Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -785,7 +785,7 @@ type Tree = int | list[Tree] # valid recursive alias Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -821,7 +821,7 @@ class Example: Default level: warn · Added in 0.0.1-alpha.16 · Related issues · -View source +View source @@ -860,7 +860,7 @@ old_func() # error: [deprecated] Default level: ignore · Added in 0.0.78 · Related issues · -View source +View source @@ -1032,7 +1032,7 @@ soundness checks from their type checker, and it may have false positives in som Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1064,7 +1064,7 @@ This rule is currently disabled by default because of the number of false positi Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1095,7 +1095,7 @@ class B(A, A): ... # error Default level: error · Added in 0.0.1-alpha.12 · Related issues · -View source +View source @@ -1135,7 +1135,7 @@ class A: # error Default level: ignore · Added in 0.0.73 · Related issues · -View source +View source @@ -1246,7 +1246,7 @@ Python code. Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -1295,7 +1295,7 @@ def bar() -> str: # error: [empty-body] Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -1356,7 +1356,7 @@ def h(x: object): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -1458,7 +1458,7 @@ def foo() -> "intt\b": ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1497,7 +1497,7 @@ def f(local fn: () -> None): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1538,7 +1538,7 @@ for x in [1, 2, 3]: Default level: warn · Added in 0.0.50 · Related issues · -View source +View source @@ -1578,7 +1578,7 @@ def g(value: ~A) -> None: ... # error: [experimental-syntax] Default level: error · Added in 0.0.20 · Related issues · -View source +View source @@ -1612,7 +1612,7 @@ def my_function() -> int: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.40 · Related issues · -View source +View source @@ -1647,7 +1647,7 @@ let a = 1 Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -1764,7 +1764,7 @@ def test() -> "Literal[5]": Default level: ignore · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -1817,7 +1817,7 @@ unpacking — is not a declaration, and is never reported. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.68 · Related issues · -View source +View source @@ -1886,13 +1886,52 @@ print(helper) # warning: prints `` print(Labelled) # warning: prints `` ``` +## `inaccessible-member` + + +Default level: error · basedpython only, so absent under ty-compatible · +Added in 0.0.80 · +Related issues · +View source + + + +**What it does** + +Checks for reads and writes of a `private` or `protected` class member +from outside the code allowed to reach it, and for a module's `private` +symbol reached as an attribute of the module from another one. + +**Why is this bad?** + +A visibility keyword draws a boundary around a member: `private` says only +the declaring class's own body may use it, and `protected` extends that to +a subclass's body. Reaching past the boundary defeats the point of drawing +it, and the member may be renamed or removed without notice. + +It is also very likely to fail at runtime. The lowering spells the +visibility in the member's name, and for `private` that name is one python +mangles: an access written outside the class names a different attribute, +or none at all. + +**Example** + + +```by +class Account: + init(private let balance: int) + +def audit(account: Account) -> int: + return account.balance # error: `balance` is private to `Account` +``` + ## `inconsistent-mro` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1928,7 +1967,7 @@ class C(A, B): ... # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1957,7 +1996,7 @@ t[3] # error Default level: warn · Added in 0.0.1-alpha.33 · Related issues · -View source +View source @@ -1994,7 +2033,7 @@ class MyClass: ... Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -2030,7 +2069,7 @@ class Point: Default level: error · Added in 0.0.1-alpha.12 · Related issues · -View source +View source @@ -2125,7 +2164,7 @@ will produce instances with an atypical memory layout. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2157,7 +2196,7 @@ func("foo") # error: [invalid-argument-type] Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2187,7 +2226,7 @@ a: int = "" # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2268,7 +2307,7 @@ box.value = 1 # okay Default level: error · Added in 0.0.33 · Related issues · -View source +View source @@ -2313,7 +2352,7 @@ class Sub(Base): Default level: error · Added in 0.0.1-alpha.19 · Related issues · -View source +View source @@ -2355,7 +2394,7 @@ asyncio.run(main()) Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2382,7 +2421,7 @@ class A(42): ... # error: [invalid-base] Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -2420,7 +2459,7 @@ build: # error: `build` is an experimental feature, and is off Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.5 · Related issues · -View source +View source @@ -2457,7 +2496,7 @@ extension str(A): # error: `str` does not answer every member of `A` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2486,7 +2525,7 @@ with 1: # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -2519,7 +2558,7 @@ class Fahrenheit: Default level: error · Added in 0.0.12 · Related issues · -View source +View source @@ -2572,7 +2611,7 @@ See: Default level: error · Added in 0.0.13 · Related issues · -View source +View source @@ -2608,7 +2647,7 @@ class A: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2640,7 +2679,7 @@ a: str # error Default level: warn · Added in 0.0.20 · Related issues · -View source +View source @@ -2696,7 +2735,7 @@ class Pet(Enum): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2761,7 +2800,7 @@ This rule corresponds to Ruff's Default level: error · Added in 0.0.1-alpha.28 · Related issues · -View source +View source @@ -2815,7 +2854,7 @@ class D(A): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -2848,7 +2887,7 @@ extension list[T: int]: # error: `list` declares no type parameter `T` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.4 · Related issues · -View source +View source @@ -2877,7 +2916,7 @@ Author.objects.filter(name__startswith=1) # error: lookup wants `str` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -2913,7 +2952,7 @@ def test_user(user: int) -> None: # error: fixture provides `str` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.68 · Related issues · -View source +View source @@ -2961,7 +3000,7 @@ f"{'name':>10}" # ok Default level: error · Added in 0.0.1-alpha.35 · Related issues · -View source +View source @@ -3011,7 +3050,7 @@ class NonFrozenChild(FrozenBase): # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3066,7 +3105,7 @@ class E(Generic[V]): Default level: error · Added in 0.0.12 · Related issues · -View source +View source @@ -3161,7 +3200,7 @@ a = 20 / 0 # type: ignore Default level: error · Added in 0.0.1-alpha.17 · Related issues · -View source +View source @@ -3208,7 +3247,7 @@ carol = Person(name="Carol", aeg=25) # typo! Default level: warn · Added in 0.0.15 · Related issues · -View source +View source @@ -3269,7 +3308,7 @@ def f(x, y, /): # Python 3.8+ syntax Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3309,7 +3348,7 @@ def f(t: TypeVar("U")): ... # ty: ignore[invalid-type-form] Default level: error · Added in 0.0.18 · Related issues · -View source +View source @@ -3360,7 +3399,7 @@ match object(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3394,7 +3433,7 @@ class B(metaclass=42): ... # error Default level: error · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -3511,7 +3550,7 @@ Correct use of `@override` is enforced by ty's [`invalid-explicit-override`](#in Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -3554,7 +3593,7 @@ implements Backend # error: `Backend` is not a protocol Default level: error · Added in 0.0.72 · Related issues · -View source +View source @@ -3592,7 +3631,7 @@ from module import missing # error Default level: error · Added in 0.0.1-alpha.19 · Related issues · -View source +View source @@ -3657,7 +3696,7 @@ TypeError: typing.ClassVar[int] is not valid as type argument Default level: warn · Added in 0.0.31 · Related issues · -View source +View source @@ -3703,7 +3742,7 @@ admin[0] # "Alice" Default level: error · Added in 0.0.1-alpha.27 · Related issues · -View source +View source @@ -3741,7 +3780,7 @@ Baz = NewType("Baz", int | str) # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3792,13 +3831,51 @@ def foo(x: int) -> int: ... - [Python documentation: `@overload`](https://docs.python.org/3/library/typing.html#typing.overload) +## `invalid-override-visibility` + + +Default level: error · basedpython only, so absent under ty-compatible · +Added in 0.0.80 · +Related issues · +View source + + + +**What it does** + +Checks for a class member declared less visible than the one it inherits +under the same name. + +**Why is this bad?** + +A visibility keyword decides the name the member is emitted under, so a +member declared less visible than the one it inherits does not override +it. It sits beside it under a different name, and the inherited member is +still what a call finds — which is never what the declaration looks like +it does. + +**Example** + + +```by +class A: + def f(self) -> int: + return 1 + +class B(A): + private def f(self) -> int: # error: `f` is public on `A` + return 2 + +B().f() # 1, not 2 +``` + ## `invalid-parameter-default` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3826,7 +3903,7 @@ def f(a: int = ""): ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -3859,7 +3936,7 @@ def test_add(a: int, b: int) -> None: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3895,7 +3972,7 @@ P2 = ParamSpec() # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3947,7 +4024,7 @@ Declare the type variable with `TypeVar("T", covariant=True)` instead. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4018,7 +4095,7 @@ def g(): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -4032,12 +4109,19 @@ exceptions. Only a `BaseException` subclass can be raised, so a clause with no exception in it can never be satisfied by anything the function does. +A type parameter in a clause stands for one type the caller chooses, so +it has to be declared an exception as well — a parameter with no bound +can be `int` as easily as `OSError`. + **Example** ```by def f() raises int: # error: `int` is not an exception ... + +def g[T](value: T) raises T: # error: `T@g` is not always an exception + ... ``` ## `invalid-regex` @@ -4046,7 +4130,7 @@ def f() raises int: # error: `int` is not an exception Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -4079,7 +4163,7 @@ if m := re.match("(a)(b)", "ab"): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -4111,7 +4195,7 @@ type Alias[reified T] = list[T] # error: an alias's parameters are erased Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4257,7 +4341,7 @@ def detail(request, pk: int): ... # ok Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -4296,7 +4380,7 @@ import "data/missing.json" as missing Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4404,7 +4488,7 @@ class C: ... Default level: error · Added in 0.0.10 · Related issues · -View source +View source @@ -4455,7 +4539,7 @@ class MyClass: Default level: error · Added in 0.0.1-alpha.6 · Related issues · -View source +View source @@ -4501,7 +4585,7 @@ InvalidAlias = TypeAliasType("InvalidAlias", list[T], type_params=(list[T],)) # Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -4567,7 +4651,7 @@ Bar[int] # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4600,7 +4684,7 @@ TYPE_CHECKING = "" # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4635,7 +4719,7 @@ b: Annotated[int] # error Default level: error · Added in 0.0.1-alpha.11 · Related issues · -View source +View source @@ -4692,7 +4776,7 @@ class C: Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -4758,7 +4842,7 @@ class Owner[U]: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4815,7 +4899,7 @@ V = TypeVar("V", list[int], int) # valid constrained Type Default level: error · Added in 0.0.16 · Related issues · -View source +View source @@ -4857,7 +4941,7 @@ U = TypeVar("U", int, str, default=bytes) # error: [invalid-type-variable-defau Default level: error · Added in 0.0.28 · Related issues · -View source +View source @@ -4893,7 +4977,7 @@ class Child(Base): Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -4934,7 +5018,7 @@ def f(options: dict[str, object]): Default level: error · Added in 0.0.9 · Related issues · -View source +View source @@ -4968,7 +5052,7 @@ class Foo(TypedDict): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -5003,13 +5087,48 @@ def f[out T](t: T) -> None: ... # error: a function's `T` has no variance type Alias[out T] = list[T] # error: `list` is invariant ``` +## `invalid-visibility` + + +Default level: error · basedpython only, so absent under ty-compatible · +Added in 0.0.80 · +Related issues · +View source + + + +**What it does** + +Checks for a visibility keyword written where it cannot do what it says. + +**Why is this bad?** + +A visibility keyword renames the member or symbol it is written on, since +that is the only enforcement python offers. Some names cannot be renamed +without changing what they mean: a dataclass field's name is its +constructor's keyword, an enum member's name is how the enum is looked up, +a `private` abstract method can never be overridden, and a dotted import +binds a package no rename can keep. The declaration would read as +restricted while doing something else. + +**Example** + + +```by +from dataclasses import dataclass + +@dataclass +class Point: + private x: int # error: a dataclass field's name is its constructor's keyword +``` + ## `invalid-yield` Default level: error · Added in 0.0.25 · Related issues · -View source +View source @@ -5043,7 +5162,7 @@ def gen() -> Iterator[int]: Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -5109,7 +5228,7 @@ def h(arg2: type): Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -5158,7 +5277,7 @@ def g(arg: object): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -5189,7 +5308,7 @@ def f(s: str): Default level: warn · Added in 0.0.30 · Related issues · -View source +View source @@ -5231,7 +5350,7 @@ Movie = TypedDict("Film", {"title": str}) # error: [mismatched-type-name] Default level: warn · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -5299,7 +5418,7 @@ and nothing is reported. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5330,7 +5449,7 @@ func() # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -5361,7 +5480,7 @@ f(1) # ok — `s` is passed implicitly Default level: ignore · Preview (since 0.0.76) · Related issues · -View source +View source @@ -5450,7 +5569,7 @@ Add `urllib3` to `project.dependencies` if your code imports it directly. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.4 · Related issues · -View source +View source @@ -5472,13 +5591,65 @@ runtime source, so most framework-aware checking silently degrades to from django.db import models # warning: install `django-stubs` for precise types ``` +## `missing-function-body` + + +Default level: error · basedpython only, so absent under ty-compatible · +Added in 0.0.81 · +Related issues · +View source + + + +**What it does** + + +Checks for a `def` written with no body at all, in a position that needs an implementation. + +**Why is this bad?** + + +A `def` with no body declares a signature. The lowering fills in `: ...`, so the function exists and +returns `None`. That is what a declaration means in a stub file; anywhere else it is an +implementation that was never written, silently stood in for by one that does nothing. + +A body may be left out where a declaration is what the position asks for: + +- in a stub file +- in an `if TYPE_CHECKING` block +- as a member of a protocol class +- as an `abstract def`, or an `@abstractmethod`-decorated method +- as an overload declaration, written `@overload` or as a run of same-name `def`s + +An `init(...)` may also be written without a body: what it does is store the attribute parameters it +declares, and that body is built for it. + +**Examples** + + +```by +def parse(s: str) -> int # ok: the run below makes this an overload declaration +def parse(s: bytes) -> int +def parse(s): + return int(s) + +# error: [missing-function-body] +def lookup() -> int +``` + +A function that is meant to do nothing says so with a body of its own: + +```by +def ignore(event: str): ... +``` + ## `missing-override-decorator` Default level: error · Level under ty-compatible: ignore · Added in 0.0.41 · Related issues · -View source +View source @@ -5539,7 +5710,7 @@ class ExplicitChild(Parent): Default level: error · Added in 0.0.75 · Related issues · -View source +View source @@ -5626,7 +5797,7 @@ class Item: Default level: error · Level under ty-compatible: ignore · Added in 0.0.45 · Related issues · -View source +View source @@ -5664,7 +5835,7 @@ def handle(m: re.Match[str]) -> str: Default level: error · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -5703,7 +5874,7 @@ alice["age"] # KeyError Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -5750,7 +5921,7 @@ def App(): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5784,7 +5955,7 @@ def f(a: int | None): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5822,7 +5993,7 @@ func("string") # error: [no-matching-overload] Default level: error · Added in 0.0.30 · Related issues · -View source +View source @@ -5859,7 +6030,7 @@ class Sub(Super): ... # error: [non-callable-init-subclass] Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -5890,7 +6061,7 @@ def f(x: int | str) -> int: Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -5919,7 +6090,7 @@ def f(a: object): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -5963,7 +6134,7 @@ def g(o: object, shape: Shape): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5992,7 +6163,7 @@ for i in 34: # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6020,7 +6191,7 @@ Subscripting an object that does not support it will raise a `TypeError` at runt Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6049,7 +6220,7 @@ def f(once done: () -> None): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6077,7 +6248,7 @@ def f(once done: () -> None): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -6115,7 +6286,7 @@ def f(x: int?): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -6170,7 +6341,7 @@ def g(name: str | None): Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -6207,7 +6378,7 @@ class B(A): Default level: error · Added in 0.0.16 · Related issues · -View source +View source @@ -6244,7 +6415,7 @@ class B(A): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.38 · Related issues · -View source +View source @@ -6287,7 +6458,7 @@ def main(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6318,7 +6489,7 @@ f(1, x=2) # error Default level: error · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -6349,7 +6520,7 @@ f(x=1) # error Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -6387,7 +6558,7 @@ A.c # error Default level: warn · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -6425,7 +6596,7 @@ A()[0] # error Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -6469,7 +6640,7 @@ from module import a # error Default level: warn · Added in 0.0.23 · Related issues · -View source +View source @@ -6501,7 +6672,7 @@ html.parser # error Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6537,7 +6708,7 @@ print(x) # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -6570,13 +6741,44 @@ class Id: Id("x") # error: `Id`'s constructor is private ``` +## `private-export` + + +Default level: error · basedpython only, so absent under ty-compatible · +Added in 0.0.80 · +Related issues · +View source + + + +**What it does** + +Checks for a `private` symbol listed in the module's `__all__`. + +**Why is this bad?** + +`__all__` lists the module's interface, and a `private` symbol is declared +not to be part of it. The lowering renames the symbol with a leading +underscore, so `from m import *` would look up a name the module does not +have, and raise. + +**Example** + + +```by +private def helper() -> int: + return 1 + +__all__ = ["helper"] # error: `helper` is private +``` + ## `private-import` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6608,7 +6810,7 @@ from helpers import Key # error: `Key` is private to `helpers` Default level: warn · Added in 0.0.60 · Related issues · -View source +View source @@ -6683,7 +6885,7 @@ def test() -> "int": Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -6725,7 +6927,7 @@ def g(a: bool | None): Default level: warn · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6760,7 +6962,7 @@ cast(int, f()) # error Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -6814,7 +7016,7 @@ if sys.version_info >= (3, 12): # ok — artificially constant Default level: warn · Added in 0.0.18 · Related issues · -View source +View source @@ -6852,7 +7054,7 @@ class C: Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -6907,7 +7109,7 @@ class Sub(Base): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -6956,7 +7158,7 @@ def f(value: int | str) -> int: Default level: error · Added in 0.0.71 · Related issues · -View source +View source @@ -7020,7 +7222,7 @@ def g(values: tuple[int, ...]) -> None: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -7053,7 +7255,7 @@ class C: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -7089,7 +7291,7 @@ class C[T]: Default level: warn · Added in 0.0.71 · Related issues · -View source +View source @@ -7132,7 +7334,7 @@ def build(t: Tag) -> None: Default level: error · Added in 0.0.20 · Related issues · -View source +View source @@ -7176,7 +7378,7 @@ class Outer[T]: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -7225,7 +7427,7 @@ def Observed(): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -7269,7 +7471,7 @@ def Counter(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7303,7 +7505,7 @@ static_assert(int(2.0 * 3.0) == 6) # error Default level: warn · Added in 0.0.39 · Related issues · -View source +View source @@ -7355,7 +7557,7 @@ limitation. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7389,7 +7591,7 @@ class B(A): ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7421,7 +7623,7 @@ class Circle(Shape): ... # error: `Shape` is sealed in another workspace Default level: error · Added in 0.0.1-alpha.30 · Related issues · -View source +View source @@ -7530,7 +7732,7 @@ class Book: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7560,7 +7762,7 @@ f("foo") # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7599,7 +7801,7 @@ def find(items: list[int]) -> int: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7646,7 +7848,7 @@ g: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7681,7 +7883,7 @@ f: # error: the block returns `None`, not `str` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7720,7 +7922,7 @@ def _(x: int): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.4 · Related issues · -View source +View source @@ -7751,7 +7953,7 @@ class User(BaseModel): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7810,7 +8012,7 @@ class A: Default level: error · Added in 0.0.20 · Related issues · -View source +View source @@ -7883,7 +8085,7 @@ the project registers with `@register.simple_block_tag`. Default level: warn · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -7949,7 +8151,7 @@ what the projects depending on it read. Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -7978,7 +8180,7 @@ def f() raises TypeError: Default level: warn · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8007,7 +8209,7 @@ reveal_type(1) # revealed: Literal[1] Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -8037,7 +8239,7 @@ def main(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8068,7 +8270,7 @@ f(x=1, y=2) # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -8279,7 +8481,7 @@ page does not render at all. Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -8313,7 +8515,7 @@ implements Backend # error: this module does not answer `Backend` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -8372,7 +8574,7 @@ def Held(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8405,7 +8607,7 @@ A().foo # error Default level: warn · Added in 0.0.1-alpha.15 · Related issues · -View source +View source @@ -8480,7 +8682,7 @@ def g(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8508,7 +8710,7 @@ import foo # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8538,7 +8740,7 @@ def check(value: int | None) -> asserts values: # error: `values` is nothing Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8656,7 +8858,7 @@ is one whose template set cannot be established. Default level: ignore · Added in 0.0.73 · Related issues · -View source +View source @@ -8784,7 +8986,7 @@ Python code. Default level: error · Added in 0.0.71 · Related issues · -View source +View source @@ -8825,7 +9027,7 @@ def f(a: object, b: int, c: Any): Default level: ignore · Added in 0.0.70 · Related issues · -View source +View source @@ -8970,7 +9172,7 @@ Python code. Default level: ignore · Added in 0.0.70 · Related issues · -View source +View source @@ -9117,7 +9319,7 @@ generator boundaries. Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -9167,7 +9369,7 @@ A() # error: nothing says which specialization this is Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -9214,7 +9416,7 @@ def Observed(items: StateList[int]): ... # ok: an observable handle Default level: warn · Added in 0.0.1-alpha.7 · Related issues · -View source +View source @@ -9260,7 +9462,7 @@ class D(C): ... # error: [unsupported-base] Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -9309,7 +9511,7 @@ b1 < b2 < b1 # error Default level: warn · Level under ty-compatible: ignore · Added in 0.0.12 · Related issues · -View source +View source @@ -9354,7 +9556,7 @@ def factory(base: type[Base]) -> type: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -9386,7 +9588,7 @@ A() + A() # error Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -9429,7 +9631,7 @@ reveal_type(project.root) # revealed: "." Default level: warn · Added in 0.0.21 · Related issues · -View source +View source @@ -9508,7 +9710,7 @@ to `false` to prevent this rule from reporting unused `type: ignore` comments. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.71 · Related issues · -View source +View source @@ -9594,7 +9796,7 @@ to `false`. Default level: warn · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -9673,7 +9875,7 @@ def foo(x: int | str) -> int | str: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index c9bd8cb902..8d9eb0aa55 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -22,12 +22,13 @@ use walkdir::WalkDir; use crate::ExitStatus; use crate::args::LoweringArgs; use crate::by_lowering::SettledLowering; -use by_stage::emit::{CheckGate, Transpiled, is_unusable_source, transpile_bug_diagnostic}; +use by_stage::emit::{CheckGate, Emit, Transpiled, is_unusable_source, transpile_bug_diagnostic}; use by_stage::project::{ BY_SOURCES, COMPILABLE_SOURCES, Rebuilder, build_project_db, may_contain_sources, module_roots, source_files, }; use by_stage::record::{BuildRecord, parse_soundness, stage_build_record}; +use by_stage::runtime::RuntimeLayout; use by_stage::sourcemap::{TracebackEntry, stage_module, write_sourcemap_module}; use by_stage::staging::{Staging, transpiled_destination}; use by_stage::verbatim::stage_verbatim; @@ -211,13 +212,19 @@ pub(crate) fn cmd_run( // lifts generated line numbers back to `.by` lines (for traceback rewriting) let mut traceback_entries: Vec = Vec::new(); let mut staging = Staging::new(tmp.path()); + let mut layout = RuntimeLayout::default(); let ok = render_check_and_transpile( &db, &handles, - &config, - CheckGate::AllErrors, - &rebuilder, - &mut by_transforms::RuntimeRequirements::default(), + &mut Emit { + config: &config, + gate: CheckGate::AllErrors, + rebuilder: &rebuilder, + requirements: &mut by_transforms::RuntimeRequirements::default(), + runtime: Some(&mut layout), + roots: &roots, + root: &root, + }, |emitted| { let relative = transpiled_destination(&roots, &root, emitted.by_path); traceback_entries.push(stage_module(&mut staging, &relative, emitted)?); @@ -231,6 +238,7 @@ pub(crate) fn cmd_run( // json file it opens, a template it renders. running out of a directory // holding only the transpiled half fails on the first of them stage_verbatim(&db, &root, &roots, &mut staging)?; + stage_runtime(&mut staging, &layout)?; stage_by_typed_markers(&db, &mut staging, &roots, &root)?; write_traceback_runtime(&mut staging, &traceback_entries)?; // what this build *was*, written into the build itself. a tree that is going @@ -381,6 +389,21 @@ fn staged_packages(staging: &Staging, roots: &[PathBuf], root: &Path) -> Vec anyhow::Result<()> { + for relative in layout.files() { + staging.write(&relative, None, by_transforms::runtime::SOURCE)?; + } + Ok(()) +} + /// Write the `by.typed` marker into every package the build ships. /// /// The marker says two things to a project that installs this one, and both are @@ -862,13 +885,19 @@ pub(crate) fn cmd_build( // is what the digests beside the map are for let mut entries: Vec = Vec::new(); let mut requirements = by_transforms::RuntimeRequirements::default(); + let mut layout = RuntimeLayout::default(); let ok = render_check_and_transpile( &db, &handles, - &config, - CheckGate::ParseErrorsOnly, - &rebuilder, - &mut requirements, + &mut Emit { + config: &config, + gate: CheckGate::ParseErrorsOnly, + rebuilder: &rebuilder, + requirements: &mut requirements, + runtime: Some(&mut layout), + roots: &roots, + root: &root, + }, |emitted| { let relative = transpiled_destination(&roots, &root, emitted.by_path); let entry = stage_module(&mut staging, &relative, emitted)?; @@ -885,6 +914,7 @@ pub(crate) fn cmd_build( // so they are staged whether or not something was reported — a `build/` a // debugger cannot read is worse than one built from a partial check stage_verbatim(&db, &root, &roots, &mut staging)?; + stage_runtime(&mut staging, &layout)?; stage_by_typed_markers(&db, &mut staging, &roots, &root)?; write_sourcemap_module(&mut staging, &entries)?; // `build/` outlives the build that wrote it and is what a debugger, a test @@ -1140,13 +1170,19 @@ pub(crate) fn cmd_compile( .collect(); let mut entries: Vec = Vec::new(); let mut requirements = by_transforms::RuntimeRequirements::default(); + let mut layout = RuntimeLayout::default(); let transpiled = render_check_and_transpile( &db, &transpilable, - &tree_config, - CheckGate::ParseErrorsOnly, - &rebuilder, - &mut requirements, + &mut Emit { + config: &tree_config, + gate: CheckGate::ParseErrorsOnly, + rebuilder: &rebuilder, + requirements: &mut requirements, + runtime: Some(&mut layout), + roots: &roots, + root: &root, + }, |emitted| { let relative = transpiled_destination(&roots, &root, emitted.by_path); entries.push(stage_module(&mut staging, &relative, emitted)?); @@ -1286,6 +1322,9 @@ pub(crate) fn cmd_compile( if let Err(error) = stage_verbatim(&db, &root, &roots, &mut staging) { break 'compiling Err(error); } + if let Err(error) = stage_runtime(&mut staging, &layout) { + break 'compiling Err(error); + } if let Err(error) = stage_by_typed_markers(&db, &mut staging, &roots, &root) { break 'compiling Err(error); } @@ -1572,19 +1611,28 @@ fn reverse_dir_converting( /// build`, but written in place rather than to `build/`). #[allow(clippy::print_stderr)] fn forward_dir(dir: &Path, config: &Config) -> anyhow::Result { - let (db, handles, rebuilder, _root) = build_project_db(dir, BY_SOURCES, None)?; + let (db, handles, rebuilder, root) = build_project_db(dir, BY_SOURCES, None)?; if handles.is_empty() { eprintln!("no .by files found"); return Ok(ExitStatus::Success); } + let roots = module_roots(&db, &root); let ok = render_check_and_transpile( &db, &handles, - config, - CheckGate::ParseErrorsOnly, - &rebuilder, - &mut by_transforms::RuntimeRequirements::default(), + &mut Emit { + config, + gate: CheckGate::ParseErrorsOnly, + rebuilder: &rebuilder, + requirements: &mut by_transforms::RuntimeRequirements::default(), + // the output lands in the source tree itself, where a runtime file + // with no `.by` beside it would read as a module the author wrote — + // to `by check`, to the linter, to `by transpile --reverse` + runtime: None, + roots: &roots, + root: &root, + }, |emitted| { let py = emitted.by_path.with_extension("py"); fs::write(&py, emitted.python).with_context(|| format!("{}", py.display()))?; @@ -1886,21 +1934,10 @@ fn compilable_files(root: &Path) -> Vec { fn render_check_and_transpile( db: &ProjectDatabase, handles: &[(PathBuf, ruff_db::files::File)], - config: &Config, - gate: CheckGate, - rebuilder: &Rebuilder, - requirements: &mut by_transforms::RuntimeRequirements, + emit: &mut Emit<'_>, consume: impl FnMut(&Transpiled<'_>) -> anyhow::Result<()>, ) -> anyhow::Result { - let emitted = by_stage::emit::check_and_transpile( - db, - handles, - config, - gate, - rebuilder, - requirements, - consume, - )?; + let emitted = by_stage::emit::check_and_transpile(db, handles, emit, consume)?; if !emitted.diagnostics.is_empty() { render_diagnostics(db, &emitted.diagnostics)?; } diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 0fe72c3909..8e99004895 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -3829,6 +3829,471 @@ def main(): ); } +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test prints why it was skipped" +)] +fn raises_guard_of_a_reified_generic_tests_the_type_argument() { + // a reified type parameter carries the type the caller chose, so the guard + // tests exactly that rather than the ceiling: `PermissionError` is an + // `OSError`, which the bound allows and `T = FileNotFoundError` does not. + // the specialization is applied after the guard decorator, so this also says + // the guard kept `rethrow[...]` answering + let Some(python) = python_at_least_312() else { + eprintln!("skipping: no python 3.12+ interpreter available"); + return; + }; + + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("main.by"), + "def boom(kind: dynamic): + raise kind(\"boom\") + +def rethrow[reified T: OSError](kind: dynamic) raises T: + boom(kind) + +def main(): + try: + rethrow[FileNotFoundError](FileNotFoundError) + except BaseException as e: + print(\"declared\", type(e).__name__) + try: + rethrow[FileNotFoundError](PermissionError) + except BaseException as e: + print(\"undeclared\", type(e).__name__) +", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "main", "--runtime-raises-checks"]) + .env("PYTHON", &python) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + assert!( + output.status.success(), + "by run failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stdout) + .replace("\r\n", "\n") + .trim(), + "declared FileNotFoundError\nundeclared AssertionError" + ); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test prints why it was skipped" +)] +fn raises_guard_of_a_reified_class_reads_the_receiver() { + // a class's type argument belongs to the instance, so a method's guard asks + // the receiver it was called on for it + let Some(python) = python_at_least_312() else { + eprintln!("skipping: no python 3.12+ interpreter available"); + return; + }; + + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("main.by"), + "def boom(kind: dynamic): + raise kind(\"boom\") + +class Rethrower[reified T: OSError]: + def rethrow(self, kind: dynamic) raises T: + boom(kind) + +def main(): + r = Rethrower[FileNotFoundError]() + try: + r.rethrow(FileNotFoundError) + except BaseException as e: + print(\"declared\", type(e).__name__) + try: + r.rethrow(PermissionError) + except BaseException as e: + print(\"undeclared\", type(e).__name__) +", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "main", "--runtime-raises-checks"]) + .env("PYTHON", &python) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + assert!( + output.status.success(), + "by run failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stdout) + .replace("\r\n", "\n") + .trim(), + "declared FileNotFoundError\nundeclared AssertionError" + ); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test prints why it was skipped" +)] +fn raises_guard_of_an_unreified_parameter_tests_its_ceiling() { + // nothing carries `T` at runtime here, and asking for it would mean reifying + // the parameter — an option that adds a check must not change the shape of + // the program. so the guard tests what the declaration states without it: + // an `OSError` passes, anything else does not. built for 3.9, the generic + // `def` goes through the pep 695 polyfill, whose `TypeVar` definitions have + // to land above the guard + let Some(python) = python_at_least_312() else { + eprintln!("skipping: no python 3.12+ interpreter available"); + return; + }; + + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("main.by"), + "def boom(kind: dynamic): + raise kind(\"boom\") + +def rethrow[T: OSError](kind: dynamic) raises T: + boom(kind) + +def main(): + try: + rethrow(PermissionError) + except BaseException as e: + print(\"inside the bound\", type(e).__name__) + try: + rethrow(ValueError) + except BaseException as e: + print(\"outside the bound\", type(e).__name__) +", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args([ + "run", + "main", + "--runtime-raises-checks", + "--min-version", + "3.9", + ]) + .env("PYTHON", &python) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + assert!( + output.status.success(), + "by run failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stdout) + .replace("\r\n", "\n") + .trim(), + "inside the bound PermissionError\noutside the bound AssertionError" + ); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test prints why it was skipped" +)] +fn raises_guard_of_a_reified_generic_method_reads_the_bound_instance() { + // a method that is itself reified is wrapped in `generic`, which binds the + // receiver and passes the rest on: the class's argument comes from the + // instance the method was looked up on, not from whatever is passed first + let Some(python) = python_at_least_312() else { + eprintln!("skipping: no python 3.12+ interpreter available"); + return; + }; + + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("main.by"), + "\ +def boom(kind: dynamic): + raise kind(\"boom\") + +class R[reified T: OSError]: + def m[reified U](self, other: dynamic, marker: U, kind: dynamic) raises T: + boom(kind) + +def main(): + r = R[FileNotFoundError]() + try: + r.m(R[PermissionError](), 1, PermissionError) + except BaseException as e: + print(\"undeclared\", type(e).__name__) + try: + r.m(R[PermissionError](), 1, FileNotFoundError) + except BaseException as e: + print(\"declared\", type(e).__name__) +", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "main", "--runtime-raises-checks"]) + .env("PYTHON", &python) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + assert!( + output.status.success(), + "by run failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stdout) + .replace("\r\n", "\n") + .trim(), + "undeclared AssertionError\ndeclared FileNotFoundError" + ); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test prints why it was skipped" +)] +fn raises_guard_of_a_nested_function_tests_a_class_parameter_at_its_ceiling() { + // `inner` is not a method, so its first argument is not a receiver and the + // class's argument cannot be read off it: the guard tests the bound instead + let Some(python) = python_at_least_312() else { + eprintln!("skipping: no python 3.12+ interpreter available"); + return; + }; + + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("main.by"), + "\ +def boom(kind: dynamic): + raise kind(\"boom\") + +class R[reified T: OSError]: + def m(self): + def inner(source: dynamic, kind: dynamic) raises T: + boom(kind) + for kind in [FileNotFoundError, PermissionError, ValueError]: + try: + inner(R[PermissionError](), kind) + except BaseException as e: + print(kind.__name__, type(e).__name__) + +def main(): + R[FileNotFoundError]().m() +", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "main", "--runtime-raises-checks"]) + .env("PYTHON", &python) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + assert!( + output.status.success(), + "by run failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stdout) + .replace("\r\n", "\n") + .trim(), + "FileNotFoundError FileNotFoundError\nPermissionError PermissionError\nValueError AssertionError" + ); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test prints why it was skipped" +)] +fn raises_guard_of_a_nested_function_reads_an_enclosing_reified_argument() { + // the guard on `inner` is evaluated inside a call of `outer`, where `T` is + // already the argument `outer` was specialized with + let Some(python) = python_at_least_312() else { + eprintln!("skipping: no python 3.12+ interpreter available"); + return; + }; + + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("main.by"), + "\ +def boom(kind: dynamic): + raise kind(\"boom\") + +def outer[reified T: OSError](kind: dynamic): + def inner() raises T: + boom(kind) + try: + inner() + except BaseException as e: + print(kind.__name__, type(e).__name__) + +def main(): + outer[FileNotFoundError](FileNotFoundError) + outer[FileNotFoundError](PermissionError) +", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "main", "--runtime-raises-checks"]) + .env("PYTHON", &python) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + assert!( + output.status.success(), + "by run failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stdout) + .replace("\r\n", "\n") + .trim(), + "FileNotFoundError FileNotFoundError\nPermissionError AssertionError" + ); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test prints why it was skipped" +)] +fn raises_guard_tests_a_subscripted_type_argument_by_its_origin() { + // `isinstance` refuses `MyErr[int]`, and a guard that raised `TypeError` + // there would replace the exception the function legitimately raised. the + // shallow test is the origin, as `list[str]` is tested as `list` + let Some(python) = python_at_least_312() else { + eprintln!("skipping: no python 3.12+ interpreter available"); + return; + }; + + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("main.by"), + "\ +def boom(kind: dynamic): + raise kind(\"boom\") + +class MyErr[X](OSError): ... + +def rethrow[reified T: OSError](kind: dynamic) raises T: + boom(kind) + +def main(): + try: + rethrow[MyErr[int]](MyErr) + except BaseException as e: + print(\"declared\", type(e).__name__) + try: + rethrow[MyErr[int]](PermissionError) + except BaseException as e: + print(\"undeclared\", type(e).__name__) +", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "main", "--runtime-raises-checks"]) + .env("PYTHON", &python) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + assert!( + output.status.success(), + "by run failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stdout) + .replace("\r\n", "\n") + .trim(), + "declared MyErr\nundeclared AssertionError" + ); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test prints why it was skipped" +)] +fn raises_guard_keeps_a_reified_generic_documented() { + // the guard wraps a reified generic in an object of its own, which must not + // answer for the function's docstring with its own + let Some(python) = python_at_least_312() else { + eprintln!("skipping: no python 3.12+ interpreter available"); + return; + }; + + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("main.by"), + "\ +def rethrow[reified T: OSError](error: T) raises T: + \"\"\"rethrows what it is given\"\"\" + raise error + +def main(): + print(rethrow.__doc__) + print(rethrow[OSError].__doc__) +", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "main", "--runtime-raises-checks"]) + .env("PYTHON", &python) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + assert!( + output.status.success(), + "by run failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stdout) + .replace("\r\n", "\n") + .trim(), + "rethrows what it is given\nrethrows what it is given" + ); +} + #[test] fn raises_guard_covers_an_async_generator() { // an async generator answers `False` to both `iscoroutinefunction` and @@ -4120,6 +4585,33 @@ fn build_writes_a_stub_as_a_stub() { ); } +/// a stub is read by a checker and never run, so a built one holds what its +/// source declares and none of what a module gets for running: its imports stay +/// imports, and `main` gets no entry point. a build hands every source the same +/// config, so it is the file that says it is a stub +#[test] +fn build_writes_a_stub_as_declarations() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("main.by"), "x = 1\n").unwrap(); + let stub = + "import json\nfrom dataclasses import dataclass\n\ndef main(name: str) -> None: ...\n"; + fs::write(dir.path().join("shapes.byi"), stub).unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .arg("build") + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "by build failed:\n{stderr}"); + assert_eq!( + fs::read_to_string(dir.path().join("build/shapes.pyi")).unwrap(), + stub + ); +} + /// `a.by` and a hand-written `a.py` are both the module `a`. picking one and /// carrying on means the build disagrees with what python will import, so this /// is reported rather than resolved @@ -5419,3 +5911,279 @@ fn init_refuses_to_write_over_a_project() { "the existing project must be untouched" ); } + +/// a project for the runtime tests: a package whose modules import one another +/// relatively, a subpackage, a hoisted named tuple and a call to a helper, and a +/// script in a folder that is no package at all +fn runtime_project(name: &str) -> PathBuf { + let dir = cli_root().join(name); + let _ = fs::remove_dir_all(&dir); + let app = dir.join("src").join("app"); + fs::create_dir_all(app.join("sub")).unwrap(); + fs::create_dir_all(dir.join("scripts")).unwrap(); + fs::write( + dir.join("pyproject.toml"), + "[project]\nname=\"s\"\nversion=\"0\"\nrequires-python=\">=3.11\"\n", + ) + .unwrap(); + fs::write(app.join("__init__.py"), "").unwrap(); + fs::write(app.join("sub").join("__init__.py"), "").unwrap(); + // `cast!` of an `object` to a parameterized class is a runtime probe, and + // `int??` in a named tuple + // field is the runtime's `Optional` evaluated as the hoisted class is built + fs::write( + app.join("one.by"), + "import json\n\n\ndef dumps(a: object) -> str:\n return json.dumps(a cast! dict[str, int])\n\n\n\ + def pair() -> (a: int??, b: int):\n return (a=None, b=2)\n", + ) + .unwrap(); + fs::write( + app.join("sub").join("deep.by"), + "from .. import one\n\n\ndef twice(a: dict[str, int]) -> str:\n return one.dumps(a) * 2\n", + ) + .unwrap(); + fs::write( + app.join("main.by"), + "from . import one\nfrom .sub import deep\n\n\ndef main():\n \ + print(one.dumps({}), deep.twice({}), one.pair().b)\n", + ) + .unwrap(); + fs::write( + dir.join("scripts").join("tool.by"), + "def f(a: object) -> int:\n return a cast! int\n\n\nprint(f(3))\n", + ) + .unwrap(); + fs::write( + dir.join("src").join("cli.by"), + "def f(a: object) -> int:\n return a cast! int\n\n\nprint(f(4))\n", + ) + .unwrap(); + dir +} + +fn build_in(dir: &Path) { + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("build") + .current_dir(dir) + .output() + .expect("failed to spawn by"); + assert!( + result.status.success(), + "by build failed:\n{}", + String::from_utf8_lossy(&result.stderr) + ); +} + +fn run_python(python: &str, dir: &Path, args: &[&str]) -> String { + let ran = Command::new(python) + .args(args) + .current_dir(dir) + .output() + .expect("failed to spawn python"); + assert!( + ran.status.success(), + "python failed:\n{}", + String::from_utf8_lossy(&ran.stderr) + ); + String::from_utf8_lossy(&ran.stdout).trim().to_owned() +} + +/// a build writes the runtime once per package and its modules call into it +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_built_package_runs_off_one_copy_of_the_runtime() { + let Some(python) = native_interpreter() else { + eprintln!("skipping: no interpreter new enough to run the built tree"); + return; + }; + let dir = runtime_project("shared_runtime_build"); + build_in(&dir); + + let build = dir.join("build"); + assert!( + build.join("app").join("_by_runtime.py").is_file(), + "the package carries the runtime" + ); + assert!( + !build + .join("app") + .join("sub") + .join("_by_runtime.py") + .exists(), + "which its subpackage shares" + ); + // a distribution ships packages, so a copy at the root would not ship + assert!(!build.join("_by_runtime.py").exists()); + + let one = fs::read_to_string(build.join("app").join("one.py")).unwrap(); + assert!( + one.contains("from app._by_runtime import"), + "the module imports what it calls:\n{one}" + ); + assert!( + !one.contains("def _checked_cast("), + "and does not define it as well:\n{one}" + ); + // a helper reached through the lazy-import proxy is a proxy call on every use + assert!( + !one.contains("_lazy_attr(\"app._by_runtime\""), + "the runtime import is eager:\n{one}" + ); + + assert_eq!( + run_python( + &python, + &build, + &["-c", "from app.main import main; main()"] + ), + "{} {}{} 2" + ); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn by_run_runs_a_package_that_imports_relatively() { + let Some(python) = native_interpreter() else { + eprintln!("skipping: no interpreter new enough to run the program"); + return; + }; + let dir = runtime_project("shared_runtime_run"); + let ran = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["run", "app.main"]) + .env("PYTHON", &python) + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + assert!( + ran.status.success(), + "by run failed:\n{}", + String::from_utf8_lossy(&ran.stderr) + ); + assert_eq!(String::from_utf8_lossy(&ran.stdout).trim(), "{} {}{} 2"); +} + +/// a script in a folder that is no package has no import that works when it is +/// run, since the folder above it is not on the path — so it carries its own +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_script_in_no_package_carries_its_own_helpers() { + let Some(python) = native_interpreter() else { + eprintln!("skipping: no interpreter new enough to run the script"); + return; + }; + let dir = runtime_project("shared_runtime_script"); + build_in(&dir); + + let scripts = dir.join("build").join("scripts"); + let tool = fs::read_to_string(scripts.join("tool.py")).unwrap(); + assert!( + !tool.contains("_by_runtime"), + "the script imports no runtime:\n{tool}" + ); + assert!(!scripts.join("_by_runtime.py").exists()); + assert_eq!(run_python(&python, &dir, &["build/scripts/tool.py"]), "3"); +} + +/// a copy at the root would be a top-level module, which a second basedpython +/// wheel built by another version overwrites on install — so a module at the +/// module root carries its own helpers too +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_root_module_carries_its_own_helpers() { + let Some(python) = native_interpreter() else { + eprintln!("skipping: no interpreter new enough to run the module"); + return; + }; + let dir = runtime_project("shared_runtime_root_module"); + build_in(&dir); + + let build = dir.join("build"); + let cli = fs::read_to_string(build.join("cli.py")).unwrap(); + assert!( + !cli.contains("_by_runtime"), + "the root module imports no runtime:\n{cli}" + ); + assert!(!build.join("_by_runtime.py").exists()); + assert_eq!(run_python(&python, &build, &["cli.py"]), "4"); +} + +fn holds_file(dir: &Path, name: &str) -> bool { + fs::read_dir(dir).unwrap().any(|entry| { + let path = entry.unwrap().path(); + path.file_name().is_some_and(|file| file == name) + || (path.is_dir() && holds_file(&path, name)) + }) +} + +/// transpiling in place writes into the source tree, where a runtime file with +/// no `.by` beside it would read as a module the author wrote +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn transpiling_a_directory_in_place_writes_no_runtime_file() { + let Some(python) = native_interpreter() else { + eprintln!("skipping: no interpreter new enough to run the output"); + return; + }; + let dir = runtime_project("shared_runtime_in_place"); + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["transpile", "src"]) + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + assert!( + result.status.success(), + "by transpile failed:\n{}", + String::from_utf8_lossy(&result.stderr) + ); + assert!(!holds_file(&dir.join("src"), "_by_runtime.py")); + assert_eq!( + run_python( + &python, + &dir.join("src"), + &["-c", "from app.main import main; main()"] + ), + "{} {}{} 2" + ); +} + +/// a re-stage patches one module into a tree an earlier build wrote, so an edit +/// that calls a helper nothing in that build called still has to find it there +#[test] +fn a_restaged_module_finds_a_helper_the_build_never_used() { + let dir = runtime_project("shared_runtime_restage"); + build_in(&dir); + let runtime = fs::read_to_string(dir.join("build").join("app").join("_by_runtime.py")).unwrap(); + assert!(runtime.contains("def _try_cast(")); + + fs::write( + dir.join("src").join("app").join("main.by"), + "from . import one\n\n\ndef check(x: object) -> str | None:\n return x cast? str\n\n\ndef main():\n print(one.dumps({}), check(1))\n", + ) + .unwrap(); + let restaged = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["restage", "build", "src/app/main.by"]) + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + let answer = String::from_utf8_lossy(&restaged.stdout); + assert!( + answer.contains("_try_cast") && answer.contains("app._by_runtime"), + "the re-staged module imports the helper from the tree's copy:\n{answer}\n{}", + String::from_utf8_lossy(&restaged.stderr) + ); +} diff --git a/crates/ty_ide/src/alignment.rs b/crates/ty_ide/src/alignment.rs index 9e54dcb053..1a5661455f 100644 --- a/crates/ty_ide/src/alignment.rs +++ b/crates/ty_ide/src/alignment.rs @@ -1,4 +1,4 @@ -//! which assignments the author lined up, so a client drawing inlay hints can keep them lined up +//! which assignments share an `=` column, so a client drawing inlay hints can keep them sharing it //! //! an inlay hint costs horizontal room. drawn after the target of an assignment — which is where a //! variable's type hint goes — it pushes everything to its right along, and a column of `=` the @@ -49,20 +49,17 @@ pub struct AlignmentMember { pub gap_start: TextSize, /// the `=` the author lined up — the end of the run of spaces, and the column to preserve + /// + /// the distance back to [`Self::gap_start`] is the room a hint has to spend before the line has + /// to grow. nothing here reads it: how much of that room a hint wants is the client's question, + /// and since the column no longer has to be padded to be a column, it is not this module's + /// either pub gap_end: TextSize, } -impl AlignmentMember { - /// the spaces the author left between the target and the `=`, which is the room a hint has to - /// spend before the line has to grow - fn gap(self) -> TextSize { - self.gap_end - self.gap_start - } -} - /// assignments the author put in one column, reported together because they have to move together /// -/// always two or more members, and always with evidence that the column was deliberate — see +/// always two or more members, and always already sharing one `=` column — see /// [`alignment_groups`] #[derive(Debug)] pub struct AlignmentGroup { @@ -76,20 +73,26 @@ pub struct AlignmentGroup { /// - siblings in one suite, so an `if` in the middle ends the run rather than being aligned across /// - unseparated by a blank line, which is how a reader tells one block of assignments from the /// next (a comment on its own line does *not* break the run — it is still one block to read) -/// - already sharing an `=` column, since a column that is not there yet is not one to preserve -/// - and padded: at least one member has two or more spaces before its `=` +/// - and already sharing an `=` column, since a column that is not there yet is not one to preserve /// -/// that last condition is the whole of the conservatism, and it is what keeps ordinary code out +/// nothing more is asked of the padding. a run of spaces before an `=` is what a *hand-aligned* +/// block looks like, and it is tempting to require one as proof the column was deliberate — but the +/// column is the thing being preserved, and this reads the same either way /// /// ```python -/// x = 1 -/// y = 2 +/// a = 1 + 1 +/// b = True or False /// ``` /// -/// those two share an `=` column, but only because the names are the same length — nobody aligned -/// anything, and a client that padded them out when their hints came back different widths would be -/// injecting space into code the author never spaced. one member with a padding run is the smallest -/// evidence that the column was typed on purpose +/// those `=` are in one column because the names are the same length rather than because anyone +/// padded them, and a reader has no way to tell the two apart, nor any reason to want to. hints of +/// unequal width (`a: Literal[2]`, `b: Literal[True]`) break that column exactly as they break a +/// padded one, and leaving it broken is the surprise. so the shared column *is* the evidence, and +/// the padding a member carries is only room the client gets to spend before the line has to grow +/// +/// the cost of taking the wider reading is a block where some lines have hints and some do not: the +/// hintless ones are pushed out to keep the column, having no hint of their own to narrow. that is +/// the same trade a padded block already makes, and it keeps the block square either way /// /// a group is reported whole even when only one of its lines is in `range`: the column is a /// property of every member at once, so half a group would be sized against the wrong maximum @@ -179,8 +182,8 @@ impl<'a> SourceOrderVisitor<'a> for AlignmentVisitor<'a> { } impl AlignmentVisitor<'_> { - /// cuts one suite into runs of assignments that share a column, and keeps the ones that were - /// deliberate + /// cuts one suite into runs of assignments that share a column, and keeps the ones with + /// somebody to share it with fn collect(&mut self, body: &[Stmt]) { let mut run: Vec = Vec::new(); // the end of the last *member*, which is where the search for a blank line starts. it does @@ -216,9 +219,12 @@ impl AlignmentVisitor<'_> { } /// keeps a finished run if it is a group worth reporting, and starts the next one either way + /// + /// two members is the whole of the test. a run got this far by sharing a column, and a column is + /// a column however it came to be one — see [`alignment_groups`] on why the padding a member + /// happens to carry is not the evidence it looks like fn flush(&mut self, run: &mut Vec) { - let padded = run.iter().any(|member| member.gap() > TextSize::from(1)); - if run.len() >= 2 && padded { + if run.len() >= 2 { self.groups.push(AlignmentGroup { members: std::mem::take(run), }); @@ -365,7 +371,7 @@ mod tests { .unwrap_or_default() .trim_end_matches('\r'); let lead = (member.gap_start - start).to_usize(); - let gap = member.gap().to_usize(); + let gap = (member.gap_end - member.gap_start).to_usize(); writeln!(out, " {line}").unwrap(); writeln!( out, @@ -410,11 +416,47 @@ basdf = 1 } #[test] - fn leaves_unpadded_assignments_alone() { + fn an_unpadded_column_is_still_a_column() { + let test = cursor_test( + "\ +a = 1 + 1 +b = True or False +", + ); + assert_snapshot!(test.alignment_groups(), @r" + group 1 + a = 1 + 1 + - gap 1 + b = True or False + - gap 1 + "); + } + + #[test] + fn a_column_can_have_no_gap_at_all() { + // no spaces is a gap of nought rather than no member: the `=` still share a column, and a + // member left out would leave the group sized against the wrong maximum. only reachable now + // that a run no longer needs a padded member to qualify + let test = cursor_test( + "\ +ab=f() +a =1 +", + ); + assert_snapshot!(test.alignment_groups(), @r" + group 1 + ab=f() + - gap 0 + a =1 + - gap 1 + "); + } + + #[test] + fn a_column_of_one_is_no_column() { let test = cursor_test( "\ x = 1 -y = 2 ", ); assert_snapshot!(test.alignment_groups(), @"no groups"); diff --git a/crates/ty_ide/src/goto.rs b/crates/ty_ide/src/goto.rs index f291e8d7d2..633fde0554 100644 --- a/crates/ty_ide/src/goto.rs +++ b/crates/ty_ide/src/goto.rs @@ -1577,11 +1577,9 @@ fn property_declaration_name(syntax: AnyNodeRef<'_>, offset: TextSize) -> Option let function = member.as_function_def_stmt()?; // only a *synthesized* property answers: an ordinary `def` keeps its name // inside its own range, so the walk above would already have found it - let is_property = function.decorator_list.iter().any(|decorator| { - matches!(&decorator.expression, ast::Expr::Name(name) if is_synthetic_marker(name.into())) - }); - (is_property && function.name.range().contains_range(token_range)) - .then_some(GotoTarget::FunctionDef(function)) + (function.property_construct_range().is_some() + && function.name.range().contains_range(token_range)) + .then_some(GotoTarget::FunctionDef(function)) }) } diff --git a/crates/ty_ide/src/inlay_hints.rs b/crates/ty_ide/src/inlay_hints.rs index f0265da138..ad42d65f7b 100644 --- a/crates/ty_ide/src/inlay_hints.rs +++ b/crates/ty_ide/src/inlay_hints.rs @@ -26,11 +26,11 @@ use ty_python_semantic::types::context_params::implicit_context_arguments; use ty_python_semantic::types::ide_support::{ InferredInvalidations, InferredStateReads, InlayHintCallArgumentDetails, StateRead, WriteSite, hintable_parameter_type, implicit_enum_member_value, inferred_derived_dependencies, - inferred_invalidations, inferred_override, inferred_raises, inferred_return_annotation, - inferred_state_reads, inferred_type_param_variance, inherited_parameter_annotation, - inherited_parameter_default, inlay_hint_call_argument_details, is_composable_function, - is_reveal_type_function, is_union_special_form, numeric_promotion, parameter_stability, - trailing_lambda_implicit_parameters, type_parameter_names, + inferred_invalidations, inferred_override, inferred_property_type, inferred_raises, + inferred_return_annotation, inferred_state_reads, inferred_type_param_variance, + inherited_parameter_annotation, inherited_parameter_default, inlay_hint_call_argument_details, + is_composable_function, is_reveal_type_function, is_union_special_form, numeric_promotion, + parameter_stability, trailing_lambda_implicit_parameters, type_parameter_names, }; use ty_python_semantic::types::{DisplaySettings, Type, TypeDetail}; use ty_python_semantic::{HasType, SemanticModel, with_display_for_file}; @@ -605,9 +605,10 @@ impl InlayHint { } } - /// The type of a parameter the source leaves unannotated, shown where the - /// annotation would be written. - fn parameter_type( + /// the type of a place the source leaves unannotated — a parameter, or a + /// basedpython property declaration — shown where the annotation would be + /// written + fn inferred_annotation( db: &dyn Db, env: &ProgramEnvironment<'_>, position: TextSize, @@ -1004,6 +1005,16 @@ pub struct InlayHintSettings { /// ``` pub inferred_return_types: bool, + /// basedpython: whether to show the type a property declaration leaves to its + /// accessors. + /// + /// ```by + /// class A: + /// let a": 1" + /// get() = 1 + /// ``` + pub property_types: bool, + /// basedpython: whether to show the arguments a call site fills implicitly /// from the `context` declarations in scope. /// @@ -1067,6 +1078,7 @@ impl InlayHintSettings { inherited_parameter_types: false, inherited_parameter_defaults: false, inferred_return_types: false, + property_types: false, implicit_arguments: false, enum_values: false, template_binding_types: false, @@ -1096,6 +1108,7 @@ impl InlayHintSettings { inherited_parameter_types, inherited_parameter_defaults, inferred_return_types, + property_types, implicit_arguments, enum_values, template_binding_types, @@ -1122,6 +1135,7 @@ impl InlayHintSettings { || inherited_parameter_types || inherited_parameter_defaults || inferred_return_types + || property_types || implicit_arguments || enum_values || template_binding_types @@ -1152,6 +1166,7 @@ impl Default for InlayHintSettings { inherited_parameter_types: true, inherited_parameter_defaults: true, inferred_return_types: true, + property_types: true, implicit_arguments: true, enum_values: true, template_binding_types: true, @@ -1809,7 +1824,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; }; - self.hints.push(InlayHint::parameter_type( + self.hints.push(InlayHint::inferred_annotation( self.db, env, parameter.name.range().end(), @@ -1836,7 +1851,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; }; - self.hints.push(InlayHint::parameter_type( + self.hints.push(InlayHint::inferred_annotation( self.db, env, parameter.name.range().end(), @@ -1882,7 +1897,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { let Some(returned) = function .inferred_type(&self.model) - .and_then(|ty| inferred_return_annotation(self.db, ty)) + .and_then(|ty| inferred_return_annotation(self.db, env, ty)) else { return; }; @@ -1895,6 +1910,38 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { )); } + /// basedpython: hint the type a property declaration leaves to its accessors + /// + /// the construct writes its name once and the getter carries that name's range, + /// so the type goes where the declaration would have written it — after the + /// name, exactly as it does for an unannotated variable + fn add_property_type(&mut self, getter: &ast::StmtFunctionDef) { + if !self.settings.property_types + || !self.is_basedpython() + || getter.property_construct_range().is_none() + // the declaration named a type; the getter carries it as its return + || getter.returns.is_some() + // a malformed accessor recovers to one with no body, which states nothing + || getter.body.is_empty() + // the getter is visited for the whole construct, but the hint sits on its name + || self.range.intersect(getter.name.range()).is_none() + { + return; + } + + let Some(ty) = inferred_property_type(&self.model, getter) else { + return; + }; + + let env = &self.model.program_environment(); + self.hints.push(InlayHint::inferred_annotation( + self.db, + env, + getter.name.range().end(), + ty, + )); + } + /// Visit an expression that denotes a type rather than a value. fn visit_type_expr(&mut self, expr: &'a Expr) { let in_type_expression = std::mem::replace(&mut self.in_type_expression, true); @@ -1925,7 +1972,18 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { impl<'a> SourceOrderVisitor<'a> for InlayHintVisitor<'a, '_> { fn enter_node(&mut self, node: AnyNodeRef<'a>) -> TraversalSignal { - if self.range.intersect(node.range()).is_some() { + // basedpython: a property getter is ranged onto its accessor, but the + // construct it was synthesized from reaches back over the declaration — + // the line the property's own hint sits on. asking about the accessor + // alone loses that hint whenever the accessor is below the visible range + let range = match node { + AnyNodeRef::StmtFunctionDef(function) => function + .property_construct_range() + .unwrap_or_else(|| function.range()), + node => node.range(), + }; + + if self.range.intersect(range).is_some() { TraversalSignal::Traverse } else { TraversalSignal::Skip @@ -2014,10 +2072,12 @@ impl<'a> SourceOrderVisitor<'a> for InlayHintVisitor<'a, '_> { return; } // basedpython: a property accessor's whole header is synthesized — the - // parameter list stands for no source and the name and declared type - // belong to the construct's head, which is written once and hinted - // there — so only the accessor body is real + // parameter list stands for no source, and the name belongs to the + // construct's head, which is written once and takes the one hint above + // — so beyond that only the accessor body is real Stmt::FunctionDef(function) if has_synthesized_header(function) => { + self.add_property_type(function); + let enclosing_class = self.enclosing_class.take(); self.visit_body(&function.body); self.enclosing_class = enclosing_class; @@ -2400,8 +2460,15 @@ pub(crate) fn untyped_declaration_value(assign: &ast::StmtAnnAssign) -> Option<& if !marker.ctx.is_invalid() { return None; } - matches!(marker.id.as_str(), "__let__" | "__modifier_assign__") - .then(|| assign.value.as_deref()) + ruff_python_ast::helpers::DeclarationMarker::from_id(marker.id.as_str()) + .is_some_and(|marker| { + matches!( + marker.kind, + ruff_python_ast::helpers::DeclarationMarkerKind::Let + | ruff_python_ast::helpers::DeclarationMarkerKind::Assign + ) + }) + .then_some(assign.value.as_deref()) .flatten() } @@ -3077,6 +3144,171 @@ Source with applied edits: "); } + /// a property declaration that names no type takes one from its initialiser, or + /// from what its getter returns when it has none, and it is shown where the + /// declaration would have written it + #[test] + fn property_type_hints() { + let mut test = basedpython_inlay_hint_test( + " + class A: + let a + get() = 1 + + let b + field = 'two' + + var c = 0 + get() = field + set(value): field = value + + let d: bytes + get() = b'' + ", + ); + + assert_snapshot!(test.inlay_hints_with_settings(&InlayHintSettings { + property_types: true, + ..InlayHintSettings::none() + })); + } + + /// the declaration a property hint sits on is a line above the accessor the + /// parser ranged the getter onto, so a request that stops short of the + /// accessor still has to reach it + #[test] + fn property_type_hint_above_the_requested_range() { + let mut test = basedpython_inlay_hint_test( + " + class A: + let a + get() = 1 + ", + ); + + assert_snapshot!(test.inlay_hints_with_settings(&InlayHintSettings { + property_types: true, + ..InlayHintSettings::none() + })); + } + + /// a request covering only the accessor leaves out the declaration the hint + /// sits on + #[test] + fn property_type_hint_outside_the_requested_range() { + let mut test = basedpython_inlay_hint_test( + " + class A: + let a + get() = 1 + ", + ); + + assert_snapshot!(test.inlay_hints_with_settings(&InlayHintSettings { + property_types: true, + ..InlayHintSettings::none() + })); + } + + /// a `private` property's getter is named `_b` while the source spells `b`, and + /// a `static let` is a class-level descriptor rather than a `property`, but both + /// are hinted after the name the declaration wrote. a getter that falls off its + /// end returns `None`, which is the property's type like any other + #[test] + fn property_type_hint_shapes() { + let mut test = basedpython_inlay_hint_test( + " + class A: + private let b + get() = 'b' + + static let c + get() = 3 + + let n + get(): pass + ", + ); + + assert_snapshot!(test.inlay_hints_with_settings(&InlayHintSettings { + property_types: true, + ..InlayHintSettings::none() + })); + } + + /// a type ty could not settle on says nothing a reader could write: a getter + /// that reads its own property never settles, and a malformed accessor recovers + /// to one with no body + #[test] + fn property_type_hint_without_a_settled_type() { + let mut test = basedpython_inlay_hint_test( + " + class A: + let rec + get() = self.rec + + let m + get() + ", + ); + + assert_snapshot!(test.inlay_hints_with_settings(&InlayHintSettings { + property_types: true, + ..InlayHintSettings::none() + })); + } + + /// accessor blocks are basedpython syntax: a python file reports them as invalid + /// and hints nothing about them + #[test] + fn property_type_hint_in_a_python_file() { + let mut test = inlay_hint_test( + " + class A: + let a + get() = 1 + ", + ); + + assert_snapshot!(test.inlay_hints_with_settings(&InlayHintSettings { + property_types: true, + ..InlayHintSettings::none() + })); + } + + #[test] + fn property_type_hint_can_be_turned_off() { + let mut test = basedpython_inlay_hint_test( + " + class A: + let a + get() = 1 + ", + ); + + assert_snapshot!(test.inlay_hints_with_settings(&InlayHintSettings { + property_types: false, + ..InlayHintSettings::default() + })); + } + + /// a function that only ever calls itself never settles on a return type, so + /// there is nothing to hint + #[test] + fn inferred_return_type_that_never_settles() { + let mut test = basedpython_inlay_hint_test( + " + def f(): + return f() + ", + ); + + assert_snapshot!(test.inlay_hints_with_settings(&InlayHintSettings { + inferred_return_types: true, + ..InlayHintSettings::none() + })); + } + #[test] fn test_assign_statement() { let mut test = inlay_hint_test( @@ -10346,6 +10578,24 @@ Source with applied edits: })); } + #[test] + fn basedpython_inferred_raises_of_a_generic_callee() { + let mut test = basedpython_inlay_hint_test( + " + def rethrow[T: BaseException](error: T) raises T: + raise error + + def caller(): + rethrow(TypeError()) + ", + ); + + assert_snapshot!(test.inlay_hints_with_settings(&InlayHintSettings { + inferred_raises: true, + ..InlayHintSettings::none() + })); + } + #[test] fn basedpython_inferred_variance() { let mut test = basedpython_inlay_hint_test( diff --git a/crates/ty_ide/src/semantic_tokens.rs b/crates/ty_ide/src/semantic_tokens.rs index ef5532d9e8..7277712f5b 100644 --- a/crates/ty_ide/src/semantic_tokens.rs +++ b/crates/ty_ide/src/semantic_tokens.rs @@ -409,22 +409,6 @@ const EXPORT_IMPORT: &[&str] = &["export"]; /// ranged onto the accessor bodies, not onto the headers. const ACCESSOR_KEYWORDS: &[&str] = &["get", "set", "field", "late"]; -/// basedpython: the range of the property construct `func` was synthesized from, -/// if it is a property getter. -/// -/// The parser lowers `var x: int` plus its accessor blocks into a getter, an -/// optional backing declaration and an optional setter, and marks the getter with -/// a synthetic decorator spanning the whole construct — the same span the -/// transpiler and the formatter claim for it. -fn property_construct(func: &ast::StmtFunctionDef) -> Option { - func.decorator_list.iter().find_map(|decorator| { - let Expr::Name(marker) = &decorator.expression else { - return None; - }; - matches!(marker.id.as_str(), "__property__" | "__static_property__").then(|| marker.range()) - }) -} - /// One member of a property construct, and which part of the source it stands /// for. Only a setter can declare a name in its header. #[derive(Debug, Clone, Copy, PartialEq)] @@ -1347,7 +1331,7 @@ impl<'db> SemanticTokenVisitor<'db> { while let Some(statement) = members.next() { let Some(construct) = statement .as_function_def_stmt() - .and_then(property_construct) + .and_then(ast::StmtFunctionDef::property_construct_range) .filter(|construct| construct.contains_range(statement.range())) else { self.visit_stmt(statement); @@ -7701,6 +7685,33 @@ var b: int = 5 "#); } + #[test] + fn semantic_tokens_visibility_keywords() { + // `protected` reaches the visitor the same way every other modifier does + // — as a synthetic decorator on a `def`, and as the annotation marker on + // a declaration + let test = SemanticTokenTest::new_by( + " +class A: + protected step: int = 2 + protected def helper(self): ... +", + ); + + let tokens = test.highlight_file(); + + assert_snapshot!(test.to_snapshot(&tokens), @r#" + "A" @ 7..8: Class [definition] + "protected" @ 14..23: Keyword + "step" @ 24..28: Variable [definition] + "int" @ 30..33: Class + "2" @ 36..37: Number + "protected" @ 42..51: Keyword + "helper" @ 56..62: Method [definition] + "self" @ 63..67: SelfParameter [definition] + "#); + } + #[test] fn semantic_tokens_modifier_annotation_keywords() { // the modifiers that carry no type meaning still declare a type, and it diff --git a/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_inferred_raises_of_a_generic_callee.snap b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_inferred_raises_of_a_generic_callee.snap new file mode 100644 index 0000000000..95a89948c2 --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_inferred_raises_of_a_generic_callee.snap @@ -0,0 +1,10 @@ +--- +source: crates/ty_ide/src/inlay_hints.rs +expression: "test.inlay_hints_with_settings(&InlayHintSettings\n{ inferred_raises: true, ..InlayHintSettings::none() })" +--- + +def rethrow[T: BaseException](error: T) raises T: + raise error + +def caller()[ raises TypeError]: + rethrow(TypeError()) diff --git a/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__inferred_return_type_that_never_settles.snap b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__inferred_return_type_that_never_settles.snap new file mode 100644 index 0000000000..7a5cad24a2 --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__inferred_return_type_that_never_settles.snap @@ -0,0 +1,7 @@ +--- +source: crates/ty_ide/src/inlay_hints.rs +expression: "test.inlay_hints_with_settings(&InlayHintSettings\n{ inferred_return_types: true, ..InlayHintSettings::none() })" +--- + +def f(): + return f() diff --git a/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hint_above_the_requested_range.snap b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hint_above_the_requested_range.snap new file mode 100644 index 0000000000..7c73702cde --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hint_above_the_requested_range.snap @@ -0,0 +1,8 @@ +--- +source: crates/ty_ide/src/inlay_hints.rs +expression: "test.inlay_hints_with_settings(&InlayHintSettings\n{ property_types: true, ..InlayHintSettings::none() })" +--- + +class A: + let a[: 1] + get() = 1 diff --git a/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hint_can_be_turned_off.snap b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hint_can_be_turned_off.snap new file mode 100644 index 0000000000..147e038913 --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hint_can_be_turned_off.snap @@ -0,0 +1,8 @@ +--- +source: crates/ty_ide/src/inlay_hints.rs +expression: "test.inlay_hints_with_settings(&InlayHintSettings\n{ property_types: false, ..InlayHintSettings::default() })" +--- + +class A: + let a + get() = 1 diff --git a/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hint_in_a_python_file.snap b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hint_in_a_python_file.snap new file mode 100644 index 0000000000..56167d4ef7 --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hint_in_a_python_file.snap @@ -0,0 +1,8 @@ +--- +source: crates/ty_ide/src/inlay_hints.rs +expression: "test.inlay_hints_with_settings(&InlayHintSettings\n{ property_types: true, ..InlayHintSettings::none() })" +--- + +class A: + let a + get() = 1 diff --git a/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hint_outside_the_requested_range.snap b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hint_outside_the_requested_range.snap new file mode 100644 index 0000000000..56167d4ef7 --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hint_outside_the_requested_range.snap @@ -0,0 +1,8 @@ +--- +source: crates/ty_ide/src/inlay_hints.rs +expression: "test.inlay_hints_with_settings(&InlayHintSettings\n{ property_types: true, ..InlayHintSettings::none() })" +--- + +class A: + let a + get() = 1 diff --git a/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hint_shapes.snap b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hint_shapes.snap new file mode 100644 index 0000000000..02459394a5 --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hint_shapes.snap @@ -0,0 +1,14 @@ +--- +source: crates/ty_ide/src/inlay_hints.rs +expression: "test.inlay_hints_with_settings(&InlayHintSettings\n{ property_types: true, ..InlayHintSettings::none() })" +--- + +class A: + private let b[: "b"] + get() = 'b' + + static let c[: 3] + get() = 3 + + let n[: None] + get(): pass diff --git a/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hint_without_a_settled_type.snap b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hint_without_a_settled_type.snap new file mode 100644 index 0000000000..84bbaff9bd --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hint_without_a_settled_type.snap @@ -0,0 +1,11 @@ +--- +source: crates/ty_ide/src/inlay_hints.rs +expression: "test.inlay_hints_with_settings(&InlayHintSettings\n{ property_types: true, ..InlayHintSettings::none() })" +--- + +class A: + let rec + get() = self.rec + + let m + get() diff --git a/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hints.snap b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hints.snap new file mode 100644 index 0000000000..9762f5ce47 --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__property_type_hints.snap @@ -0,0 +1,18 @@ +--- +source: crates/ty_ide/src/inlay_hints.rs +expression: "test.inlay_hints_with_settings(&InlayHintSettings\n{ property_types: true, ..InlayHintSettings::none() })" +--- + +class A: + let a[: 1] + get() = 1 + + let b[: str] + field = 'two' + + var c[: int] = 0 + get() = field + set(value): field = value + + let d: bytes + get() = b'' diff --git a/crates/ty_ide/src/symbols.rs b/crates/ty_ide/src/symbols.rs index eb03666a39..09dcfc2d77 100644 --- a/crates/ty_ide/src/symbols.rs +++ b/crates/ty_ide/src/symbols.rs @@ -1310,7 +1310,7 @@ impl<'db> SymbolVisitor<'db> { fn visit_stmt_impl(&mut self, stmt: &'db ast::Stmt) { match stmt { ast::Stmt::FunctionDef(func_def) => { - if let Some(construct) = property_construct(func_def) { + if let Some(construct) = func_def.property_construct_range() { self.property_construct = Some(construct); self.add_symbol(SymbolTree { parent: None, @@ -1564,22 +1564,6 @@ impl<'db> SymbolVisitor<'db> { } } -/// basedpython: the range of the property construct `func` was synthesized -/// from, if it is that construct's getter. -/// -/// The parser lowers `var x: int` plus its accessor blocks into a getter, an -/// optional backing declaration and an optional setter, and marks the getter -/// with a synthetic decorator spanning the whole construct. All three are -/// ranged inside that span, and the source spells one member. -fn property_construct(func: &ast::StmtFunctionDef) -> Option { - func.decorator_list.iter().find_map(|decorator| { - let ast::Expr::Name(marker) = &decorator.expression else { - return None; - }; - matches!(marker.id.as_str(), "__property__" | "__static_property__").then(|| marker.range()) - }) -} - impl<'db> SourceOrderVisitor<'db> for SymbolVisitor<'db> { fn visit_stmt(&mut self, stmt: &'db ast::Stmt) { // basedpython: everything else the property construct was lowered into diff --git a/crates/ty_python_core/src/re_exports.rs b/crates/ty_python_core/src/re_exports.rs index 04ae261750..57ff6b3715 100644 --- a/crates/ty_python_core/src/re_exports.rs +++ b/crates/ty_python_core/src/re_exports.rs @@ -27,7 +27,7 @@ use ruff_python_ast::{ name::Name, visitor::{Visitor, walk_expr, walk_pattern, walk_stmt}, }; -use rustc_hash::FxHashMap; +use rustc_hash::{FxHashMap, FxHashSet}; use ty_module_resolver::{ImportingFile, resolve_module_for_import_from}; use crate::{Db, ProgramFile}; @@ -55,6 +55,10 @@ struct ExportFinder<'db> { program_file: ProgramFile<'db>, visiting_stub_file: bool, exports: FxHashMap<&'db Name, PossibleExportKind>, + /// basedpython: the names the module declares `private`. the lowering renames + /// them with a leading underscore, so python's own rule leaves them out of a + /// `from m import *` exactly as it does a name written with one + private: FxHashSet<&'db Name>, dunder_all: DunderAll, } @@ -65,6 +69,7 @@ impl<'db> ExportFinder<'db> { program_file: file, visiting_stub_file: file.file(db).is_stub(db), exports: FxHashMap::default(), + private: FxHashSet::default(), dunder_all: DunderAll::NotPresent, } } @@ -86,7 +91,7 @@ impl<'db> ExportFinder<'db> { if kind == PossibleExportKind::StubImportWithoutRedundantAlias { return None; } - if name.starts_with('_') { + if name.starts_with('_') || self.private.contains(&name) { return None; } Some(name.clone()) @@ -186,6 +191,9 @@ impl<'db> Visitor<'db> for ExportFinder<'db> { range: _, node_index: _, }) => { + if has_private_modifier(decorator_list) { + self.private.insert(&name.id); + } self.possibly_add_export(&name.id, PossibleExportKind::Normal); for decorator in decorator_list { self.visit_decorator(decorator); @@ -209,6 +217,9 @@ impl<'db> Visitor<'db> for ExportFinder<'db> { is_trailing_lambda: _, is_asserts_return: _, }) => { + if has_private_modifier(decorator_list) { + self.private.insert(&name.id); + } self.possibly_add_export(&name.id, PossibleExportKind::Normal); for decorator in decorator_list { self.visit_decorator(decorator); @@ -232,6 +243,12 @@ impl<'db> Visitor<'db> for ExportFinder<'db> { node_index: _, decorator_list: _, }) => { + if ruff_python_ast::helpers::declaration_marker_visibility(annotation) + == ruff_python_ast::helpers::MemberVisibility::Private + && let ast::Expr::Name(target_name) = target.as_ref() + { + self.private.insert(&target_name.id); + } if value.is_some() || self.visiting_stub_file { self.visit_expr(target); } @@ -248,8 +265,11 @@ impl<'db> Visitor<'db> for ExportFinder<'db> { cases: _, range: _, node_index: _, - is_private: _, + is_private, }) => { + if *is_private && let ast::Expr::Name(alias_name) = name.as_ref() { + self.private.insert(&alias_name.id); + } self.visit_expr(name); // Neither walrus expressions nor statements cannot appear in type aliases; // no need to recursively visit the `value` or `type_params` @@ -460,3 +480,15 @@ enum DunderAll { NotPresent, Present, } + +/// basedpython: whether a `def` or `class` carries the synthetic decorator the +/// `private` keyword parses to. a real `@private` decorator is an ordinary name +fn has_private_modifier(decorators: &[ast::Decorator]) -> bool { + decorators.iter().any(|decorator| { + matches!( + &decorator.expression, + ast::Expr::Name(name) + if name.ctx == ast::ExprContext::Invalid && name.id.as_str() == "private" + ) + }) +} diff --git a/crates/ty_python_semantic/resources/lint_docs/missing-function-body.md b/crates/ty_python_semantic/resources/lint_docs/missing-function-body.md new file mode 100644 index 0000000000..94fd4d94c6 --- /dev/null +++ b/crates/ty_python_semantic/resources/lint_docs/missing-function-body.md @@ -0,0 +1,38 @@ +## What it does + +Checks for a `def` written with no body at all, in a position that needs an implementation. + +## Why is this bad? + +A `def` with no body declares a signature. The lowering fills in `: ...`, so the function exists and +returns `None`. That is what a declaration means in a stub file; anywhere else it is an +implementation that was never written, silently stood in for by one that does nothing. + +A body may be left out where a declaration is what the position asks for: + +- in a stub file +- in an `if TYPE_CHECKING` block +- as a member of a protocol class +- as an `abstract def`, or an `@abstractmethod`-decorated method +- as an overload declaration, written `@overload` or as a run of same-name `def`s + +An `init(...)` may also be written without a body: what it does is store the attribute parameters it +declares, and that body is built for it. + +## Examples + +```by +def parse(s: str) -> int # ok: the run below makes this an overload declaration +def parse(s: bytes) -> int +def parse(s): + return int(s) + +# error: [missing-function-body] +def lookup() -> int +``` + +A function that is meant to do nothing says so with a body of its own: + +```by +def ignore(event: str): ... +``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_decorator_keyword.md b/crates/ty_python_semantic/resources/mdtest/basedpython_decorator_keyword.md index 3ca87f3884..1dd7e25e0d 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_decorator_keyword.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_decorator_keyword.md @@ -60,7 +60,7 @@ parameters of a lambda passed to `d` directly, and it is not particular to `deco `bidirectional.md` ```by -decorator def d(fn: (int) -> None) +decorator def d(fn: (int) -> None): ... @d def f(i): diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_empty_declarations.md b/crates/ty_python_semantic/resources/mdtest/basedpython_empty_declarations.md new file mode 100644 index 0000000000..4326086e25 --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_empty_declarations.md @@ -0,0 +1,206 @@ +# Empty declarations + +basedpython lets a `class` or a `def` be written with no body at all. An empty class is a whole +class — nothing about it is left out. A `def` with no body is a declaration: the lowering fills the +body in with `: ...`, so what runs is a function that returns `None`. + +A stub file, a protocol, an `abstract def`, an overload group and an `if TYPE_CHECKING` block each +ask for exactly that. Anywhere else the implementation the signature promises was never written, and +`missing-function-body` reports it. + +## A `def` with no body + +At module scope, in a class, nested in another function, and in a loop body — a `def` written there +is one the program is going to run: + +```by +# error: [missing-function-body] +def lookup() -> int + +class C: + # error: [missing-function-body] + def m(self) -> int + +def outer() -> int: + # error: [missing-function-body] + def inner() -> int + + return 1 + +for _ in range(3): + # error: [missing-function-body] + def each() -> int +``` + +## The return type makes no difference + +The body is what went missing, whether or not `None` would have satisfied what the signature says +the function returns. A `def` that declares no return type at all is the case that says nothing +about returning: + +```by +# error: [missing-function-body] +def a() + +# error: [missing-function-body] +def b() -> int +``` + +## An empty class body + +A class with no body is a class with no members, which is a whole class. Nothing is reported: + +```by +class Empty + +class Sub(Empty) + +reveal_type(Sub()) # revealed: final Sub +``` + +## A stub file declares + +```byi +def f() -> int + +class C: + def m(self) -> str +``` + +## A protocol member declares + +A protocol says what its implementations provide, so a member is a signature and nothing else: + +```by +from typing import Protocol + +class P(Protocol): + def m(self) -> int + +protocol Q: + def m(self) -> int + +def use(p: P, q: Q) -> int: + return p.m() + q.m() +``` + +## A class that inherits from a protocol does not declare + +Only a class that inherits from `Protocol` directly is a protocol, so a subclass of one needs bodies +like any other class: + +```by +from typing import Protocol + +class P(Protocol): + def m(self) -> int + +class Impl(P): + # error: [missing-function-body] + def m(self) -> int +``` + +## An abstract method declares + +An `abstract def` is given a `raise NotImplementedError` body, and an `@abstractmethod` a `: ...` +one. Either way the method exists to be overridden: + +```by +from abc import ABC, abstractmethod + +class A(ABC): + abstract def m(self) -> int + + @abstractmethod + def n(self) -> int + +class B(A): + def m(self) -> int: + return 1 + + def n(self) -> int: + return 2 +``` + +## An overload declaration declares + +A run of same-name bodyless `def`s is an overload group: the lowering writes the `@overload` +decorators the source leaves out, and the implementation is the one that carries a body. + +```by +def parse(s: str) -> int +def parse(s: bytes) -> int +def parse(s): + return int(s) + +reveal_type(parse("1")) # revealed: int +``` + +A written `@overload` declares the same way: + +```by +from typing import overload + +@overload +def parse(s: str) -> int +@overload +def parse(s: bytes) -> int +def parse(s): + return int(s) +``` + +## An `if TYPE_CHECKING` block declares + +Nothing in such a block runs, so there is nothing for a body to do: + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + def declared() -> int +``` + +## `empty-body` is about a body that is there + +A body written as `: ...` is a body: the function is meant to do nothing, and only its return type +can be wrong about that. So the two reports never both apply to one `def`: + +```by +# error: [empty-body] +def written() -> int: ... + +# error: [missing-function-body] +def unwritten() -> int +``` + +## A `decorator def` needs a body too + +The lowering writes a `decorator def` a dispatcher, which calls the body the source is supposed to +supply — so the missing one is missing there as well: + +```by +# error: [missing-function-body] +decorator def route(fn: (int) -> None) +``` + +In a stub file it declares the decorator's shape, like any other declaration: + +```byi +decorator def route(fn: (int) -> None) +``` + +## `init(...)` writes its own body + +An `init(...)` has its body built from its parameter list — the attribute parameters it declares are +the whole of what it does — so there is never one left out: + +```by +class Point: + init(let x: int, let y: int) + +class Nothing: + init() + +reveal_type(Point(1, 2).x) # revealed: int +reveal_type(Nothing()) # revealed: final Nothing +``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_exceptions.md b/crates/ty_python_semantic/resources/mdtest/basedpython_exceptions.md index 463f28610c..f6fafc92d8 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_exceptions.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_exceptions.md @@ -232,6 +232,202 @@ def f() raises int: return ``` +## a type parameter in a clause must be bounded by an exception + +A type parameter stands for one type the caller chooses, so `raises T` says something about +exceptions only when every type `T` can be is an exception. That is what its declaration says: a +parameter with no bound at all can be `int` as easily as `OSError`. + +```by +# error: [invalid-raises-clause] "`T@f` is not always an exception, so it cannot appear in a `raises` clause" +def f[T](value: T) raises T: + return +``` + +A set of constraints has to be all exceptions, since the caller can pick any one of them. + +```by +# error: [invalid-raises-clause] "`T@g` is not always an exception, so it cannot appear in a `raises` clause" +def g[T in (int, KeyError)](value: T) raises T: + return +``` + +An upper bound of `BaseException`, or of any exception below it, is enough, and so is a set of +constraints that are all exceptions. + +```by +def bounded[T: OSError](value: T) raises T: + return + +def constrained[T in (KeyError, IndexError)](value: T) raises T: + return +``` + +Only a member of the set itself has to be an exception. A type parameter inside one is a type +argument of an exception class, and `E[int]` is as much an exception as `E[OSError]`. + +```by +class E[X](Exception): ... + +def wrapped[X](value: X) raises E[X]: + return +``` + +## a call reads a clause's type parameter as what it solved + +The callee writes its exception set in terms of its own type parameters, so what escapes a +particular call is that set with what the call solved them to. `raise_it(TypeError())` raises a +`TypeError`, and nothing else. + +```by +def raise_it[T: BaseException](error: T) raises T: + raise error + +def caller() raises TypeError: + raise_it(TypeError()) + +def wrong() raises ValueError: + # error: [undeclared-raise] "`wrong` can raise `TypeError`, which its `raises` clause does not include" + raise_it(TypeError()) +``` + +A body with no clause of its own is read the same way, since the set recovered from it is in the +same type parameters. + +```by +def undeclared[T: BaseException](error: T): + raise error + +def calls_undeclared() raises KeyError: + undeclared(KeyError()) +``` + +The caller's own type parameters survive: a call that solves the callee's `T` to the caller's `U` +raises `U`, which is exactly what the caller declared. + +```by +def forwards[U: BaseException](error: U) raises U: + raise_it(error) +``` + +## a recursive call raises what it solves the parameter to + +A call back into the same function is read the same way. When it solves the type parameter to +something else, it raises that something else: `f(KeyError(), False)` raises a `KeyError` whatever +the outer call was made with. + +```by +def f[T: BaseException](error: T, again: bool) raises T: + if again: + # error: [undeclared-raise] "`f` can raise `KeyError`, which its `raises` clause does not include" + f(KeyError(), False) + raise error +``` + +A body with no clause gets the same answer, and passes it on to its callers. + +```by +def g[T: BaseException](error: T, again: bool): + if again: + g(KeyError(), False) + raise error + +def caller() raises ValueError: + # error: [undeclared-raise] "`caller` can raise `KeyError`, which its `raises` clause does not include" + g(ValueError(), True) +``` + +## a closure names its enclosing function's type parameter + +A function nested in a generic one can raise a value of the enclosing function's type parameter. +Where the closure is called, that parameter is still in scope and still means what the enclosing +function was called with, so it stays as it is. + +```by +def outer[T: BaseException](error: T) raises T: + def inner(): + raise error + inner() +``` + +## a class's type parameter in a clause is read through the receiver + +A method may name its class's type parameter, and the receiver is what says which exception that is. +It is found wherever in the receiver's ancestry the method was declared, and a call through the +class itself reads it the same way. + +```by +class Reader[T: BaseException]: + def read(self) raises T: + return + +class FileReader(Reader[OSError]): + pass + +def read_one(reader: Reader[KeyError]) raises KeyError: + reader.read() + +def read_file(reader: FileReader) raises OSError: + reader.read() + +def read_unbound(reader: Reader[KeyError]) raises KeyError: + Reader[KeyError].read(reader) +``` + +A classmethod has no instance at all, and the class it is called through says which exception it is. + +```by +class Source[T: BaseException]: + @classmethod + def open(cls) raises T: + return + +def open_one() raises KeyError: + Source[KeyError].open() +``` + +## an explicit specialization is read like a solved call + +Specializing a function explicitly names its type parameter where a call would otherwise solve it, +and the call raises what was named. + +```by +def raise_os[T: OSError](error: T) raises T: + raise error + +def explicit() raises FileNotFoundError: + raise_os[FileNotFoundError](FileNotFoundError()) + +def wrong() raises PermissionError: + # error: [undeclared-raise] "`wrong` can raise `FileNotFoundError`, which its `raises` clause does not include" + raise_os[FileNotFoundError](FileNotFoundError()) +``` + +The specialization belongs to the function's type, so it is still there when that is held in a +variable, and a caller can specialize with its own type parameter. + +```by +def aliased() raises FileNotFoundError: + raise_found = raise_os[FileNotFoundError] + raise_found(FileNotFoundError()) + +def forwards[U: OSError](error: U) raises U: + raise_os[U](error) +``` + +## a call that does not bind raises nothing known + +A call that does not type-check solves nothing, so its callee's type parameter says nothing about +what it raises. The call is already reported, and the exception analysis adds nothing to it. + +```by +def raise_it[T: BaseException](error: T) raises T: + raise error + +def caller() raises Never: + raise_it(1) # error: [invalid-argument-type] +``` + ## `assert` raises `AssertionError` ```by diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_init_method.md b/crates/ty_python_semantic/resources/mdtest/basedpython_init_method.md index 9b4d27402f..bde4de3668 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_init_method.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_init_method.md @@ -139,19 +139,19 @@ class B: reveal_type(self.a) # revealed: int ``` -## `private` name-mangles the attribute +## a visibility keyword decides who may reach the attribute -A `private let` / `private var` parameter self-assigns to the name-mangled `self.__name`. The -parameter itself keeps its declared name, so the constructor signature is unchanged. +A `private let` / `private var` parameter declares an attribute the class's own body may reach and +nothing else. The parameter itself keeps its declared name, so the constructor signature is +unchanged, and the attribute is reached by the same name from inside the class. ```by class A: init(self, private var a: int): - reveal_type(self.__a) # revealed: int + reveal_type(self.a) # revealed: int x = A(1) -# the public name is not an attribute — it is name-mangled -x.a # error: [unresolved-attribute] +x.a # error: [inaccessible-member] ``` ## a modifier chain may precede `init` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_method_modifiers.md b/crates/ty_python_semantic/resources/mdtest/basedpython_method_modifiers.md index c3289c3af0..b36bbdd175 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_method_modifiers.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_method_modifiers.md @@ -12,11 +12,11 @@ python-version = "3.12" ## a method modifier needs a class body ```by -class def make() # error: [invalid-syntax] "`class` is only a modifier on a method" +class def make(): ... # error: [invalid-syntax] "`class` is only a modifier on a method" -static def helper() # error: [invalid-syntax] "`static` is only a modifier on a method" +static def helper(): ... # error: [invalid-syntax] "`static` is only a modifier on a method" -override def replace() # error: [invalid-syntax] "`override` is only a modifier on a method" +override def replace(): ... # error: [invalid-syntax] "`override` is only a modifier on a method" ``` ## a function nested in a method is not itself a method @@ -26,7 +26,7 @@ the class owns the method, not the functions the method makes ```by class A: def run(self): - static def inner() # error: [invalid-syntax] "`static` is only a modifier on a method" + static def inner(): ... # error: [invalid-syntax] "`static` is only a modifier on a method" ``` ## a modifier that reads on a class too is left alone diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_override_raise.md b/crates/ty_python_semantic/resources/mdtest/basedpython_override_raise.md index b97d097841..85e1d0aea0 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_override_raise.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_override_raise.md @@ -132,6 +132,31 @@ class C(B): raise TypeError ``` +## a generic base bounds its overrides as the subclass specializes it + +A base method's clause can name the base's type parameter. What bounds an override is that clause as +the subclass specializes the base: `FileReader(Reader[OSError])` inherits a `read` that raises +`OSError`, and a subclass passing its own `U` inherits one that raises `U`. + +```by +class Reader[T: BaseException]: + def read(self) raises T: + return + +class FileReader(Reader[OSError]): + override def read(self) raises OSError: + return + +class Forwarding[U: BaseException](Reader[U]): + override def read(self) raises U: + return + +class Wider(Reader[OSError]): + # error: [override-raise] "`read` can raise `ValueError`, which the method it overrides cannot" + override def read(self) raises OSError | ValueError: + return +``` + ## a constructor is not checked `ty` exempts constructors from override compatibility, and this follows that. diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_properties.md b/crates/ty_python_semantic/resources/mdtest/basedpython_properties.md index 269b9032be..b74e83b3a9 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_properties.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_properties.md @@ -79,6 +79,64 @@ reveal_type(p.name) # revealed: str p.name = "bob" ``` +## an untyped property takes its type from its initialiser + +a declaration that names no type but has an initialiser is the typed declaration with the type left +for the initialiser to say, the way any declaration's is: `var count = 0` is `var count: int = 0`. +the setter accepts that type and the property reads as it + +```by +class Counter: + var count = 0 + get() = field + set(value): + reveal_type(value) # revealed: int + field = value + +c = Counter() +reveal_type(c.count) # revealed: int +# error: [invalid-assignment] +c.count = "many" +``` + +## a getter is held to the type its initialiser declares + +as it would be to a written type, so a getter that returns something else is an error rather than a +second type for the property to read as + +```by +class Counter: + var count = 0 + # error: [invalid-return-type] + get() = str(field) + set(value): + field = value +``` + +## without an initialiser a property takes its type from the getter + +only a `let` can leave both the type and the initialiser out. what `get` returns is the property's +type, as precisely as the accessor states it + +```by +class A: + let a + get() = 1 + +reveal_type(A().a) # revealed: 1 +``` + +## a property declared `None` has no return annotation to remove + +the declared type rides on the getter as its return annotation, but it is the property's type. +writing it is not the redundant `-> None` of a `def` that falls off its end + +```by +class A: + let n: None + get(): pass +``` + ## a computed property has no backing storage An accessor block that never mentions `field` allocates no backing field — the property is computed @@ -95,6 +153,19 @@ r = Rect() reveal_type(r.area) # revealed: int ``` +## a computed property takes no initialiser + +an initialiser is stored in the backing field, and a property whose accessors never mention `field` +has none to store it in + +```by +class Rect: + var w: int = 0 + # error: [invalid-syntax] "a property with no backing `field` takes no initialiser" + let area: int = 0 + get() = self.w * 2 +``` + ## an explicit `field` declaration decouples storage from the public type ```by @@ -163,6 +234,20 @@ class A: reveal_type(self.a) # revealed: int ``` +## an explicit `field` initialiser types the storage, not the property + +only an initialiser written on the declaration itself states the property's type. one on an explicit +`field` states the storage's, so an untyped property still takes its type from the getter + +```by +class Name: + let length + field = "ada" + get() = len(field) + +reveal_type(Name().length) # revealed: int +``` + ## the property's type is the context an unannotated `field` is solved against An initialiser that carries no type information of its own — a bare `[]` — is solved against the @@ -253,11 +338,10 @@ a.bump() assert a.age == 1 ``` -## a `private` property is not reachable under its public name +## a `private` property is not reachable from outside its class -`private` emits the property one underscore deeper (`_x`, storage `__x`), so it simply does not -exist under the name the author wrote. That makes privacy self-enforcing: an access from outside is -an unresolved attribute rather than something needing its own check. +`private` renames a property the way it renames any member, so an access from outside the class is +reported: the property is emitted as `__x`, which Python name-mangles per class. ```by class A: @@ -272,7 +356,7 @@ class A: a = A() a.bump() -# error: [unresolved-attribute] +# error: [inaccessible-member] print(a.x) ``` @@ -287,13 +371,13 @@ class A: class B(A): def f(self): - # error: [unresolved-attribute] + # error: [inaccessible-member] return self.x ``` ## a write to a `private` property still runs the setter -The redirect targets the property, not its storage, so validation is not bypassed. +A write inside the class goes through the property, not its storage, so validation is not bypassed. ```by class A: @@ -329,6 +413,24 @@ a.x = 3 reveal_type(a.x) # revealed: int ``` +## a suite on the accessor's line runs every statement + +a suite may share the accessor's line, as it may a `def`'s. every statement in it runs, not only the +first + +```by +class Account: + var balance: int = 0 + get() = field + set(value): checked = max(value, 0); field = checked + +a = Account() +a.balance = -5 +assert a.balance == 0 +a.balance = 7 +assert a.balance == 7 +``` + ## accessor bodies are type-checked The getter's body is a real method body, so a return that contradicts the declared property type is diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_safe_variance.md b/crates/ty_python_semantic/resources/mdtest/basedpython_safe_variance.md index 75c623b0bb..e35b659510 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_safe_variance.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_safe_variance.md @@ -42,6 +42,21 @@ class A[T]: print(a1.t) ``` +## a protected member is not erased + +A `protected` member is reachable from a subclass's body, where the receiver can be any +specialization of the class rather than only `Self`. It tells specializations apart like a public +member does, so a read through a specialization picks up its argument. + +```by +class A[T]: + protected t: T + + def f(self): + a1 = A[int]() + reveal_type(a1.t) # revealed: int +``` + ## …which is what keeps a contravariant read honest `A[object]` is an `A[int]` under `in T`, so an `A[int]` may really be holding a `str`. The erased @@ -225,9 +240,9 @@ argument leaves a callback nothing can be passed to. class A[T]: private def consume(self, t: T): ... -def f(a: A[int]): - # error: [invalid-argument-type] "Argument to bound method `A.consume` is incorrect: Expected `Never`, found `1`" - a.consume(1) + def g(self, other: A[int]): + # error: [invalid-argument-type] "Argument to bound method `A.consume` is incorrect: Expected `Never`, found `1`" + other.consume(1) ``` ## a private producer stays callable through a widened view @@ -237,8 +252,8 @@ class A[T]: private def produce(self) -> T: raise NotImplementedError -def f(a: A[int]): - reveal_type(a.produce()) # revealed: object + def g(self, other: A[int]): + reveal_type(other.produce()) # revealed: object ``` ## a `__getattr__` result is not a declared member @@ -348,14 +363,18 @@ class A[T]: ## a subclass that stays generic carries the constraint forward +The class's own body can hold a widened view of a subclass, and the private member is erased through +it the same way. + ```by class A[T]: private t: T -class C[U](A[U]): def f(self, other: C[object]): # error: [invalid-assignment] "Object of type `1` is not assignable to attribute `t` of type `Never`" other.t = 1 + +class C[U](A[U]): ... ``` ## a nested occurrence is erased soundly @@ -368,9 +387,9 @@ the erased element stays gradual and a write is neither precise nor rejected. class A[T]: private items: list[T] -def f(a: A[int]): - reveal_type(a.items) # revealed: list[*] - a.items = [1] + def g(self, other: A[int]): + reveal_type(other.items) # revealed: list[*] + other.items = [1] ``` ## a union of two specializations is a widened view of each @@ -386,7 +405,7 @@ class A[out T]: def produce(self) -> T: raise NotImplementedError -def f(a: A[int] | A[str]): - # error: [invalid-assignment] "Object of type `1` is not assignable to attribute `t` on type `A[int] | A[str]`" - a.t = 1 + def f(self, a: A[int] | A[str]): + # error: [invalid-assignment] "Object of type `1` is not assignable to attribute `t` on type `A[int] | A[str]`" + a.t = 1 ``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_visibility.md b/crates/ty_python_semantic/resources/mdtest/basedpython_visibility.md index ac5bc36c56..300dd130cd 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_visibility.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_visibility.md @@ -1,14 +1,20 @@ # basedpython: visibility modifiers -`private` and `export`/`public` are transpile-time visibility modifiers: `export`/`public` add the -symbol to the module's generated `__all__`, and `private` renames it with an underscore prefix (or, -inside a class body, name-mangles it with `__`). they carry no type-level effect — the decorated -class or function keeps its ordinary type rather than being erased to `Unknown`. +`export`/`public` add a module-level symbol to the generated `__all__`. `private` and `protected` +say who may reach a member: `private` only the declaring class's own body, `protected` a subclass's +body as well. neither carries a type-level effect — the declaration keeps its ordinary type rather +than being erased to `Unknown`. + +the lowering spells the answer in the member's name, because that is the only enforcement the +runtime offers: `private` renames to `__name`, which python name-mangles per class, and `protected` +to `_name`, python's own convention for "not part of the interface". the type checker enforces both +directly, so neither spelling has to be relied on. a dunder is the exception: python looks one up by its exact name, and mangles only names with at most one trailing underscore, so renaming would change what the method *is* rather than who can -reach it. `private` on one is therefore reported as having no effect — except on `__init__`, the one -dunder where it says something, which is checked at the construction site instead. +reach it. a visibility keyword on one is therefore reported as having no effect — except on +`__init__`, the one dunder where it says something, which is checked at the construction site +instead. ## a private class keeps its type @@ -327,3 +333,725 @@ from deco import helper reveal_type(helper()) # revealed: int ``` + +## a private member is reachable from its own class + +the class's own body reaches a private member by the name it was written with; the lowering is what +spells out the mangled one. + +```by +class Account: + private balance: int = 0 + + init(private let owner: str) + + private def audit(self) -> str: + return f"{self.owner}:{self.balance}" + + def report(self) -> str: + return self.audit() + +reveal_type(Account("a").report()) # revealed: str +``` + +## a private member is not reachable from a subclass + +python mangles `__name` with the name of the class whose body it is written in, so a subclass's body +names a different attribute entirely. that is what `private` means, and it is refused rather than +silently renamed to reach across. + +```by +class Base: + private secret: int = 1 + +class Derived(Base): + def leak(self) -> int: + return self.secret # error: [inaccessible-member] +``` + +## a private member is not reachable from outside any class + +```by +class Base: + private secret: int = 1 + +def read(b: Base) -> int: + return b.secret # error: [inaccessible-member] +``` + +## a protected member is reachable from a subclass + +`protected` is the visibility `private` is usually mistaken for: the declaring class and everything +that inherits from it. + +```by +class Base: + protected step: int = 2 + + init(protected let limit: int) + +class Derived(Base): + def describe(self) -> str: + return f"{self.step}/{self.limit}" + +reveal_type(Derived(4).describe()) # revealed: str +``` + +## a protected member is not reachable from outside the hierarchy + +```by +class Base: + protected step: int = 2 + +def read(b: Base) -> int: + return b.step # error: [inaccessible-member] +``` + +## a member is not reachable through a write either + +the boundary is about the member, not about which direction it is used in. + +```by +class Base: + private count: int = 0 + +def bump(b: Base) -> None: + b.count = 1 # error: [inaccessible-member] +``` + +## a plain underscore name is left to convention + +`_name` and `__name` written out are what python itself offers, and they mean whatever the author +meant by them. only a visibility keyword makes the boundary something to enforce. + +```by +class Base: + _step: int = 2 + +def read(b: Base) -> int: + return b._step +``` + +## declaring a member less visible than the one it inherits + +a visibility keyword decides the name the member is emitted under, so a member declared less visible +than the one it inherits does not override it — it sits beside it under a different name, and the +inherited one still answers. + +```by +class A: + def f(self) -> int: + return 1 + +class B(A): + # error: [invalid-override-visibility] + private def f(self) -> int: + return 2 +``` + +## the narrowing is reported whether or not `override` is written + +writing `override` as well states the opposite of what `private` does, but the declaration is wrong +either way, so the report does not depend on it. + +```by +class A: + def f(self) -> int: + return 1 + +class B(A): + # error: [invalid-override-visibility] + private override def f(self) -> int: + return 2 +``` + +## `protected` over `public` narrows too + +```by +class A: + def f(self) -> int: + return 1 + +class B(A): + # error: [invalid-override-visibility] + protected def f(self) -> int: + return 2 +``` + +## widening is not narrowing + +a subclass is free to declare a member of its own under a name a base kept private: the two are +different attributes, and the public one is new rather than a replacement. + +```by +class A: + private def f(self) -> int: + return 1 + +class B(A): + def f(self) -> int: + return 2 + +reveal_type(B().f()) # revealed: int +``` + +## two private members of the same name are unrelated + +each is mangled with the name of the class that declares it, so neither overrides the other and +neither has to match the other's signature. + +```by +class A: + private def helper(self, n: int) -> int: + return n + +class B(A): + private def helper(self, s: str) -> str: + return s + + def use(self) -> str: + return self.helper("x") + +reveal_type(B().use()) # revealed: str +``` + +## a protected member overrides like any other + +`protected` keeps one name across the hierarchy, so a subclass's declaration really does replace the +one it inherits, and has to be substitutable for it. + +```by +class A: + protected def f(self, n: int) -> int: + return n + +class B(A): + # error: [invalid-method-override] + protected override def f(self, n: str) -> int: + return len(n) +``` + +## `protected` is only a modifier on a class member + +outside a class body there is nothing for the "and its subclasses" half of `protected` to mean. + +```by +protected def helper() -> int: # error: [invalid-syntax] + return 1 +``` + +## a protected constructor may be called by a subclass + +`private init` says the class decides how its instances are made. `protected init` extends that to +the classes that inherit it, which is what a base class does when only its subclasses should +construct it — a subclass's body may construct itself, or the base. + +```by +class Shape: + protected init(let sides: int) + +class Square(Shape): + @classmethod + def make(cls) -> Square: + return Square(4) + + @classmethod + def base(cls) -> Shape: + return Shape(4) + +reveal_type(Square.make().sides) # revealed: int +reveal_type(Square.base().sides) # revealed: int +``` + +## a protected constructor is still refused outside the hierarchy + +```by +class Shape: + protected init(let sides: int) + +Shape(3) # error: [private-constructor] +``` + +## a visibility keyword composes with a class variable + +`class var`, `class let` and `class x = v` take no other modifier, since their own keyword fills the +one slot a declaration has. A visibility keyword is the exception: it says who may reach the +variable, which composes with any declaration, and the class reaches it through the class object as +readily as through an instance. + +```by +class Counter: + private class var made: int = 0 + protected class let LIMIT: int = 3 + private class hits = 0 + + @classmethod + def total(cls) -> int: + return cls.made + cls.LIMIT + cls.hits + +reveal_type(Counter.total()) # revealed: int +``` + +## a private class variable is not reachable through the class from outside + +```by +class Counter: + private class var made: int = 0 + +Counter.made # error: [inaccessible-member] +``` + +## a module-level private variable cannot be imported + +A variable is renamed like a function, class or type alias is, so it is taken off the module's +interface the same way. + +`helpers.by`: + +```by +private count: int = 0 +private total = 0 +``` + +`main.by`: + +```by +from helpers import count # error: [private-import] "`count` is private to `helpers`" +from helpers import total # error: [private-import] "`total` is private to `helpers`" +``` + +## a module's private symbol is not reachable as an attribute of the module + +Reaching the symbol through the module object crosses the same boundary an import does, and the +lowering has renamed it, so the attribute is not there at runtime either. + +`helpers.by`: + +```by +private count: int = 0 + +private def secret() -> int: + return 1 +``` + +`main.by`: + +```by +import helpers + +# error: [inaccessible-member] "`count` is private to module `helpers`" +helpers.count +# error: [inaccessible-member] "`secret` is private to module `helpers`" +helpers.secret() +``` + +## a module's own code reaches its private variable + +```by +private count: int = 0 + +def bump() -> int: + global count + count += 1 + return count + +reveal_type(bump()) # revealed: int +``` + +## a bare name in the class body reaches a restricted member + +The class body names its own members without a receiver, and the lowering renames those names along +with the declarations. + +```by +class Counter: + private start = 1 + step = start + 1 + + private def helper(self) -> int: + return self.step + + alias = helper + +reveal_type(Counter.step) # revealed: int +``` + +## a declaration inside a compound statement in the class body + +A declaration written under an `if` or a `try` is as much the class's member as one written at the +top of the body. + +```by +import sys + +class A: + if sys.version_info >= (3, 8): + private x: int = 1 + +A().x # error: [inaccessible-member] +``` + +## a decorated private method is still private + +The method's visibility is read off its declaration, not off whatever type a decorator turns it +into. + +```by +import functools + +class A: + @functools.cache + private def cached(self) -> int: + return 1 + + def use(self) -> None: + self.cached() + +def outside(a: A) -> None: + a.cached() # error: [inaccessible-member] +``` + +## a private nested class + +```by +class A: + private class Inner: ... + + def make(self) -> None: + self.Inner() + +A.Inner # error: [inaccessible-member] +``` + +## a protected member reached through a union + +Every class a union can be declares the member the same way, so it has one name to be reached by. + +```by +class A: + protected x: int = 1 + +class B(A): ... + +class C(A): + def other(self, o: B | C) -> int: + return o.x + +reveal_type(C().other(B())) # revealed: int +``` + +## a protected member one class of a union overrides + +`B` overrides `x` and `C` inherits it from `A`, but both emit it as `_x`, so the union has one name +to reach it by. Each class's declaration is still checked, and `D` is a subclass of both `B` and +`A`. + +```by +class A: + protected def x(self) -> int: + return 1 + +class B(A): + protected override def x(self) -> int: + return 2 + +class C(A): ... + +class D(B): + def other(self, o: B | C) -> int: + return o.x() + +reveal_type(D().other(C())) # revealed: int +``` + +## an override of a protected member is protected to the overriding class + +The member `o.x` reaches is `B`'s, and `C` is not a subclass of `B`. + +```by +class A: + protected def x(self) -> int: + return 1 + +class B(A): + protected override def x(self) -> int: + return 2 + +class C(A): + def other(self, o: B) -> int: + # error: [inaccessible-member] "`x` is protected: only `B` and its subclasses may reach it" + return o.x() +``` + +## a union whose classes restrict a member differently + +`A` emits `x` as `_x` and `D` emits it as `x`, so no one attribute access reaches both. + +```by +class A: + protected x: int = 1 + + def read(self, o: A | D) -> None: + # error: [inaccessible-member] "`x` is emitted under a different name on the classes `A | D` can be" + o.x + +class D: + x: int = 2 +``` + +## `super()` reaches a protected member + +```by +class A: + protected class var x: int = 1 + +class B(A): + def read(self) -> int: + return super().x + +reveal_type(B().read()) # revealed: int +``` + +## `__slots__` names a private member by its name + +Python mangles a `__slots__` entry the way it mangles the class body's own names, so the string is +renamed with the declaration. + +```by +class A: + __slots__ = ("x",) + + init(private let x: int) + + def get(self) -> int: + return self.x + +reveal_type(A(1).get()) # revealed: int +``` + +## a class pattern's keyword reads the member it names + +`case A(x=1)` reads `x` off the subject, so it is renamed and checked like `a.x`. + +```by +class A: + __match_args__ = ("x",) + + init(private let x: int) + + def matches(self) -> bool: + match self: + case A(x=1): + return True + return False + +reveal_type(A(1).matches()) # revealed: bool +``` + +## a class pattern's keyword is checked like an access + +```by +class A: + init(private let x: int) + +def f(a: A) -> None: + match a: + # error: [inaccessible-member] + case A(x=1): + pass +``` + +## `protected` is an ordinary name in python + +The keyword only means something where a modifier chain is written, so everywhere else it is a name +like any other. + +```py +protected = [1] +protected.append(2) +``` + +## a visibility keyword on a local is an error + +A declaration in a function body is a local, which nothing outside the function reaches anyway. + +```by +def f() -> int: + private x = 1 # error: [invalid-syntax] + return x +``` + +## widening a protected member is not an override + +`A.f` is emitted as `_f` and `B.f` as `f`, so `B.f` sits beside the inherited method rather than +replacing it. + +```by +class A: + protected def f(self) -> int: + return 1 + +class B(A): + # error: [invalid-override-visibility] + def f(self) -> int: + return 2 +``` + +## a dataclass field cannot be private + +The field's name is its constructor's keyword. + +```by +from dataclasses import dataclass + +@dataclass +class Point: + # error: [invalid-visibility] + private x: int +``` + +## a named tuple's field cannot be protected + +```by +from typing import NamedTuple + +class P(NamedTuple): + # error: [invalid-visibility] + protected x: int +``` + +## a typed dict's key cannot be private + +```by +from typing import TypedDict + +class TD(TypedDict): + # error: [invalid-visibility] + private x: int +``` + +## an enum member cannot be private + +```by +from enum import Enum + +class E(Enum): + # error: [invalid-visibility] + private A = 1 +``` + +## an abstract method cannot be private + +A `private` method is renamed per class, so no subclass could ever override it. + +```by +from abc import ABC, abstractmethod + +class A(ABC): + @abstractmethod + # error: [invalid-visibility] + private def f(self) -> int: ... +``` + +## `protected` on a name python already mangles + +```by +class A: + # error: [ineffective-private] + protected __x: int = 1 +``` + +## `private` in a class named only with underscores + +Python mangles nothing in such a class, so `private` would hide nothing. + +```by +class __: + # error: [ineffective-private] + private x: int = 1 +``` + +## a module-level private variable declared under an `if` + +```by +import sys + +if sys.version_info >= (3, 8): + private flag = True + +def read() -> bool: + return flag + +reveal_type(read()) # revealed: bool +``` + +## a class body reads the module's private variable before binding its own + +Python reads a class's own binding only once the body has made it, so the right-hand side here is +the module's `count`. + +```by +private count = 1 + +class H: + count = count + 1 + +reveal_type(H.count) # revealed: int +``` + +## a class body that may or may not have bound the name + +The read is the class's own `count` on one path and the module's private one on the other, and no +single emitted name reads both. + +```by +import random + +private count = 1 + +class H: + if random.random() > 0.5: + count = 2 + # error: [invalid-visibility] + x = count +``` + +## a star import leaves private symbols out + +The lowering renames a private symbol with a leading underscore, so python's own rule leaves it out +of `from helpers import *`. + +`helpers.by`: + +```by +private hidden = 1 +visible = 2 +``` + +`main.by`: + +```by +from helpers import * + +reveal_type(visible) # revealed: 2 +hidden # error: [unresolved-reference] +``` + +## a private symbol cannot be listed in `__all__` + +```by +private def helper() -> int: + return 1 + +__all__ = ["helper"] # error: [private-export] +``` + +## a dotted import cannot rebind a private symbol + +`import os.path` binds the package `os`, which no alias can keep under the underscored name. + +```by +private os = 1 + +import os.path # error: [invalid-visibility] +``` diff --git a/crates/ty_python_semantic/resources/mdtest/function/return_type.md b/crates/ty_python_semantic/resources/mdtest/function/return_type.md index 9b879f731f..4cd18a49ce 100644 --- a/crates/ty_python_semantic/resources/mdtest/function/return_type.md +++ b/crates/ty_python_semantic/resources/mdtest/function/return_type.md @@ -126,61 +126,16 @@ def f(x: int | str): return x ``` -### A basedpython bodyless `def` +### A basedpython `def` with no body -basedpython lets a `def` be written with no body at all, which the lowering fills in with `: ...`. -That is the same empty body written a shorter way, so it is permissible in the same places and -reported everywhere else. +basedpython lets a `def` be written with no body at all. There is no body to check a return type +against, so this rule has nothing to say about one: where a declaration is not what the position +asks for, the missing body is reported instead, by `missing-function-body` — see +`basedpython_empty_declarations.md`. ```by -def implicitly_returns_none() - -# error: [empty-body] -def f() -> int - -class C: - # error: [empty-body] - def m(self) -> int -``` - -### A bodyless `def` in an implicit overload run - -A run of same-name bodyless `def`s is an overload group — the lowering writes the `@overload` -decorators the source leaves out — so its members are stubs like any other overload. - -```by -def parse(s: str) -> int -def parse(s: bytes) -> int -def parse(s): - return int(s) -``` - -### A bodyless `def` a `Protocol` or an abstract class declares - -```by -from abc import ABC, abstractmethod -from typing import Protocol - -class P(Protocol): - def m(self) -> int - -protocol Q: - def m(self) -> int - -class A(ABC): - @abstractmethod - def m(self) -> int - - abstract def n(self) -> int -``` - -### A bodyless `def` in a stub file - -```byi +# error: [missing-function-body] def f() -> int - -class C: - def m(self) -> str ``` ### In `if TYPE_CHECKING` block diff --git a/crates/ty_python_semantic/src/place.rs b/crates/ty_python_semantic/src/place.rs index 2d75a51128..e9d4573ca6 100644 --- a/crates/ty_python_semantic/src/place.rs +++ b/crates/ty_python_semantic/src/place.rs @@ -24,6 +24,8 @@ use crate::types::{ may_exist_at_runtime, }; use crate::{Db, FxIndexSet, FxOrderSet}; +use ruff_db::parsed::{ParsedModuleRef, parsed_module}; +use ruff_text_size::Ranged; use ty_python_core::definition::{Definition, DefinitionKind, DefinitionState}; use ty_python_core::narrowing_constraints::ScopedNarrowingConstraint; use ty_python_core::place::{PlaceExpr, ScopedPlaceId}; @@ -104,11 +106,24 @@ impl PublicTypePolicy { ) -> Type<'db> { match self { Self::Raw => ty, - Self::Promote => ty.promote(db, env).promote_singletons(db, env), + Self::Promote => promote_undeclared(db, env, ty), } } } +/// the type an undeclared place exposes publicly: what an instance reads back off a class-body +/// `x = 0` is `int`, not the literal the assignment stores +/// +/// basedpython: this is also the type an untyped declaration declares, wherever something else +/// is held to it — see `OverloadLiteral::property_initialiser_type` +pub(crate) fn promote_undeclared<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + ty: Type<'db>, +) -> Type<'db> { + ty.promote(db, env).promote_singletons(db, env) +} + /// The source definition provenance for a place. #[derive( Debug, Clone, Copy, Default, Hash, PartialEq, Eq, get_size2::GetSize, salsa::SalsaValue, @@ -1083,6 +1098,26 @@ impl<'db> PlaceAndQualifiers<'db> { } } + /// basedpython: `Some(…)` if the place carries a visibility qualifier and no + /// type of its own — `private x = 1`, whose declaration says who may reach + /// the member and nothing else. + fn is_bare_visibility(&self) -> Option { + match self { + PlaceAndQualifiers { place, qualifiers } + if qualifiers.intersects(TypeQualifiers::PRIVATE | TypeQualifiers::PROTECTED) + // a bare `ClassVar` has its own rule below, which keeps every + // qualifier the declaration carries + && !qualifiers.contains(TypeQualifiers::CLASS_VAR) + && place + .ignore_possibly_undefined() + .is_some_and(|ty| ty.is_unknown()) => + { + Some(*qualifiers) + } + _ => None, + } + } + #[must_use] pub(crate) fn map_type( self, @@ -1282,6 +1317,19 @@ pub(crate) fn place_by_id<'db>( .with_qualifiers(qualifiers); } + // basedpython: a visibility keyword on an assignment states who may reach the + // member and nothing about its type, so what the right-hand side infers is + // what the declaration means — the same reasoning as the bare `Final` above. + // the member stays writable, so what it exposes is the promoted view an + // undeclared assignment exposes, not the literal the binding inferred + if let Some(qualifiers) = declared.is_bare_visibility() { + let bindings = all_considered_bindings(); + return place_from_bindings_impl(db, &env, bindings, requires_explicit_reexport, None) + .place + .with_public_type_policy(PublicTypePolicy::Promote) + .with_qualifiers(qualifiers); + } + // basedpython: under `sound-types` a bare `ClassVar` uses the inferred type directly, the // same way an unannotated class-body assignment already does. Otherwise adding `ClassVar` — // a strengthening of intent — would degrade `x = 1` from `int` to `Unknown | Literal[1]` @@ -1705,6 +1753,213 @@ fn declared_place(place: PlaceAndQualifiers<'_>) -> SourceDeclaration<'_> { } } +/// basedpython: whether the name `expr` loads, read in an annotation, is a forward reference — +/// a name the program binds, but that python does not find bound when the annotation is +/// evaluated where it is written. +/// +/// ty checks every annotation in a basedpython file as deferred, so a name in one resolves +/// against every binding its scope makes: `def f() -> Later` is fine with `class Later` further +/// down. Before 3.14 python evaluates annotations as the definition runs, and the same `Later` +/// raises `NameError` there, so this is how the transpiler learns which annotations it has to +/// defer. A binding made only under `if TYPE_CHECKING:` never runs, so a name only it binds is +/// a forward reference too. +/// +/// A name no binding in the program supplies is not one: it is either a name basedpython +/// resolves itself (`dynamic`, an implicit `Any`), which the transpiler makes available, or an +/// error the checker reports already. A name unbound on only some paths is one, since deferring +/// it costs nothing. `None` when the index never saw `expr`, as for a name inside a string +/// annotation, which is deferred already. +pub(crate) fn is_forward_reference<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: ProgramFile<'db>, + expr: ast::ExprRef<'_>, +) -> Option { + let index = semantic_index(db, file); + let scope = index.try_expression_scope_id(&expr)?.to_scope_id(db, file); + index.try_use_id(expr)?; + let module = parsed_module(db, file.python_file(db)).load(db); + + let mut resolution = resolve_place_load( + db, + index, + scope, + PlaceExpr::try_from_expr(expr)?, + PlaceLoadMode::AtExpression(expr), + ); + let bound_where_written = loop { + match resolution.next() { + Some(PlaceLoadResolutionStep::Source(source)) => { + match runtime_definedness(db, env, index, &module, source) { + RuntimeDefinedness::Defined => break true, + RuntimeDefinedness::PossiblyUndefined => break false, + RuntimeDefinedness::Undefined => {} + } + } + Some(PlaceLoadResolutionStep::MemberResolutionCondition(_)) => {} + Some(PlaceLoadResolutionStep::Exhausted(_)) | None => break false, + } + }; + if bound_where_written { + return Some(false); + } + + // resolved as the checker resolves it, against every binding its scope makes: a binding + // there means the program supplies the name, only not by the point the annotation runs + let mut resolution = resolve_place_load( + db, + index, + scope, + PlaceExpr::try_from_expr(expr)?, + PlaceLoadMode::Deferred, + ); + loop { + match resolution.next() { + Some(PlaceLoadResolutionStep::Source(source)) => { + if source.is_post_lexical() { + return Some(false); + } + if lexical_binding_exists(db, source) { + return Some(true); + } + } + Some(PlaceLoadResolutionStep::MemberResolutionCondition(_)) => {} + Some(PlaceLoadResolutionStep::Exhausted(_)) | None => return Some(false), + } + } +} + +/// Whether a lexical source of a place load holds any definition at all, run or not. +fn lexical_binding_exists<'db>(db: &'db dyn Db, source: PlaceLoadSource<'db>) -> bool { + let any_defined = |bindings: BindingWithConstraintsIterator<'_, 'db>| { + bindings + .into_iter() + .any(|binding| matches!(binding.binding, DefinitionState::Defined(_))) + }; + match source.kind { + PlaceLoadSourceKind::Bindings(bindings) => any_defined(bindings), + PlaceLoadSourceKind::DefinitionsFromOwningScope { scope, id } => { + any_defined(use_def_map(db, scope).end_of_scope_bindings(id)) + } + PlaceLoadSourceKind::Implicit(ImplicitPlaceLoad::ExplicitGlobalSymbol { file, name }) => { + let global = global_scope(db, file); + place_table(db, global) + .symbol_id(&name) + .is_some_and(|symbol| { + any_defined(use_def_map(db, global).end_of_scope_symbol_bindings(symbol)) + }) + } + PlaceLoadSourceKind::Observed(_) | PlaceLoadSourceKind::Implicit(_) => false, + } +} + +/// Whether one source of a place load holds a value once the program is running. +enum RuntimeDefinedness { + Defined, + /// the source holds a value on some paths to the load and not on others + PossiblyUndefined, + /// the source holds no value here, so resolution moves on to the next one + Undefined, +} + +fn runtime_definedness<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + index: &'db SemanticIndex<'db>, + module: &ParsedModuleRef, + source: PlaceLoadSource<'db>, +) -> RuntimeDefinedness { + match source.kind { + PlaceLoadSourceKind::Bindings(bindings) => { + bindings_runtime_definedness(db, index, module, bindings) + } + // a lazy scope reads the owner's value when it runs, by which point the owner has made + // every binding it is going to + PlaceLoadSourceKind::DefinitionsFromOwningScope { scope, id } => { + bindings_runtime_definedness( + db, + index, + module, + use_def_map(db, scope).end_of_scope_bindings(id), + ) + } + PlaceLoadSourceKind::Implicit(ImplicitPlaceLoad::ExplicitGlobalSymbol { file, name }) => { + let global = global_scope(db, file); + match place_table(db, global).symbol_id(&name) { + Some(symbol) => bindings_runtime_definedness( + db, + index, + module, + use_def_map(db, global).end_of_scope_symbol_bindings(symbol), + ), + None => RuntimeDefinedness::Undefined, + } + } + PlaceLoadSourceKind::Implicit(ImplicitPlaceLoad::Builtin(name)) => { + defined_if(implicit_builtins_symbol(db, env, &name)) + } + PlaceLoadSourceKind::Implicit(ImplicitPlaceLoad::ClassBodySymbol(name)) => { + defined_if(class_body_implicit_symbol(db, env, &name)) + } + PlaceLoadSourceKind::Implicit(ImplicitPlaceLoad::ModuleImplicitGlobal { file, name }) => { + defined_if(module_type_implicit_global_symbol(db, file, &name)) + } + PlaceLoadSourceKind::Observed(_) + | PlaceLoadSourceKind::Implicit(ImplicitPlaceLoad::DunderClass(_)) => { + RuntimeDefinedness::Defined + } + } +} + +/// an implicit source holds a value only where its lookup finds one +fn defined_if(place: PlaceAndQualifiers<'_>) -> RuntimeDefinedness { + if place.place.is_definitely_bound() { + RuntimeDefinedness::Defined + } else { + RuntimeDefinedness::Undefined + } +} + +fn bindings_runtime_definedness<'db>( + db: &'db dyn Db, + index: &'db SemanticIndex<'db>, + module: &ParsedModuleRef, + bindings: BindingWithConstraintsIterator<'_, 'db>, +) -> RuntimeDefinedness { + let predicates = bindings.predicates(); + let reachability_constraints = bindings.reachability_constraints(); + let (mut defined, mut undefined) = (false, false); + for binding in bindings { + if evaluate_reachability_with_cache( + db, + None, + reachability_constraints, + predicates, + binding.reachability_constraint, + ) + .is_always_false() + { + continue; + } + match binding.binding { + DefinitionState::Defined(definition) + if !index.is_in_type_checking_block( + definition.file_scope(db), + definition.full_range(db, module).range(), + ) => + { + defined = true; + } + _ => undefined = true, + } + } + match (defined, undefined) { + (true, false) => RuntimeDefinedness::Defined, + (true, true) => RuntimeDefinedness::PossiblyUndefined, + (false, _) => RuntimeDefinedness::Undefined, + } +} + /// The type declared for the place a binding writes to. /// /// The binding may carry the annotation itself, as `a: int = 1` does, or a separate statement may diff --git a/crates/ty_python_semantic/src/reified.rs b/crates/ty_python_semantic/src/reified.rs index 00ec3545c0..f385f17047 100644 --- a/crates/ty_python_semantic/src/reified.rs +++ b/crates/ty_python_semantic/src/reified.rs @@ -377,7 +377,7 @@ pub fn inferred_reified_class_type_param_names( /// /// `None` when the function is called without one: a `staticmethod`, or a /// signature whose parameters are all keyword-only or variadic -fn method_receiver(function: &ast::StmtFunctionDef) -> Option<&str> { +pub(crate) fn method_receiver(function: &ast::StmtFunctionDef) -> Option<&str> { if function .decorator_list .iter() diff --git a/crates/ty_python_semantic/src/semantic_model.rs b/crates/ty_python_semantic/src/semantic_model.rs index 161ca28435..02e669e1e8 100644 --- a/crates/ty_python_semantic/src/semantic_model.rs +++ b/crates/ty_python_semantic/src/semantic_model.rs @@ -100,17 +100,62 @@ impl<'db> SemanticModel<'db> { /// altogether — so the mangled name is written out in full instead, which /// reads the same from everywhere. /// - /// `None` when the attribute is not a private method. - pub fn private_method_name(&self, attribute: &ast::ExprAttribute) -> Option { - crate::types::visibility::private_method_name( + /// `None` when the attribute is not a member a visibility keyword renamed. + pub fn restricted_member_name(&self, attribute: &ast::ExprAttribute) -> Option { + crate::types::visibility::restricted_member_name( self.db, &self.program_environment(), attribute.value.inferred_type(self)?, - attribute.inferred_type(self)?, attribute.attr.as_str(), ) } + /// basedpython: see `crate::types::visibility::class_member_spellings` + pub fn class_member_spellings( + &self, + class: &ast::StmtClassDef, + name: &str, + ) -> Option<(String, String)> { + crate::types::visibility::class_member_spellings(self.db, self.file, class, name) + } + + /// basedpython: the name a class pattern's keyword (`case A(x=...)`) is emitted + /// under — the attribute it reads, spelled so it is reached from anywhere + pub fn class_pattern_keyword_name( + &self, + pattern: &ast::PatternMatchClass, + keyword: &str, + ) -> Option { + crate::types::visibility::restricted_member_name( + self.db, + &self.program_environment(), + pattern.cls.inferred_type(self)?, + keyword, + ) + } + + /// basedpython: see `crate::types::visibility::class_body_member_name` + pub fn class_body_member_name(&self, name: &ast::ExprName) -> Option { + crate::types::visibility::class_body_member_name(self.db, self.file, name) + } + + /// basedpython: see `crate::types::visibility::resolves_to_module_scope` + pub fn resolves_to_module_scope(&self, reference: &ast::ExprName) -> Option { + crate::types::visibility::resolves_to_module_scope(self.db, self.file, reference) + } + + /// basedpython: the module-level names this file declares `private` — the + /// set `private-import` reads, and whose references the lowering renames + pub fn private_module_symbols(&self) -> Vec { + let mut names: Vec = + crate::types::visibility::private_symbols(self.db, self.file.file(self.db)) + .iter() + .cloned() + .collect(); + names.sort_unstable_by(|a, b| a.as_str().cmp(b.as_str())); + names + } + /// basedpython: how many entries `cls`'s `__match_args__` has, which is what /// places the subpatterns a class pattern writes after its `*_` — the last of /// them names the last entry, whatever the count turns out to be. @@ -1646,6 +1691,19 @@ impl<'db> SemanticModel<'db> { ) } + /// basedpython: whether `name`, read in an annotation, is a forward reference: a name the + /// program binds, but not by the point python evaluates the annotation where it is written, + /// as it does before 3.14. a binding made only under `if TYPE_CHECKING:` does not count. + /// `None` when ty did not index the name + pub fn is_forward_reference(&self, name: &ast::ExprName) -> Option { + crate::place::is_forward_reference( + self.db, + &self.program_environment(), + self.program_file(), + ast::ExprRef::from(name), + ) + } + /// Returns the scope in which `node` is defined (handles string annotations). pub fn scope(&self, node: ast::AnyNodeRef<'_>) -> Option { let index = semantic_index(self.db, self.program_file()); diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 5942bef818..e918e6ca2d 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -17,7 +17,7 @@ use ruff_db::Instant; use ruff_db::diagnostic::{Annotation, Diagnostic, Span}; use ruff_db::parsed::parsed_module; use ruff_python_ast as ast; -use ruff_python_ast::helpers::TypeModifier; +use ruff_python_ast::helpers::{MemberVisibility, TypeModifier}; use ruff_python_ast::name::Name; use ruff_text_size::Ranged; use smallvec::smallvec_inline; @@ -12273,6 +12273,10 @@ bitflags! { /// A private member is invisible to a widened view of its class, which is what /// makes a mutable field under a covariant type parameter sound. const PRIVATE = 1 << 8; + /// A non-standard type qualifier for the basedpython `protected` member keyword. + /// A protected member is reachable from the declaring class's body and from a + /// subclass's, and from nowhere else. + const PROTECTED = 1 << 9; } } @@ -12307,40 +12311,80 @@ impl TypeQualifiers { pub fn is_non_standard(self) -> bool { const NON_STANDARD: TypeQualifiers = TypeQualifiers::IMPLICIT_INSTANCE_ATTRIBUTE .union(TypeQualifiers::FROM_MODULE_GETATTR) - .union(TypeQualifiers::PRIVATE); + .union(TypeQualifiers::PRIVATE) + .union(TypeQualifiers::PROTECTED); self.intersects(NON_STANDARD) } } -/// basedpython: whether the class member `name`, declared with `qualifiers` and of type -/// `ty`, is *private* — invisible to any observer outside the class. +/// basedpython: the visibility a member's declaration states outright, `None` when it +/// states none and the member's name is all there is to go on. /// -/// Privacy is what makes variance safe: an invisible member cannot be used to tell two -/// specializations of its class apart, so it neither constrains the class's variance nor -/// may be reached through a widened view of it. A dunder is *not* private — it is part of -/// the public protocol surface. -pub(crate) fn is_private_member<'db>( +/// This is the question a *rename* asks. A member whose name already says how visible it +/// is keeps the name its author wrote; only a modifier keyword asks for a new one. +fn declared_visibility<'db>( db: &'db dyn Db, - name: &str, qualifiers: TypeQualifiers, ty: Type<'db>, -) -> bool { - if qualifiers.contains(TypeQualifiers::PRIVATE) - || matches!(NameKind::classify(name), NameKind::Sunder) - { - return true; +) -> Option { + if qualifiers.contains(TypeQualifiers::PRIVATE) { + return Some(MemberVisibility::Private); + } + if qualifiers.contains(TypeQualifiers::PROTECTED) { + return Some(MemberVisibility::Protected); } // a `private def` carries no qualifier: the keyword parses as a synthetic decorator, - // so its privacy is recorded on the function rather than on the declaration. reached + // so its visibility is recorded on the function rather than on the declaration. reached // off an instance the member is already bound, so unwrap that too - match ty { - Type::FunctionLiteral(function) => { - function.has_known_decorator(db, FunctionDecorators::PRIVATE) - } - Type::BoundMethod(method) => method - .function(db) - .has_known_decorator(db, FunctionDecorators::PRIVATE), - _ => false, + let function = match ty { + Type::FunctionLiteral(function) => Some(function), + Type::BoundMethod(method) => Some(method.function(db)), + _ => None, + }?; + if function.has_known_decorator(db, FunctionDecorators::PRIVATE) { + Some(MemberVisibility::Private) + } else if function.has_known_decorator(db, FunctionDecorators::PROTECTED) { + Some(MemberVisibility::Protected) + } else { + None + } +} + +/// basedpython: the visibility a member's name states, which is all python itself has. +/// +/// A dunder is *not* private: it is part of the public protocol surface, and python does +/// not mangle it either. +fn visibility_from_name(name: &str) -> MemberVisibility { + match NameKind::classify(name) { + NameKind::Dunder | NameKind::Normal => MemberVisibility::Public, + // `__name` is the spelling python itself mangles, so it is private in + // every dialect; a single underscore is the convention for the rest + NameKind::Sunder if name.starts_with("__") => MemberVisibility::Private, + NameKind::Sunder => MemberVisibility::Protected, + } +} + +/// basedpython: whether the class member `name` is invisible to a *widened view* of its +/// class, which is what makes variance safe: such a member cannot be used to tell two +/// specializations of its class apart, so it neither constrains the class's variance nor may +/// be reached through a widened view of it +/// +/// a `private` member qualifies, however it is spelled. a `protected` one does not: it is +/// reachable from a subclass's body, where the receiver can be any specialization of the class +/// rather than only `Self`. a name spelled with a leading underscore and no keyword keeps the +/// treatment python's convention always had here +pub(crate) fn is_private_member<'db>( + db: &'db dyn Db, + name: &str, + qualifiers: TypeQualifiers, + ty: Type<'db>, +) -> bool { + match declared_visibility(db, qualifiers, ty) { + Some(MemberVisibility::Private) => true, + Some(MemberVisibility::Protected) => false, + Some(MemberVisibility::Public) | None => { + visibility_from_name(name) != MemberVisibility::Public + } } } diff --git a/crates/ty_python_semantic/src/types/bound_super.rs b/crates/ty_python_semantic/src/types/bound_super.rs index 0df09508ab..66ced38cad 100644 --- a/crates/ty_python_semantic/src/types/bound_super.rs +++ b/crates/ty_python_semantic/src/types/bound_super.rs @@ -951,6 +951,20 @@ impl<'db> BoundSuperType<'db> { /// /// If the pivot class is a dynamic type, its MRO can't be determined, /// so we fall back to using the MRO of `DynamicType::Unknown`. + /// basedpython: the classes an attribute read through this `super()` object is + /// looked up in — the owner's MRO after the pivot. `None` when the owner is not + /// a resolved class, which leaves nothing to search + pub(super) fn lookup_mro_after_pivot( + self, + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + ) -> Option>> { + let SuperOwnerKind::Resolved(resolved_owner) = self.owner(db) else { + return None; + }; + Some(self.skip_until_after_pivot(db, env, resolved_owner.lookup_anchor.iter_mro(db))) + } + fn skip_until_after_pivot( self, db: &'db dyn Db, diff --git a/crates/ty_python_semantic/src/types/class/static_literal.rs b/crates/ty_python_semantic/src/types/class/static_literal.rs index 7f8b9841e5..81315e6197 100644 --- a/crates/ty_python_semantic/src/types/class/static_literal.rs +++ b/crates/ty_python_semantic/src/types/class/static_literal.rs @@ -7,9 +7,10 @@ use ruff_db::{ parsed::{ParsedModuleRef, parsed_module}, }; use ruff_python_ast as ast; +use ruff_python_ast::statement_visitor::{StatementVisitor, walk_stmt}; use ruff_python_ast::{PythonVersion, name::Name}; use ruff_text_size::{Ranged, TextRange}; -use rustc_hash::FxHashSet; +use rustc_hash::{FxHashMap, FxHashSet}; use std::cell::RefCell; use super::implicit_attributes::implicit_attribute_names; @@ -622,6 +623,27 @@ impl<'db> StaticClassLiteral<'db> { index.expect_single_definition(body_scope.node(db).expect_class()) } + /// basedpython: the members this class's own body declares with a visibility + /// keyword, and the visibility each carries — a `private def`, a `protected x: + /// T`, a `private class`, an `init(private let x)` parameter's attribute, + /// wherever in the body the declaration is written + /// + /// read off the declarations rather than the members' types, so a decorated + /// method, a nested class and a property answer the same as a plain attribute. + /// tracked because it reads the class's AST node + #[salsa::tracked(returns(ref), heap_size = ruff_memory_usage::heap_size)] + pub(crate) fn member_visibilities( + self, + db: &'db dyn Db, + ) -> FxHashMap { + let module = parsed_module(db, self.program_file(db).python_file(db)).load(db); + let mut collector = MemberVisibilityCollector::default(); + collector.visit_body(&self.node(db, &module).body); + let mut members = collector.members; + members.shrink_to_fit(); + members + } + /// basedpython: the names a class body *declares without a value* — an /// annotation with no assignment (`label: str`). /// @@ -4517,3 +4539,78 @@ fn annotated_field_specifier<'db>( } }) } + +/// basedpython: collects [`StaticClassLiteral::member_visibilities`] from a class +/// body +#[derive(Default)] +struct MemberVisibilityCollector { + members: FxHashMap, + /// set inside a method's body, where the only members declared are the + /// attributes an `init(...)` parameter stands for (`self.x`) + in_method: bool, +} + +impl<'a> StatementVisitor<'a> for MemberVisibilityCollector { + fn visit_stmt(&mut self, stmt: &'a ast::Stmt) { + match stmt { + ast::Stmt::FunctionDef(function) => { + // a function nested in a method declares no member + if self.in_method { + return; + } + if let Some(visibility) = visibility_modifier(&function.decorator_list) { + self.members.insert(function.name.id.clone(), visibility); + } + self.in_method = true; + self.visit_body(&function.body); + self.in_method = false; + } + // a nested class is a member, but its own members are its own + ast::Stmt::ClassDef(class) => { + if !self.in_method + && let Some(visibility) = visibility_modifier(&class.decorator_list) + { + self.members.insert(class.name.id.clone(), visibility); + } + } + ast::Stmt::AnnAssign(assign) => { + let visibility = + ruff_python_ast::helpers::declaration_marker_visibility(&assign.annotation); + if visibility == ruff_python_ast::helpers::MemberVisibility::Public { + return; + } + match (assign.target.as_ref(), self.in_method) { + (ast::Expr::Name(name), false) => { + self.members.insert(name.id.clone(), visibility); + } + (ast::Expr::Attribute(attribute), true) => { + self.members.insert(attribute.attr.id.clone(), visibility); + } + _ => {} + } + } + // a compound statement's bodies are still the class's body + _ => walk_stmt(self, stmt), + } + } +} + +/// basedpython: the visibility a `def` or `class` declares through the synthetic +/// decorator its `private` / `protected` keyword parses to. a real `@private` +/// decorator is an ordinary name, not the modifier +fn visibility_modifier( + decorators: &[ast::Decorator], +) -> Option { + decorators + .iter() + .find_map(|decorator| match &decorator.expression { + ast::Expr::Name(name) if name.ctx == ast::ExprContext::Invalid => { + match name.id.as_str() { + "private" => Some(ruff_python_ast::helpers::MemberVisibility::Private), + "protected" => Some(ruff_python_ast::helpers::MemberVisibility::Protected), + _ => None, + } + } + _ => None, + }) +} diff --git a/crates/ty_python_semantic/src/types/context_params.rs b/crates/ty_python_semantic/src/types/context_params.rs index aecfa3ade1..79ec4d02d4 100644 --- a/crates/ty_python_semantic/src/types/context_params.rs +++ b/crates/ty_python_semantic/src/types/context_params.rs @@ -201,6 +201,12 @@ pub struct ImplicitContextArgument { /// lowering gives the receiver a name of its own, so the transpiler must /// write that rather than `self` pub is_block_receiver: bool, + /// whether the binding is a module-level `private` variable. The lowering + /// emits it under an underscored name, so the transpiler must write that + /// rather than the name the source spells. A nearer binding that merely + /// shares the name is not this one, which is why the question is answered + /// off the binding rather than the name + pub is_module_private: bool, } /// the implicit arguments the transpiler must append to `call`: for each @@ -274,11 +280,19 @@ pub fn implicit_context_arguments<'db>( )), CandidateBinding::BlockArgument(_) | CandidateBinding::BlockReceiver(_) => None, }; + let is_module_private = match binding { + CandidateBinding::Written(definition) => { + definition.scope(db).file_scope_id(db).is_global() + && crate::types::visibility::private_symbols(db, file).contains(&variable) + } + CandidateBinding::BlockArgument(_) | CandidateBinding::BlockReceiver(_) => false, + }; implicit.push(ImplicitContextArgument { parameter: name.clone(), variable, declaration, is_block_receiver: matches!(binding, CandidateBinding::BlockReceiver(_)), + is_module_private, }); } } diff --git a/crates/ty_python_semantic/src/types/diagnostic.rs b/crates/ty_python_semantic/src/types/diagnostic.rs index 06d00896e6..d10f530c85 100644 --- a/crates/ty_python_semantic/src/types/diagnostic.rs +++ b/crates/ty_python_semantic/src/types/diagnostic.rs @@ -50,6 +50,7 @@ use ruff_db::{ parsed::parsed_module, }; use ruff_diagnostics::{Edit, Fix, IsolationLevel}; +use ruff_python_ast::helpers::MemberVisibility; use ruff_python_ast::name::Name; use ruff_python_ast::token::parentheses_iterator; use ruff_python_ast::{self as ast, AnyNodeRef, HasNodeIndex, StringFlags}; @@ -139,6 +140,7 @@ pub(crate) fn register_lints(registry: &mut LintRegistryBuilder) { registry.register_lint(&UNBOUND_TYPE_VARIABLE); registry.register_lint(&MISSING_ARGUMENT); registry.register_lint(&MISSING_DIRECT_DEPENDENCY); + registry.register_lint(&MISSING_FUNCTION_BODY); registry.register_lint(&MISSING_TYPE_ARGUMENT); registry.register_lint(&NO_MATCHING_OVERLOAD); registry.register_lint(&NON_CALLABLE_INIT_SUBCLASS); @@ -156,6 +158,10 @@ pub(crate) fn register_lints(registry: &mut LintRegistryBuilder) { registry.register_lint(&PRIVATE_CONSTRUCTOR); registry.register_lint(&INEFFECTIVE_PRIVATE); registry.register_lint(&PRIVATE_IMPORT); + registry.register_lint(&INACCESSIBLE_MEMBER); + registry.register_lint(&INVALID_OVERRIDE_VISIBILITY); + registry.register_lint(&INVALID_VISIBILITY); + registry.register_lint(&PRIVATE_EXPORT); registry.register_lint(&INVALID_EXTENSION); registry.register_lint(&AMBIGUOUS_EXTENSION_MEMBER); registry.register_lint(&INVALID_CONFORMANCE); @@ -562,6 +568,16 @@ declare_lint! { } } +declare_lint! { + #[doc = include_str!("../../resources/lint_docs/missing-function-body.md")] + pub(crate) static MISSING_FUNCTION_BODY = { + summary: "detects a `def` written with no body in a position that needs an implementation", + status: LintStatus::stable("0.0.81"), + default_level: Level::Error, + ty_compat: TyCompat::BasedPython, + } +} + declare_lint! { #[doc = include_str!("../../resources/lint_docs/implicit-declaration.md")] pub(crate) static IMPLICIT_DECLARATION = { @@ -1182,13 +1198,136 @@ declare_lint! { /// return "Point()" /// ``` pub(crate) static INEFFECTIVE_PRIVATE = { - summary: "detects a `private` modifier on a name it cannot hide", + summary: "detects a visibility keyword on a name it cannot act on", status: LintStatus::stable("0.0.79"), default_level: Level::Error, ty_compat: TyCompat::BasedPython, } } +declare_lint! { + /// ## What it does + /// Checks for a visibility keyword written where it cannot do what it says. + /// + /// ## Why is this bad? + /// A visibility keyword renames the member or symbol it is written on, since + /// that is the only enforcement python offers. Some names cannot be renamed + /// without changing what they mean: a dataclass field's name is its + /// constructor's keyword, an enum member's name is how the enum is looked up, + /// a `private` abstract method can never be overridden, and a dotted import + /// binds a package no rename can keep. The declaration would read as + /// restricted while doing something else. + /// + /// ## Example + /// + /// ```by + /// from dataclasses import dataclass + /// + /// @dataclass + /// class Point: + /// private x: int # error: a dataclass field's name is its constructor's keyword + /// ``` + pub(crate) static INVALID_VISIBILITY = { + summary: "detects a visibility keyword where it cannot do what it says", + status: LintStatus::stable("0.0.80"), + default_level: Level::Error, + ty_compat: TyCompat::BasedPython, + } +} + +declare_lint! { + /// ## What it does + /// Checks for a `private` symbol listed in the module's `__all__`. + /// + /// ## Why is this bad? + /// `__all__` lists the module's interface, and a `private` symbol is declared + /// not to be part of it. The lowering renames the symbol with a leading + /// underscore, so `from m import *` would look up a name the module does not + /// have, and raise. + /// + /// ## Example + /// + /// ```by + /// private def helper() -> int: + /// return 1 + /// + /// __all__ = ["helper"] # error: `helper` is private + /// ``` + pub(crate) static PRIVATE_EXPORT = { + summary: "detects a `private` symbol listed in `__all__`", + status: LintStatus::stable("0.0.80"), + default_level: Level::Error, + ty_compat: TyCompat::BasedPython, + } +} + +declare_lint! { + /// ## What it does + /// Checks for reads and writes of a `private` or `protected` class member + /// from outside the code allowed to reach it, and for a module's `private` + /// symbol reached as an attribute of the module from another one. + /// + /// ## Why is this bad? + /// A visibility keyword draws a boundary around a member: `private` says only + /// the declaring class's own body may use it, and `protected` extends that to + /// a subclass's body. Reaching past the boundary defeats the point of drawing + /// it, and the member may be renamed or removed without notice. + /// + /// It is also very likely to fail at runtime. The lowering spells the + /// visibility in the member's name, and for `private` that name is one python + /// mangles: an access written outside the class names a different attribute, + /// or none at all. + /// + /// ## Example + /// + /// ```by + /// class Account: + /// init(private let balance: int) + /// + /// def audit(account: Account) -> int: + /// return account.balance # error: `balance` is private to `Account` + /// ``` + pub(crate) static INACCESSIBLE_MEMBER = { + summary: "detects access to a `private` or `protected` member from outside where it may be reached", + status: LintStatus::stable("0.0.80"), + default_level: Level::Error, + ty_compat: TyCompat::BasedPython, + } +} + +declare_lint! { + /// ## What it does + /// Checks for a class member declared less visible than the one it inherits + /// under the same name. + /// + /// ## Why is this bad? + /// A visibility keyword decides the name the member is emitted under, so a + /// member declared less visible than the one it inherits does not override + /// it. It sits beside it under a different name, and the inherited member is + /// still what a call finds — which is never what the declaration looks like + /// it does. + /// + /// ## Example + /// + /// ```by + /// class A: + /// def f(self) -> int: + /// return 1 + /// + /// class B(A): + /// private def f(self) -> int: # error: `f` is public on `A` + /// return 2 + /// + /// B().f() # 1, not 2 + /// ``` + pub(crate) static INVALID_OVERRIDE_VISIBILITY = { + summary: "detects a member declared less visible than the one it overrides", + status: LintStatus::stable("0.0.80"), + default_level: Level::Error, + ty_compat: TyCompat::BasedPython, + } +} + declare_lint! { /// ## What it does /// Checks for imports of a symbol another module declared `private`. @@ -1694,11 +1833,18 @@ declare_lint! { /// Only a `BaseException` subclass can be raised, so a clause with no /// exception in it can never be satisfied by anything the function does. /// + /// A type parameter in a clause stands for one type the caller chooses, so + /// it has to be declared an exception as well — a parameter with no bound + /// can be `int` as easily as `OSError`. + /// /// ## Example /// /// ```by /// def f() raises int: # error: `int` is not an exception /// ... + /// + /// def g[T](value: T) raises T: # error: `T@g` is not always an exception + /// ... /// ``` pub(crate) static INVALID_RAISES_CLAUSE = { summary: "detects a `raises` clause that is not a set of exceptions", @@ -5628,6 +5774,35 @@ pub(super) fn report_implicit_return_type( } } +/// basedpython: report a `def` written with no body at all, where the position asks for an +/// implementation rather than a declaration. +/// +/// The lowering fills the missing body in with `: ...`, so what runs is a function that returns +/// `None`. In a stub file, a protocol, an `abstract def`, an overload group or an +/// `if TYPE_CHECKING` block that is exactly what was meant, and nothing is reported. Everywhere +/// else the declaration stands where the implementation should be, and nothing else says so: the +/// return type may well be `None` already, in which case `empty-body` has nothing to complain +/// about either. +pub(super) fn report_missing_function_body( + context: &InferContext, + function: &ast::StmtFunctionDef, +) { + let Some(builder) = context.report_lint(&MISSING_FUNCTION_BODY, &function.name) else { + return; + }; + let mut diagnostic = builder.into_diagnostic(format_args!( + "Function `{name}` is declared with no body", + name = function.name.id + )); + diagnostic.info("A `def` with no body declares a signature, which is only permitted:"); + diagnostic.info(" - in stub files"); + diagnostic.info(" - in `if TYPE_CHECKING` blocks"); + diagnostic.info(" - as a member of a protocol class"); + diagnostic.info(" - as an `abstract def` or an `@abstractmethod`-decorated method"); + diagnostic.info(" - or as an overload declaration"); + diagnostic.help("Write the body, or `: ...` if the function is meant to do nothing"); +} + pub(super) fn report_invalid_type_checking_constant(context: &InferContext, node: AnyNodeRef) { let Some(builder) = context.report_lint(&INVALID_TYPE_CHECKING_CONSTANT, node) else { return; @@ -6445,13 +6620,54 @@ fn add_non_runtime_checkable_protocol_context<'db>( diagnostic.sub(class_def_diagnostic); } +/// basedpython: a class member declared less visible than the one it inherits +/// under the same name. +pub(crate) fn report_invalid_override_visibility<'db>( + context: &InferContext<'db, '_>, + name: &str, + definition: crate::types::Definition<'db>, + class: ClassType<'db>, + superclass: ClassType<'db>, + declared: MemberVisibility, + inherited: MemberVisibility, +) { + let db = context.db(); + let Some(builder) = context.report_lint( + &INVALID_OVERRIDE_VISIBILITY, + definition.focus_range(db, context.module()), + ) else { + return; + }; + let mut diagnostic = builder.into_diagnostic(format_args!( + "`{name}` is declared `{keyword}`, so it does not override `{superclass}.{name}`", + keyword = declared.keyword(), + superclass = superclass.name(db), + )); + diagnostic.info(format_args!( + "`{keyword}` emits the member as `{prefix}{name}`, a different attribute from the \ + `{inherited_keyword}` one it inherits", + keyword = declared.keyword(), + prefix = declared.name_prefix(), + inherited_keyword = inherited.keyword(), + )); + diagnostic.info(format_args!( + "`{superclass}.{name}` is what `{class}` still answers with", + superclass = superclass.name(db), + class = class.name(db), + )); + diagnostic.help(format_args!( + "declare it `{keyword}` to override it", + keyword = inherited.keyword(), + )); +} + /// basedpython: `A(...)` where `A`'s constructor is declared `private` and the /// call is not inside `A`'s own body. -pub(crate) fn report_private_constructor<'db>( +pub(crate) fn report_restricted_constructor<'db>( context: &InferContext<'db, '_>, call: &ast::ExprCall, class: ClassType<'db>, - constructor: crate::types::visibility::PrivateConstructor<'db>, + constructor: crate::types::visibility::RestrictedConstructor<'db>, ) { let Some(builder) = context.report_lint(&PRIVATE_CONSTRUCTOR, call) else { return; @@ -6465,23 +6681,30 @@ pub(crate) fn report_private_constructor<'db>( // and then the two messages would read the same while meaning different // things let inherited = class.class_literal(db).as_static() != Some(constructor.owner); + let keyword = constructor.visibility.keyword(); let mut diagnostic = if inherited { builder.into_diagnostic(format_args!( - "Cannot construct `{class_name}`: it inherits `{owner_name}`'s private constructor" + "Cannot construct `{class_name}`: it inherits `{owner_name}`'s {keyword} constructor" )) } else { builder.into_diagnostic(format_args!( - "Cannot construct `{class_name}`: its constructor is private" + "Cannot construct `{class_name}`: its constructor is {keyword}" )) }; - let mut declaration = SubDiagnostic::new( - SubDiagnosticSeverity::Info, - format_args!("Only code inside `{owner_name}` may construct it"), - ); + let mut declaration = match constructor.visibility { + MemberVisibility::Protected => SubDiagnostic::new( + SubDiagnosticSeverity::Info, + format_args!("Only code inside `{owner_name}` or a subclass of it may construct it"), + ), + _ => SubDiagnostic::new( + SubDiagnosticSeverity::Info, + format_args!("Only code inside `{owner_name}` may construct it"), + ), + }; declaration.annotate( Annotation::secondary(constructor.function.spans(db).name).message(format_args!( - "`{owner_name}`'s constructor declared private here" + "`{owner_name}`'s constructor declared {keyword} here" )), ); diagnostic.sub(declaration); diff --git a/crates/ty_python_semantic/src/types/exceptions.rs b/crates/ty_python_semantic/src/types/exceptions.rs index bb28ce827f..db211d07df 100644 --- a/crates/ty_python_semantic/src/types/exceptions.rs +++ b/crates/ty_python_semantic/src/types/exceptions.rs @@ -24,6 +24,11 @@ //! That is the only workable default: assuming an unannotated callee raises //! anything would make every set `BaseException`. //! +//! A clause may name a type parameter — `def f[T: OSError](e: T) raises T` — and +//! then what escapes a particular call is the set with what that call solved the +//! parameter to. So the parameter has to be declared an exception, since it +//! stands for one type the caller chooses. +//! //! `try` narrows the set: exceptions raised in the `try` body that an `except` //! clause catches do not escape, while the handler, `else` and `finally` bodies //! contribute their own raises. `except*` is treated as catching nothing, since @@ -36,24 +41,32 @@ //! //! See `docs/basedpython/features/exceptions.md`. +use std::cell::RefCell; + use ruff_db::diagnostic::Annotation; use ruff_db::parsed::parsed_module; use ruff_python_ast::helpers::is_dunder; +use ruff_python_ast::name::Name; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{self as ast, Expr, Stmt}; +use ruff_python_ast::{self as ast, Expr, PySourceType, Stmt}; use ruff_text_size::{Ranged, TextRange}; use ty_python_core::definition::Definition; -use ty_python_core::scope::ScopeId; +use ty_python_core::scope::{NodeWithScopeKind, ScopeId}; +use ty_python_core::{ProgramFile, semantic_index}; use crate::Db; +use crate::reified::{method_receiver, reified_class_reads, reified_type_param_names}; use crate::types::ProgramEnvironment; use crate::types::context::InferContext; use crate::types::diagnostic::{ INVALID_RAISES_CLAUSE, OVERRIDE_RAISE, UNDECLARED_RAISE, UNHANDLED_EXCEPTION, }; use crate::types::function::{FunctionLiteral, FunctionType, OverloadLiteral}; +use crate::types::generics::{ApplySpecialization, Specialization, enclosing_binding_contexts}; +use crate::types::typevar::{BindingContext, BoundTypeVarInstance, TypeVarBoundOrConstraints}; +use crate::types::visitor::any_over_type; use crate::types::{ - ClassType, KnownClass, Type, TypeContext, UnionType, definition_expression_type, + ClassType, KnownClass, Type, TypeContext, TypeMapping, UnionType, definition_expression_type, infer_scope_types, }; @@ -66,11 +79,29 @@ pub(crate) struct RaiseEffect<'db> { range: TextRange, } +/// What a call solved its callee's own type parameters to, recorded by inference. +/// +/// A callee's exception set is written in terms of its type parameters, so what escapes a +/// particular call depends on what that call solved them to. Only the call itself knows — a +/// solution can come from the expected type as much as from the arguments — so inference records +/// it where it binds the call, rather than the exception analysis binding the call a second time. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, get_size2::GetSize, salsa::SalsaValue)] +pub(crate) enum CallSolution<'db> { + /// The call bound, solving the callee's type parameters to this. + Solved(Specialization<'db>), + /// The call did not bind, so nothing is known about what it solved. + Unbound, +} + /// One call in a function body whose exceptions are not fully handled there. #[derive(Debug, Clone, PartialEq, Eq, Hash, get_size2::GetSize, salsa::SalsaValue)] pub(crate) struct CallEffect<'db> { - /// the called function, whose own exception set is resolved separately - callee: FunctionLiteral<'db>, + /// the called function as the call sees it: its type carries whatever + /// specializations it went through on the way, as `f[OSError]`, + /// `Reader[KeyError].read` and a bound method all do + callee: FunctionType<'db>, + /// what the call itself solved the callee's own type parameters to + solution: Option>, /// the exception instance types caught by the `except` clauses around the call caught: Box<[Type<'db>]>, /// the call expression @@ -126,6 +157,101 @@ pub(crate) fn function_raised_exceptions<'db>( ) } +/// The exceptions a call through `function` can raise: the set of the function it +/// is a type of, specialized the way that type has been. +fn function_type_raised_exceptions<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + function: FunctionType<'db>, +) -> Type<'db> { + function.applied_specializations(db).iter().fold( + function_raised_exceptions(db, env, function.literal(db)), + |raised, specialization| substitute_solution(db, env, raised, *specialization), + ) +} + +/// `ty` with each type parameter `solution` solves replaced by what it solved it to. +/// +/// The parameters are matched as parameters, not as occurrences: a call binds a fresh +/// occurrence of its callee's type parameters, so a solution is keyed on that, while the +/// callee's set is written in the source-level one. `f(KeyError())` inside +/// `def f[T](e: T)` solves the fresh `T` to `KeyError`, and that is the `T` of `f`'s clause. +fn substitute_solution<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + ty: Type<'db>, + solution: Specialization<'db>, +) -> Type<'db> { + let solved: Vec<(BoundTypeVarInstance<'db>, Type<'db>)> = solution + .generic_context(db) + .variables(db) + .zip(solution.types(db).iter().copied()) + .collect(); + let solved_for = |bound_typevar: BoundTypeVarInstance<'db>| { + solved + .iter() + .find(|(variable, _)| variable.is_occurrence_of_same_parameter(db, bound_typevar)) + .map(|(_, ty)| *ty) + }; + widen_type_parameters( + db, + env, + ty, + |bound_typevar| solved_for(bound_typevar).is_some(), + |bound_typevar| solved_for(bound_typevar).unwrap_or(Type::TypeVar(bound_typevar)), + ) +} + +/// Whether applying `specialization` to `function` can change what a call to it raises — which +/// is when its [`FunctionType`] has to remember the specialization. +/// +/// Only a function with an exception set of its own has one to change: a `.by` function, whose +/// body may raise, or a stub that declares a `raises` clause. And a specialization only reaches +/// that set when it substitutes type parameters the function can name — its own, or those of the +/// class or function it is defined in. Anything else, such as the solution of a generic call the +/// function is merely passed to, substitutes nothing the set can mention. +pub(crate) fn specialization_reaches_exception_set<'db>( + db: &'db dyn Db, + function: FunctionLiteral<'db>, + specialization: Specialization<'db>, +) -> bool { + let overload = function.last_definition; + let has_exception_set = match overload.file(db).source_type(db) { + PySourceType::BasedPython => true, + PySourceType::BasedPythonStub => function + .iter_overloads_and_implementation(db) + .any(|overload| declares_raises(db, overload)), + PySourceType::Python | PySourceType::Stub | PySourceType::Ipynb => false, + }; + if !has_exception_set { + return false; + } + + let Some(substituted) = specialization + .generic_context(db) + .variables(db) + .next() + .map(|variable| variable.binding_context(db)) + else { + return false; + }; + enclosing_binding_contexts( + semantic_index(db, overload.program_file(db)), + overload.body_scope(db).file_scope_id(db), + ) + .any(|visible| visible == substituted) +} + +/// Whether `overload` writes a `raises` clause, read off the syntax alone. +#[salsa::tracked(returns(copy), heap_size = ruff_memory_usage::heap_size)] +fn declares_raises<'db>(db: &'db dyn Db, overload: OverloadLiteral<'db>) -> bool { + let module = parsed_module(db, overload.python_file(db)).load(db); + overload + .node(db, overload.file(db), &module) + .raises + .is_some() +} + /// The type named by `overload`'s `raises` clause, or `None` when it has none. /// /// `raises ...` is the gradual set: it declares that the function may raise @@ -161,6 +287,18 @@ pub(crate) fn declared_exceptions<'db>( #[salsa::tracked( returns(copy), cycle_initial = |_, _, _| Type::Never, + cycle_fn = |db, cycle: &salsa::Cycle, _previous: &Type<'db>, current: Type<'db>, overload: OverloadLiteral<'db>| { + if cycle.iteration() <= crate::TAINTED_CYCLES { + current + } else { + // a recursive call that solves a type parameter to something built from it + // (`f(Wrapped(e))` inside `def f[T](e: T)`) adds to the set on every round. + // with every type parameter at its ceiling nothing is left to substitute, so + // the next round reproduces this one + let env = &ProgramEnvironment::from_file(overload.program_file(db)); + widen_to_ceilings(db, env, current) + } + }, heap_size = ruff_memory_usage::heap_size, )] pub(crate) fn inferred_exceptions<'db>( @@ -168,11 +306,17 @@ pub(crate) fn inferred_exceptions<'db>( overload: OverloadLiteral<'db>, ) -> Type<'db> { let env = &ProgramEnvironment::from_file(overload.program_file(db)); + let body_scope = overload.body_scope(db); + let visible = visible_binding_contexts(db, overload.program_file(db), body_scope); resolve_effects( db, env, body_exception_effects(db, overload), - Some(overload.body_scope(db)), + &CallSite { + body_scope, + visible: &visible, + resolves_recursion: true, + }, ) } @@ -195,7 +339,40 @@ pub(crate) fn body_exception_effects<'db>( let node = overload.node(db, file, &module); let inference = infer_scope_types(db, overload.body_scope(db), TypeContext::default()); - collect_exception_effects(db, env, &node.body, |expr| inference.expression_type(expr)) + collect_exception_effects( + db, + env, + &node.body, + |expr| inference.expression_type(expr), + |call| inference.call_solution(call), + ) +} + +/// Where a body's calls are made from, which decides what the type parameters in +/// their callees' sets can name. +struct CallSite<'a, 'db> { + /// the body the calls are written in + body_scope: ScopeId<'db>, + /// the binding contexts whose type parameters are in scope in that body + visible: &'a [BindingContext<'db>], + /// whether a call back into this body's own function may be resolved, which + /// for a function without a declared clause means computing its set from + /// inside the computation of that same set + resolves_recursion: bool, +} + +/// The binding contexts whose type parameters name something inside `body_scope`: +/// the function's own, and those of every class and function it is nested in. +fn visible_binding_contexts<'db>( + db: &'db dyn Db, + program_file: ProgramFile<'db>, + body_scope: ScopeId<'db>, +) -> Vec> { + enclosing_binding_contexts( + semantic_index(db, program_file), + body_scope.file_scope_id(db), + ) + .collect() } /// Union the exceptions escaping `effects`, following each call into its callee. @@ -203,12 +380,12 @@ fn resolve_effects<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, effects: &ExceptionEffects<'db>, - self_body_scope: Option>, + site: &CallSite<'_, 'db>, ) -> Type<'db> { UnionType::from_elements( db, env, - escaping_sites(db, env, effects, self_body_scope, &[]) + escaping_sites(db, env, effects, site, &[]) .into_iter() .map(|(_, raised)| raised), ) @@ -216,16 +393,11 @@ fn resolve_effects<'db>( /// Each place in a body that can raise something none of `allowed` covers, /// paired with what escapes there. -/// -/// `self_body_scope` is the scope of the function the effects belong to, when it -/// is known: a directly recursive call contributes exactly the set being -/// computed, so it is the identity of the union and can be dropped rather than -/// re-entered. fn escaping_sites<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, effects: &ExceptionEffects<'db>, - self_body_scope: Option>, + site: &CallSite<'_, 'db>, allowed: &[Type<'db>], ) -> Vec<(TextRange, Type<'db>)> { let direct = effects @@ -236,17 +408,12 @@ fn escaping_sites<'db>( let from_calls = effects .calls .iter() - .filter(|call| { - !call - .callee - .iter_overloads_and_implementation(db) - .any(|overload| Some(overload.body_scope(db)) == self_body_scope) - }) + .filter(|call| !skips_recursive_call(db, call, site)) .filter_map(|call| { let raised = escaping( db, env, - function_raised_exceptions(db, env, call.callee), + call_raised_exceptions(db, env, call, site), &call.caught, )?; Some((call.range, escaping(db, env, raised, allowed)?)) @@ -255,6 +422,132 @@ fn escaping_sites<'db>( direct.chain(from_calls).collect() } +/// Whether `call` is a call back into the function whose body `site` is, and is +/// left out of that body's set. +/// +/// A function with a declared clause is resolved from the declaration, never +/// from its body, so a call back into it is resolved like any other call. Without +/// one, the call raises what the body is being computed to raise, which is where +/// two cases part: +/// +/// - a call that changes nothing — no specialization on the way, and a solution +/// that substitutes nothing — contributes exactly the set being computed. It is +/// the identity of the union, and dropping it is exact +/// - a call that does change something (`f(KeyError())` inside `def f[T](e: T)`) +/// raises the set with that change, so it has to be resolved. That is a fixpoint +/// over [`inferred_exceptions`], which is only safe where that query is what is +/// running: the check on a body is inside the body's own inference, and +/// resolving there would re-enter it +fn skips_recursive_call<'db>( + db: &'db dyn Db, + call: &CallEffect<'db>, + site: &CallSite<'_, 'db>, +) -> bool { + let callee = call.callee.literal(db); + if !callee + .iter_overloads_and_implementation(db) + .any(|overload| overload.body_scope(db) == site.body_scope) + { + return false; + } + if callee + .iter_overloads_and_implementation(db) + .any(|overload| declared_exceptions(db, overload).is_some()) + { + return false; + } + let changes_nothing = call.callee.applied_specializations(db).is_empty() + && !matches!(call.solution, Some(CallSolution::Solved(_))); + changes_nothing || !site.resolves_recursion +} + +/// The exceptions escaping one call: the callee's set, specialized the way the +/// call sees the callee and by what the call solved. +/// +/// A type parameter still left in it is one the call did not solve — an explicit +/// specialization does not survive into anything a call can read, and a call that +/// fails to bind solves nothing. Whether that matters depends on where the call is +/// written. A parameter bound by something enclosing the call names a real type +/// there: a caller's own `U` passed on to `f(u)`, a closure over its enclosing +/// function's `T`, a method's `self.m()` within its class. Anything else names +/// nothing at the call, so it stands for everything it was declared to allow — +/// and a `raises` clause may only name a parameter whose ceiling is exceptions. +/// For a call that did not bind, it stands for nothing known at all. +fn call_raised_exceptions<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + call: &CallEffect<'db>, + site: &CallSite<'_, 'db>, +) -> Type<'db> { + let raised = function_type_raised_exceptions(db, env, call.callee); + let raised = match call.solution { + Some(CallSolution::Solved(solution)) => substitute_solution(db, env, raised, solution), + Some(CallSolution::Unbound) | None => raised, + }; + widen_type_parameters( + db, + env, + raised, + |bound_typevar| !site.visible.contains(&bound_typevar.binding_context(db)), + |bound_typevar| match call.solution { + Some(CallSolution::Unbound) => Type::unknown(), + Some(CallSolution::Solved(_)) | None => { + bound_typevar.typevar(db).declared_ceiling(db, env) + } + }, + ) +} + +/// `ty` with every type parameter in it widened to its declared ceiling. +fn widen_to_ceilings<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + ty: Type<'db>, +) -> Type<'db> { + widen_type_parameters( + db, + env, + ty, + |_| true, + |bound_typevar| bound_typevar.typevar(db).declared_ceiling(db, env), + ) +} + +/// `ty` with every type parameter `widen` accepts replaced by `replacement` of it. +fn widen_type_parameters<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + ty: Type<'db>, + widen: impl Fn(BoundTypeVarInstance<'db>) -> bool, + replacement: impl Fn(BoundTypeVarInstance<'db>) -> Type<'db>, +) -> Type<'db> { + let widened = RefCell::new(Vec::new()); + any_over_type(db, env, ty, false, |nested| { + if let Type::TypeVar(bound_typevar) = nested + && !widened.borrow().contains(&bound_typevar) + && widen(bound_typevar) + { + widened.borrow_mut().push(bound_typevar); + } + false + }); + + widened + .into_inner() + .into_iter() + .fold(ty, |ty, bound_typevar| { + ty.apply_type_mapping( + db, + env, + &TypeMapping::ApplySpecialization(ApplySpecialization::Single( + bound_typevar, + replacement(bound_typevar), + )), + TypeContext::default(), + ) + }) +} + /// The part of `raised` that no type in `caught` handles, or `None` when it is /// caught entirely. /// @@ -298,19 +591,22 @@ fn union_elements<'db>(db: &'db dyn Db, ty: Type<'db>) -> Vec> { /// Collect the [`ExceptionEffects`] of `body`. /// -/// `expression_type` supplies inferred types for expressions in the body. It is -/// a callback so that the check for the function currently being inferred can +/// `expression_type` supplies inferred types for expressions in the body, and +/// `call_solution` what each call solved its callee's type parameters to. They +/// are callbacks so that the check for the function currently being inferred can /// read that in-progress inference rather than re-entering it as a query. fn collect_exception_effects<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, body: &[Stmt], expression_type: impl Fn(&Expr) -> Type<'db>, + call_solution: impl Fn(&ast::ExprCall) -> Option>, ) -> ExceptionEffects<'db> { let mut collector = EffectsCollector { db, env: env.clone(), expression_type, + call_solution, caught: Vec::new(), handling: Vec::new(), direct: Vec::new(), @@ -324,10 +620,11 @@ fn collect_exception_effects<'db>( } } -struct EffectsCollector<'db, F> { +struct EffectsCollector<'db, F, G> { db: &'db dyn Db, env: ProgramEnvironment<'db>, expression_type: F, + call_solution: G, /// the exception types caught by the `except` clauses currently enclosing /// the node being visited, innermost last caught: Vec>, @@ -338,9 +635,10 @@ struct EffectsCollector<'db, F> { calls: Vec>, } -impl<'db, F> EffectsCollector<'db, F> +impl<'db, F, G> EffectsCollector<'db, F, G> where F: Fn(&Expr) -> Type<'db>, + G: Fn(&ast::ExprCall) -> Option>, { fn visit_try(&mut self, try_stmt: &ast::StmtTry) { let env = self.env.clone(); @@ -460,18 +758,25 @@ where } /// Record a call to `callee`, minus whatever the enclosing handlers catch. - fn record_call(&mut self, callee: FunctionLiteral<'db>, range: TextRange) { + fn record_call( + &mut self, + callee: FunctionType<'db>, + solution: Option>, + range: TextRange, + ) { self.calls.push(CallEffect { callee, + solution, caught: self.caught.clone().into_boxed_slice(), range, }); } } -impl<'db, F> Visitor<'_> for EffectsCollector<'db, F> +impl<'db, F, G> Visitor<'_> for EffectsCollector<'db, F, G> where F: Fn(&Expr) -> Type<'db>, + G: Fn(&ast::ExprCall) -> Option>, { fn visit_stmt(&mut self, stmt: &Stmt) { let env = self.env.clone(); @@ -529,7 +834,8 @@ where } Expr::Call(call) => { if let Some(callee) = callee_function(self.db, (self.expression_type)(&call.func)) { - self.record_call(callee, call.range()); + let solution = (self.call_solution)(call); + self.record_call(callee, solution, call.range()); } } _ => {} @@ -541,21 +847,22 @@ where /// The function a call resolves to, when it is one whose body can be analysed. /// -/// The whole literal is returned rather than a single overload: which overload a -/// call matched is not known here, so resolution unions over all of them. +/// The whole function is returned rather than a single overload: which overload a +/// call matched is not known here, so resolution unions over all of them. It is +/// the function's type rather than its literal, which is what carries any +/// specialization it went through on the way to the call. /// /// Callables, unions of callables, overload sets matched by argument, and /// constructor calls are all left alone: this analysis reports nothing rather /// than guessing at a set it cannot see. -fn callee_function<'db>(db: &'db dyn Db, callee: Type<'db>) -> Option> { +fn callee_function<'db>(db: &'db dyn Db, callee: Type<'db>) -> Option> { let function = match callee { Type::FunctionLiteral(function) => function, Type::BoundMethod(method) => method.function(db), _ => return None, }; - let literal = function.literal(db); - let overload = literal.last_definition; + let overload = function.literal(db).last_definition; // a dunder is reached through implicit dispatch far more often than through a // written call, and this analysis only sees the written ones. reporting the // visible half of a set would be worse than reporting none of it @@ -563,7 +870,7 @@ fn callee_function<'db>(db: &'db dyn Db, callee: Type<'db>) -> Option( return; } - let allowed = function_raised_exceptions(db, env, superclass_function.literal(db)); - let raised = function_raised_exceptions(db, env, subclass_function.literal(db)); + // the base method is the one the subclass inherits, specialized by whatever the + // subclass wrote for the base's type parameters (`class FileReader(Reader[OSError])`) + let allowed = function_type_raised_exceptions(db, env, superclass_function); + let raised = function_type_raised_exceptions(db, env, subclass_function); let Some(extra) = escaping(db, env, raised, &[allowed]) else { return; }; @@ -640,6 +949,7 @@ pub(super) fn check_function_exceptions<'db, 'ast>( body_scope: ScopeId<'db>, definition: Definition<'db>, expression_type: impl Fn(&Expr) -> Type<'db>, + call_solution: impl Fn(&ast::ExprCall) -> Option>, ) { let env = context.program_environment(); let db = context.db(); @@ -666,12 +976,21 @@ pub(super) fn check_function_exceptions<'db, 'ast>( None => return, }; - let effects = collect_exception_effects(db, env, &function.body, expression_type); + let effects = + collect_exception_effects(db, env, &function.body, expression_type, call_solution); if effects.is_empty() { return; } - for (range, escaped) in escaping_sites(db, env, &effects, Some(body_scope), &allowed) { + let visible = visible_binding_contexts(db, context.program_file(), body_scope); + let site = CallSite { + body_scope, + visible: &visible, + // this runs inside the body's own inference, so a call back into an + // undeclared function here cannot be resolved without re-entering it + resolves_recursion: declared.is_some(), + }; + for (range, escaped) in escaping_sites(db, env, &effects, &site, &allowed) { let name = &function.name.id; if declared.is_some() { let Some(builder) = context.report_lint(&UNDECLARED_RAISE, range) else { @@ -721,43 +1040,311 @@ fn check_raises_clause_is_exceptions<'db, 'ast>( if declared.is_never() || declared.is_dynamic() { return; } - if !declared.is_disjoint_from(db, env, KnownClass::BaseException.to_instance(db, env)) { + if declared.is_disjoint_from(db, env, KnownClass::BaseException.to_instance(db, env)) { + if let Some(builder) = context.report_lint(&INVALID_RAISES_CLAUSE, clause) { + builder.into_diagnostic(format_args!( + "`{}` contains no exception, so nothing can satisfy this `raises` clause", + declared.display(db, env) + )); + } return; } - if let Some(builder) = context.report_lint(&INVALID_RAISES_CLAUSE, clause) { - builder.into_diagnostic(format_args!( - "`{}` contains no exception, so nothing can satisfy this `raises` clause", - declared.display(db, env) + check_raises_clause_type_parameters(context, clause, declared); +} + +/// Report a type parameter in a `raises` clause that a caller could pick a +/// non-exception for. +/// +/// A type parameter stands for one type the caller chooses, so `raises T` says +/// something about exceptions only when every type `T` can be is an exception. +/// That is what its declaration says: `def f[T: OSError](...) raises T` and +/// `def f[T in (KeyError, IndexError)](...) raises T` both hold, while a +/// parameter with no bound at all can be `int` as easily as `OSError`. +/// +/// Only a member of the set itself has to be an exception. A parameter inside one +/// — `raises E[X]` for `class E[X](Exception)` — is a type argument of an +/// exception, and says nothing about what is raised. +fn check_raises_clause_type_parameters<'db, 'ast>( + context: &InferContext<'db, 'ast>, + clause: &'ast Expr, + declared: Type<'db>, +) { + let env = context.program_environment(); + let db = context.db(); + let exception = KnownClass::BaseException.to_instance(db, env); + + for element in union_elements(db, declared) { + let Type::TypeVar(bound_typevar) = element else { + continue; + }; + let typevar = bound_typevar.typevar(db); + if typevar + .declared_ceiling(db, env) + .is_assignable_to(db, env, exception) + { + continue; + } + + let range = union_operand_named(clause, typevar.name(db)).unwrap_or(clause); + let Some(builder) = context.report_lint(&INVALID_RAISES_CLAUSE, range) else { + continue; + }; + let name = element.display(db, env); + let mut diagnostic = builder.into_diagnostic(format_args!( + "`{name}` is not always an exception, so it cannot appear in a `raises` clause" )); + match typevar.bound_or_constraints(db, env) { + None => diagnostic.info(format_args!("`{name}` has no bound")), + Some(TypeVarBoundOrConstraints::UpperBound(bound)) => diagnostic.info(format_args!( + "`{name}` is bounded by `{}`", + bound.display(db, env) + )), + Some(TypeVarBoundOrConstraints::Constraints(constraints)) => { + if let Some(constraint) = constraints + .elements(db) + .iter() + .find(|constraint| !constraint.is_assignable_to(db, env, exception)) + { + diagnostic.info(format_args!( + "`{name}` can be `{}`", + constraint.display(db, env) + )); + } + } + } + } +} + +/// The top-level `|` operand of `clause` that is the bare name `name`, if any. +fn union_operand_named<'e>(clause: &'e Expr, name: &str) -> Option<&'e Expr> { + match clause { + Expr::BinOp(ast::ExprBinOp { + left, + op: ast::Operator::BitOr, + right, + .. + }) => union_operand_named(left, name).or_else(|| union_operand_named(right, name)), + Expr::Name(named) if named.id.as_str() == name => Some(clause), + _ => None, } } -/// The `isinstance` target for a function's declared exception set, for a -/// runtime guard on the lowered function. +/// The runtime test for a function's declared exception set, for a guard on the +/// lowered function. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RaisesRuntimeTarget { + /// The `isinstance` target with every type parameter widened to its declared + /// ceiling — what the guard tests when nothing says which type the caller + /// chose. + pub ceiling: String, + /// How to build the exact target at the call, when the clause names a type + /// parameter that carries a runtime value. + pub resolved: Option, +} + +/// The `isinstance` target for a clause naming type parameters that carry a +/// runtime value, spelled in terms of them. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedRaisesTarget { + /// The target, with each of those parameters spelled by its name. + pub expression: String, + /// The guarded function's own reified parameters, read off the specialization + /// it is called through. + pub own: Vec, + /// Its class's reified parameters, read off the instance the method is + /// called on. + pub receiver: Vec, +} + +/// The runtime test for `function`'s declared exception set. /// /// `None` when there is no faithful runtime test — a gradual clause, or a set /// whose members have no runtime spelling (a negation, a protocol). `Never` /// becomes the empty tuple, which no exception is an instance of. +/// +/// A type parameter has no runtime spelling of its own: which exception it is +/// was chosen by the caller, and the guard runs inside the callee. What the +/// declaration always states is the parameter's ceiling, and every value it can +/// take is one of those, so that is the test the guard can always make — it +/// never rejects an exception the clause allows, and still catches one it does +/// not. A reified parameter does carry its value, and where the guard can read +/// it, it tests that instead. pub fn declared_raises_runtime_target<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: ruff_db::files::File, function: Type<'db>, -) -> Option { +) -> Option { let Type::FunctionLiteral(function) = function else { return None; }; - let declared = declared_exceptions(db, function.literal(db).last_definition)?; + let overload = function.literal(db).last_definition; + let declared = declared_exceptions(db, overload)?; if declared.is_dynamic() { return None; } if declared.is_never() { - return Some("()".to_string()); + return Some(RaisesRuntimeTarget { + ceiling: "()".to_string(), + resolved: None, + }); + } + + let ceiling = crate::types::soundness::runtime_check_target( + db, + env, + file, + widen_to_ceilings(db, env, declared), + )?; + + Some(RaisesRuntimeTarget { + ceiling, + resolved: resolved_runtime_target(db, env, file, overload, declared), + }) +} + +/// Where a reified type parameter's value can be read, from inside the guard on a +/// function. +#[derive(Clone, Copy, PartialEq, Eq)] +enum RuntimeSource { + /// the guarded function's own parameter: its `generic` wrapper binds it for + /// each call + Own, + /// the class parameter of a method with a receiver: the instance the method is + /// called on carries it + Receiver, + /// an enclosing function's parameter: the guard is evaluated inside that + /// function's call, where the value is already bound + Closure, +} + +/// The exact `isinstance` target for `declared`, when it names type parameters +/// the guard on `overload` can read the values of. +/// +/// Each parameter is looked up by what binds it rather than by its name, which an +/// inner parameter can shadow. A parameter with no readable value — a class +/// parameter in a function nested in a method, whose first argument is not a +/// receiver, or anything not reified at all — is tested at its ceiling. Asking for +/// the value of a parameter that is not reified would mean reifying it, and +/// turning a check on must not change how the program is built. +fn resolved_runtime_target<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: ruff_db::files::File, + overload: OverloadLiteral<'db>, + declared: Type<'db>, +) -> Option { + let sources = runtime_type_parameter_sources(db, overload); + let source_of = |bound_typevar: BoundTypeVarInstance<'db>| { + let name = bound_typevar.typevar(db).name(db); + sources + .iter() + .find(|(context, _, _)| *context == bound_typevar.binding_context(db)) + .filter(|(_, _, reified)| reified.contains(name)) + .map(|(_, source, _)| *source) + }; + + let mut own = Vec::new(); + let mut receiver = Vec::new(); + let mut reads_any = false; + let mut parts: Vec = Vec::new(); + for element in union_elements(db, declared) { + let part = match element { + Type::TypeVar(bound_typevar) if let Some(source) = source_of(bound_typevar) => { + let name = bound_typevar.typevar(db).name(db).to_string(); + let names = match source { + RuntimeSource::Own => Some(&mut own), + RuntimeSource::Receiver => Some(&mut receiver), + RuntimeSource::Closure => None, + }; + if let Some(names) = names + && !names.contains(&name) + { + names.push(name.clone()); + } + reads_any = true; + name + } + element => crate::types::soundness::runtime_check_target( + db, + env, + file, + widen_to_ceilings(db, env, element), + )?, + }; + if !parts.contains(&part) { + parts.push(part); + } } - crate::types::soundness::runtime_check_target(db, env, file, declared) + if !reads_any { + return None; + } + // isinstance accepts nested tuples, so a rendered union composes without + // flattening + let expression = match parts.len() { + 1 => parts.pop()?, + _ => format!("({})", parts.join(", ")), + }; + Some(ResolvedRaisesTarget { + expression, + own, + receiver, + }) +} + +/// The binding contexts around `overload`'s body whose reified parameters the +/// guard can read, each with where it reads them and which of its parameters are +/// reified. +/// +/// The reified sets are the ones the lowering reifies by: a function is wrapped +/// in `generic` exactly when its set is not empty, and a class decorated with +/// `generic_class` likewise. +fn runtime_type_parameter_sources<'db>( + db: &'db dyn Db, + overload: OverloadLiteral<'db>, +) -> Vec<(BindingContext<'db>, RuntimeSource, Vec)> { + let module = parsed_module(db, overload.python_file(db)).load(db); + let index = semantic_index(db, overload.program_file(db)); + let function = overload.node(db, overload.file(db), &module); + let mut sources = vec![( + BindingContext::from(overload.definition(db)), + RuntimeSource::Own, + reified_type_param_names(PySourceType::BasedPython, function), + )]; + + // the first class or function around the body decides whether this is a + // method. a type-parameter scope may sit in between, and carries nothing + let mut direct = true; + for (_, scope) in index + .ancestor_scopes(overload.body_scope(db).file_scope_id(db)) + .skip(1) + { + match scope.node() { + NodeWithScopeKind::Class(class) => { + if direct && method_receiver(function).is_some() { + sources.push(( + index.expect_single_definition(class).into(), + RuntimeSource::Receiver, + reified_class_reads(PySourceType::BasedPython, class.node(&module)).names, + )); + } + direct = false; + } + NodeWithScopeKind::Function(enclosing) => { + sources.push(( + index.expect_single_definition(enclosing).into(), + RuntimeSource::Closure, + reified_type_param_names(PySourceType::BasedPython, enclosing.node(&module)), + )); + direct = false; + } + _ => {} + } + } + sources } /// Whether `function` is the module's entry point — a `main` defined directly at diff --git a/crates/ty_python_semantic/src/types/function.rs b/crates/ty_python_semantic/src/types/function.rs index 3b80ade284..45ed33a3d7 100644 --- a/crates/ty_python_semantic/src/types/function.rs +++ b/crates/ty_python_semantic/src/types/function.rs @@ -67,7 +67,10 @@ use ruff_text_size::Ranged; use salsa::plumbing::AsId; use ty_module_resolver::{ImportingFile, KnownModule, ModuleName, file_to_module, resolve_module}; -use crate::place::{DefinedPlace, Definedness, Place, declared_type_at_load, place_from_bindings}; +use crate::place::{ + DefinedPlace, Definedness, Place, declared_type_at_load, place_from_bindings, + promote_undeclared, +}; use crate::types::call::{Binding, CallArguments}; use crate::types::callable::CallableTypeKind; use crate::types::constraints::ConstraintSet; @@ -82,7 +85,7 @@ use crate::types::diagnostic::{ report_runtime_check_against_typed_dict, }; use crate::types::display::DisplaySettings; -use crate::types::generics::{GenericContext, typing_self}; +use crate::types::generics::{ApplySpecialization, GenericContext, Specialization, typing_self}; use crate::types::infer::{ function_known_decorators, infer_definition_types, nearest_enclosing_class, original_class_type, }; @@ -218,6 +221,10 @@ bitflags! { /// `@builder` — a widget builder, which emits into the composition being /// built and so, like a composable, can only be called while composing const UI_BUILDER = 1 << 14; + /// basedpython: the method carries the `protected` modifier keyword. A + /// protected member is reachable from the declaring class's body and from + /// a subclass's, and from nowhere else + const PROTECTED = 1 << 15; } } @@ -446,6 +453,67 @@ pub(crate) struct CallbackParameterModifiers { #[salsa::tracked] impl<'db> OverloadLiteral<'db> { + /// basedpython: the type an untyped property declares through its initialiser, when this + /// function is one of that property's accessors + /// + /// `var c = 0` with an accessor block is the property `var c: int = 0` with its type left for + /// the initialiser to say, the way a declaration's type is — `int`, not the literal `0`. both + /// accessors are held to it: the getter returns it and the setter accepts it. without an + /// initialiser (only a `let` may leave both out) the property's type is whatever its getter + /// returns, and this answers `None` + /// + /// the parser lowers the construct to a getter carrying the construct's range, a backing + /// declaration and a setter, all ranged inside that span — see + /// [`ast::StmtFunctionDef::property_construct_range`] + #[salsa::tracked( + returns(copy), + cycle_initial=|_, _, _| None, + heap_size=ruff_memory_usage::heap_size, + )] + pub(crate) fn property_initialiser_type(self, db: &'db dyn Db) -> Option> { + if !self.file(db).source_type(db).is_basedpython() { + return None; + } + let body_scope = self.body_scope(db); + let module = parsed_module(db, self.python_file(db)).load(db); + let accessor = body_scope.node(db).expect_function().node(&module); + let index = semantic_index(db, body_scope.program_file(db)); + let class = index + .class_definition_of_method(body_scope.file_scope_id(db))? + .kind(db) + .as_class()? + .node(&module); + + let (getter, construct) = class.body.iter().find_map(|member| { + let getter = member.as_function_def_stmt()?; + let construct = getter.property_construct_range()?; + (getter.name.id == accessor.name.id && construct.contains_range(accessor.range())) + .then_some((getter, construct)) + })?; + // a written type rides on the getter as its return annotation + if getter.returns.is_some() { + return None; + } + + // the initialiser written on the declaration is stored by a backing assignment the parser + // synthesizes, which spans no source of its own. an explicit `field = ...` is ranged on + // what was written, and states the storage's type rather than the property's + let target = class.body.iter().find_map(|member| match member { + ast::Stmt::Assign(assign) + if assign.range.is_empty() && construct.contains_range(assign.range) => + { + match assign.targets.as_slice() { + [ast::Expr::Name(target)] => Some(target), + _ => None, + } + } + _ => None, + })?; + let definition = index.try_definition(target)?; + let env = ProgramEnvironment::from_scope(body_scope); + Some(promote_undeclared(db, &env, binding_type(db, definition))) + } + pub(super) fn with_deprecated( self, db: &'db dyn Db, @@ -549,6 +617,13 @@ impl<'db> OverloadLiteral<'db> { let Some(definition) = binding.binding.definition() else { continue; }; + // a loop header is not a `def`: it stands for whatever the loop carries round to + // the next iteration, which for a `def` in a loop body is that same `def`. the + // definitions it stands for are in this very list, so skipping it loses nothing + // and keeps a lone `def` in a loop body from reading as a run of two + if definition.kind(db).is_loop_header() { + continue; + } if definition != self.definition(db) { return true; } @@ -1095,6 +1170,22 @@ impl<'db> OverloadLiteral<'db> { } }); + // basedpython: an untyped property with an initialiser is held to the type that + // initialiser declares as though it were written — see `property_initialiser_type`. so it + // counts as written for every source below, which only fill in what the source left out + let property_type = self.property_initialiser_type(db); + let is_property_getter = function_stmt_node.property_construct_range().is_some(); + if let Some(property_type) = property_type { + if is_property_getter { + raw_signature.return_ty = property_type; + } else if let Some(value) = function_stmt_node.parameters.args.get(1) { + raw_signature + .declare_unannotated_parameter(&value.parameter.name.id, property_type); + } + } + let returns_written = + function_stmt_node.returns.is_some() || (is_property_getter && property_type.is_some()); + // basedpython: if this is the implementation of an overloaded function // (i.e. preceded by `@overload` stubs), infer unannotated parameter // types and an unannotated return type from the union of the sibling @@ -1114,7 +1205,7 @@ impl<'db> OverloadLiteral<'db> { db, env, &overload_sigs, - function_stmt_node.returns.is_none(), + !returns_written, ); } } @@ -1126,13 +1217,13 @@ impl<'db> OverloadLiteral<'db> { // while skipping this would answer with what the placeholder body happens to do rather // than with what the base already declared if infers_unannotated_signatures(db, self.file(db)) - && raw_signature.has_inherited_annotations_to_fill(function_stmt_node.returns.is_none()) + && raw_signature.has_inherited_annotations_to_fill(!returns_written) && let Some(base_signature) = self.overridden_signature(db, env) { // a narrowing return type is the exception. it is a claim about what the body tests, // so an override that tests something else — or nothing — would be handed a claim it // does not make, and every call through it would narrow on the strength of it - let inherits_return_type = function_stmt_node.returns.is_none() + let inherits_return_type = !returns_written && !matches!( base_signature.return_ty, Type::TypeIs(_) | Type::TypeGuard(_) @@ -1168,7 +1259,7 @@ impl<'db> OverloadLiteral<'db> { // not make, and every call through it would narrow on the strength of it. so that one is // left to the override's own body if infers_unannotated_signatures(db, self.file(db)) - && function_stmt_node.returns.is_none() + && !returns_written && !function_stmt_node.is_asserts_return && raw_signature.return_ty.is_unknown() && let OverriddenReturnType::Declared(base_return) = @@ -1209,7 +1300,7 @@ impl<'db> OverloadLiteral<'db> { // type out draws on; [`OverloadLiteral::return_type_without_annotation`] mirrors them in // this order, so a change here belongs there too if infers_unannotated_signatures(db, self.file(db)) - && function_stmt_node.returns.is_none() + && !returns_written && !function_stmt_node.is_asserts_return && raw_signature.return_ty.is_unknown() && self.recovers_return_type_from_body(db, env) @@ -1367,6 +1458,14 @@ impl<'db> OverloadLiteral<'db> { env: &ProgramEnvironment<'db>, from_body: impl FnOnce() -> Type<'db>, ) -> Type<'db> { + // the getter of an untyped property with an initialiser — the setter always writes its + // `-> None` — is held to that initialiser's type ahead of everything below + if !self.has_explicit_return_annotation(db) + && let Some(property_type) = self.property_initialiser_type(db) + { + return property_type; + } + let mut return_ty = Type::unknown(); if !self.is_overload(db) @@ -1966,22 +2065,95 @@ pub struct UpdatedFunctionSignatures<'db> { /// /// See also: [`FunctionLiteral::last_definition_signature`]. implementation_callables: Option]>>, + + /// basedpython: the specializations applied to this function, one per generic context, each + /// already composed with every specialization applied after it. + /// + /// Applying a specialization rewrites the signature and keeps nothing of the mapping. A + /// function's exception set is not part of its signature, though: it is written in terms of + /// the function's own type parameters and its class's, and has to be specialized the same + /// way for a call through `f[OSError]`, `Reader[KeyError].read` or a bound method to raise + /// what it says. Only a basedpython function has an exception set, so only its type records + /// them. + specializations: Box<[Specialization<'db>]>, } impl<'db> UpdatedFunctionSignatures<'db> { fn new( signature: Option>, implementation_callables: Option]>>, + specializations: Box<[Specialization<'db>]>, ) -> Option> { - (signature.is_some() || implementation_callables.is_some()).then(|| { - Box::new(Self { - signature, - implementation_callables, + (signature.is_some() || implementation_callables.is_some() || !specializations.is_empty()) + .then(|| { + Box::new(Self { + signature, + implementation_callables, + specializations, + }) }) - }) } } +/// `recorded` with the specialization `type_mapping` applies composed onto it, when that +/// specialization can change what a call to `literal` raises. +/// +/// Every earlier entry has the new specialization applied to its types, so a class projection +/// (`T@Reader` to `U@Sub`) followed by a specialization of the subclass (`U@Sub` to `KeyError`) +/// records `T@Reader` to `KeyError`. A generic context keeps its first entry: once its variables +/// are substituted they no longer appear, so a later specialization of the same context changes +/// nothing. +/// +/// Anything recorded is part of the function type's identity, so nothing is recorded that cannot +/// matter: a specialization that substitutes nothing, or one +/// [`specialization_reaches_exception_set`](crate::types::exceptions::specialization_reaches_exception_set) +/// rules out. Without that, passing a function through a generic call (`reveal_type(f)`) would +/// hand back a type that differs from `f` by the call's own solution. +fn compose_applied_specializations<'db>( + db: &'db dyn Db, + literal: FunctionLiteral<'db>, + recorded: &[Specialization<'db>], + type_mapping: &TypeMapping<'_, 'db>, +) -> Box<[Specialization<'db>]> { + let (TypeMapping::ApplySpecialization(ApplySpecialization::Specialization { + specialization, + .. + }) + | TypeMapping::ApplySpecializationWithMaterialization { + specialization: ApplySpecialization::Specialization { specialization, .. }, + .. + } + | TypeMapping::ProjectUseSiteVariance { + specialization: ApplySpecialization::Specialization { specialization, .. }, + .. + }) = type_mapping + else { + return recorded.into(); + }; + let specialization = *specialization; + if specialization.substitutes_nothing(db) + || !crate::types::exceptions::specialization_reaches_exception_set( + db, + literal, + specialization, + ) + { + return recorded.into(); + } + + let mut composed: Vec> = recorded + .iter() + .map(|earlier| earlier.apply_specialization(db, specialization)) + .collect(); + if !composed + .iter() + .any(|earlier| earlier.generic_context(db) == specialization.generic_context(db)) + { + composed.push(specialization); + } + composed.into_boxed_slice() +} + /// Represents a function type, which might be a non-generic function, or a specialization of a /// generic function. #[salsa::interned(debug, heap_size=ruff_memory_usage::heap_size)] @@ -2015,6 +2187,14 @@ pub(super) fn walk_function_type<'db, V: super::visitor::TypeVisitor<'db> + ?Siz #[salsa::tracked] impl<'db> FunctionType<'db> { + /// basedpython: the specializations applied to this function — see + /// [`UpdatedFunctionSignatures::specializations`]. + pub(crate) fn applied_specializations(self, db: &'db dyn Db) -> &'db [Specialization<'db>] { + self.updated_signatures(db) + .as_deref() + .map_or(&[], |updated| &updated.specializations) + } + pub(super) fn updated_signature(self, db: &'db dyn Db) -> Option<&'db CallableSignature<'db>> { self.updated_signatures(db) .as_deref() @@ -2062,6 +2242,7 @@ impl<'db> FunctionType<'db> { UpdatedFunctionSignatures::new( self.updated_signature(db).cloned(), Some(implementation_callables), + self.applied_specializations(db).into(), ), ) } @@ -2094,6 +2275,7 @@ impl<'db> FunctionType<'db> { UpdatedFunctionSignatures::new( Some(updated_signature), updated_implementation_callables, + self.applied_specializations(db).into(), ), ) } @@ -2175,7 +2357,16 @@ impl<'db> FunctionType<'db> { Self::new( db, literal, - UpdatedFunctionSignatures::new(updated_signature, updated_implementation_callables), + UpdatedFunctionSignatures::new( + updated_signature, + updated_implementation_callables, + compose_applied_specializations( + db, + literal, + self.applied_specializations(db), + type_mapping, + ), + ), ) } } @@ -2641,6 +2832,7 @@ impl<'db> FunctionType<'db> { UpdatedFunctionSignatures::new( updated_signature, updated_implementation_callables, + self.applied_specializations(db).into(), ), )) }, diff --git a/crates/ty_python_semantic/src/types/generics.rs b/crates/ty_python_semantic/src/types/generics.rs index 7865ac31bc..bbfac0812d 100644 --- a/crates/ty_python_semantic/src/types/generics.rs +++ b/crates/ty_python_semantic/src/types/generics.rs @@ -1624,10 +1624,22 @@ impl<'db> Specialization<'db> { /// `{U: int}`, we can apply the second specialization to the first, resulting in `T: int`. /// That lets us produce the generic alias `A[int]`, which is the corresponding entry in the /// MRO of `B[int]`. - fn apply_specialization(self, db: &'db dyn Db, other: Specialization<'db>) -> Self { + pub(super) fn apply_specialization(self, db: &'db dyn Db, other: Specialization<'db>) -> Self { self.apply_specialization_with_recursion(db, other, None) } + /// basedpython: whether this maps every type variable of its generic context to an + /// occurrence of that same parameter, so that applying it substitutes nothing — see + /// [`BoundTypeVarInstance::is_occurrence_of_same_parameter`]. + pub(crate) fn substitutes_nothing(self, db: &'db dyn Db) -> bool { + self.generic_context(db) + .variables(db) + .zip(self.types(db)) + .all(|(variable, ty)| { + matches!(ty, Type::TypeVar(solved) if solved.is_occurrence_of_same_parameter(db, variable)) + }) + } + pub(super) fn apply_specialization_with_recursion( self, db: &'db dyn Db, diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 29bf43ea5b..fa706ef8b2 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -19,7 +19,7 @@ use crate::types::extensions::{ use crate::types::function::FunctionDecorators; use crate::types::generics::GenericContext; use crate::types::implicit_names::{ImplicitNamePosition, implicit_name}; -use crate::types::infer::nearest_enclosing_function; +use crate::types::infer::{infer_definition_types, nearest_enclosing_function}; use crate::types::list_members::all_end_of_scope_members; use crate::types::overrides::is_constructor_like_method; use crate::types::receivers; @@ -3356,7 +3356,11 @@ pub fn inherited_parameter_default( /// /// `None` is what such a `def` already means, so recovering it says nothing the source did not /// — the `redundant-return-annotation` lint reports writing it down. -pub fn inferred_return_annotation<'db>(db: &'db dyn Db, function: Type<'db>) -> Option> { +pub fn inferred_return_annotation<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + function: Type<'db>, +) -> Option> { let Type::FunctionLiteral(function) = function else { return None; }; @@ -3365,7 +3369,40 @@ pub fn inferred_return_annotation<'db>(db: &'db dyn Db, function: Type<'db>) -> .last_definition_raw_signature(db, ReturnCallableTypeVarScope::Public) .return_ty; - (!return_ty.is_unknown() && !return_ty.is_none(db)).then_some(return_ty) + (!return_ty.is_none(db) && is_settled(db, env, return_ty)).then_some(return_ty) +} + +/// whether `ty` is one ty settled on, rather than a stand-in for one it could not read +/// +/// `Unknown` is what nothing supplied, and a type mentioning the marker a cycle leaves behind +/// (`def f(): return f()`) is one the cycle never settled. neither says anything a reader could +/// write down +fn is_settled<'db>(db: &'db dyn Db, env: &ProgramEnvironment<'db>, ty: Type<'db>) -> bool { + !ty.is_unknown() && !ty.mentions_divergence(db, env) +} + +/// basedpython: the type recovered for a property whose declaration names none +/// +/// a property construct (`let a` plus a `get` / `set` / `field` suite) is lowered to a getter +/// whose return type is the property's type — the one its initialiser declares, or what the +/// getter's body returns when it has none — so what the reader would have written after the name +/// is what that getter returns. `None` when ty could not read a return type from it at all +pub fn inferred_property_type<'db>( + model: &SemanticModel<'db>, + getter: &ast::StmtFunctionDef, +) -> Option> { + let db = model.db(); + let index = semantic_index(db, model.program_file()); + // the getter is decorated with `property`, so its *binding* type is the descriptor — + // the undecorated function is what carries the signature the source wrote + let definition = index.try_definition(getter)?; + let function = infer_definition_types(db, definition).function_type(definition)?; + + let return_ty = function + .last_definition_raw_signature(db, ReturnCallableTypeVarScope::Public) + .return_ty; + + is_settled(db, &model.program_environment(), return_ty).then_some(return_ty) } /// The type worth showing for a parameter the source leaves unannotated. diff --git a/crates/ty_python_semantic/src/types/infer.rs b/crates/ty_python_semantic/src/types/infer.rs index 7536bf20aa..8695875897 100644 --- a/crates/ty_python_semantic/src/types/infer.rs +++ b/crates/ty_python_semantic/src/types/infer.rs @@ -55,6 +55,7 @@ use std::borrow::Cow; pub(super) use ty_python_core::frozen::{FrozenMap, FrozenSet, FrozenValueMap}; use crate::types::diagnostic::TypeCheckDiagnostics; +use crate::types::exceptions::CallSolution; use crate::types::function::{FunctionDecorators, FunctionType}; use crate::types::generics::Specialization; use crate::types::unpacker::{UnpackResult, Unpacker}; @@ -1087,6 +1088,10 @@ struct ScopeInferenceExtra<'db> { /// candidate's typevars adopts and locks the specialization. fluid_adoptions: FxHashMap>, + /// basedpython: what each call to a generic function solved that function's own type + /// parameters to — see [`CallSolution`]. + call_solutions: FxHashMap>, + /// The fallback type for missing expressions/bindings/declarations or recursive type inference. cycle_recovery: Option>, @@ -1095,6 +1100,15 @@ struct ScopeInferenceExtra<'db> { } impl<'db> ScopeInference<'db> { + /// basedpython: what `call` solved its callee's own type parameters to. + pub(crate) fn call_solution(&self, call: &ast::ExprCall) -> Option> { + self.extra + .as_deref()? + .call_solutions + .get(&ExpressionNodeKey::from(call)) + .copied() + } + fn cycle_initial(cycle_recovery: Type<'db>) -> Self { Self { extra: Some(Box::new(ScopeInferenceExtra { @@ -1499,6 +1513,10 @@ struct OtherDefinitionInferenceExtra<'db> { /// candidate's typevars adopts and locks the specialization. fluid_adoptions: FxHashMap>, + /// basedpython: what each call to a generic function solved that function's own type + /// parameters to — see [`CallSolution`]. + call_solutions: FxHashMap>, + /// The creation-time type of a fluid specialization candidate defined by this /// region, with literal types retained. fluid_creation: Option>, @@ -2021,6 +2039,10 @@ struct ExpressionInferenceExtra<'db> { /// candidate's typevars adopts and locks the specialization. fluid_adoptions: FxHashMap>, + /// basedpython: what each call to a generic function solved that function's own type + /// parameters to — see [`CallSolution`]. + call_solutions: FxHashMap>, + /// The creation-time type of a fluid specialization candidate whose assigned value /// is this region, with literal types retained. fluid_creation: Option>, @@ -2325,6 +2347,10 @@ struct StatementInferenceInnerExtra<'db> { /// candidate's typevars adopts and locks the specialization. fluid_adoptions: FxHashMap>, + /// basedpython: what each call to a generic function solved that function's own type + /// parameters to — see [`CallSolution`]. + call_solutions: FxHashMap>, + /// The fallback type for missing expressions/bindings/declarations or recursive type inference. cycle_recovery: Option>, diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index e8f87f036a..991956091a 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -10,8 +10,9 @@ use ruff_db::parsed::ParsedModuleRef; use ruff_db::source::source_text; use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::helpers::{ - BindingKeyword, TypeModifier, is_declaration_marker, is_dotted_name, - is_untyped_declaration_marker, statement_expression_values, untyped_declaration_context, + BindingKeyword, MemberVisibility, TypeModifier, is_declaration_marker, is_dotted_name, + is_let_marker_id, is_untyped_declaration_marker, statement_expression_values, + untyped_declaration_context, }; use ruff_python_ast::name::Name; use ruff_python_ast::{ @@ -77,10 +78,10 @@ use crate::types::diagnostic::{ self, AMBIGUOUS_EXTENSION_MEMBER, CALL_NON_CALLABLE, CONFLICTING_DECLARATIONS, CYCLIC_TYPE_ALIAS_DEFINITION, DYNAMIC_FUNCTION_DECORATOR_RETURN, ERASED_CAST_ARGUMENT, ERASED_TYPE_CHECK, FINAL_ON_VARIABLE, GeneratorMismatchKind, IMPLICIT_DECLARATION, - INEFFECTIVE_FINAL, INVALID_ARGUMENT_TYPE, INVALID_ASSIGNMENT, INVALID_ATTRIBUTE_ACCESS, - INVALID_DECLARATION, INVALID_ENUM_MEMBER_ANNOTATION, INVALID_FIELD_LOOKUP, - INVALID_LEGACY_TYPE_VARIABLE, INVALID_NEWTYPE, INVALID_PARAMSPEC, INVALID_REGEX, - INVALID_REIFIED_TYPE_PARAM, INVALID_TYPE_ALIAS_TYPE, INVALID_TYPE_FORM, + INACCESSIBLE_MEMBER, INEFFECTIVE_FINAL, INVALID_ARGUMENT_TYPE, INVALID_ASSIGNMENT, + INVALID_ATTRIBUTE_ACCESS, INVALID_DECLARATION, INVALID_ENUM_MEMBER_ANNOTATION, + INVALID_FIELD_LOOKUP, INVALID_LEGACY_TYPE_VARIABLE, INVALID_NEWTYPE, INVALID_PARAMSPEC, + INVALID_REGEX, INVALID_REIFIED_TYPE_PARAM, INVALID_TYPE_ALIAS_TYPE, INVALID_TYPE_FORM, INVALID_TYPE_VARIABLE_CONSTRAINTS, INVALID_TYPE_VARIABLE_DEFAULT, INVALID_VARIANCE_DECLARATION, NARROWING_GUARD_AS_VALUE, NON_EXHAUSTIVE_STATEMENT_EXPRESSION, NON_OVERLAPPING_CAST, NON_OVERLAPPING_TYPE_TEST, OPTIONAL_OBJECT_CONVERSION, POSSIBLY_MISSING_IMPLICIT_CALL, @@ -104,11 +105,12 @@ use crate::types::diagnostic::{ report_match_pattern_against_non_runtime_checkable_protocol, report_match_pattern_against_typed_dict, report_mismatched_type_name, report_possibly_missing_attribute, report_possibly_unresolved_reference, - report_private_constructor, report_too_many_positional_patterns_for_class_pattern, + report_restricted_constructor, report_too_many_positional_patterns_for_class_pattern, report_unplaceable_starred_class_pattern, report_unsound_assignment, report_unsound_yield, report_unsupported_augmented_assignment, report_unsupported_comparison, }; use crate::types::enums::{enum_ignored_names, is_enum_class_by_inheritance}; +use crate::types::exceptions::CallSolution; use crate::types::extensions; use crate::types::format; use crate::types::function::{ @@ -158,7 +160,9 @@ use crate::types::unpacker::{ UnpackResult, fixed_sequence_elements, sequence_from_literal_elements, tuple_literal_needs_promotion, }; -use crate::types::visibility::{private_constructor, scope_is_within_class}; +use crate::types::visibility::{ + AmbiguousAccess, MemberAccess, restricted_constructor, scope_is_within_class, +}; use crate::types::{ BindingContext, BoundTypeVarInstance, CallDunderError, CallableBinding, CallableType, CallableTypes, ClassType, DeferredOperation, DeferredType, DynamicType, GeneratorTypeMode, @@ -399,6 +403,10 @@ pub(super) struct TypeInferenceBuilder<'db, 'ast> { /// bidirectional type context, the contextual type. fluid_adoptions: FxHashMap>, + /// basedpython: what each call to a generic function solved that function's own type + /// parameters to — see [`CallSolution`]. + call_solutions: FxHashMap>, + /// basedpython `?.`: for each link of an optional chain, the type that link has /// when every `?.` receiver in the chain is present. /// @@ -651,6 +659,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { type_expression_flags: FxHashMap::default(), collection_use_constraints: FxHashMap::default(), fluid_adoptions: FxHashMap::default(), + call_solutions: FxHashMap::default(), basedpython_chain_present: FxHashMap::default(), basedpython_statement_expression_values: Vec::new(), fluid_creation: None, @@ -798,6 +807,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } self.fluid_adoptions.extend(extra.fluid_adoptions.iter()); + + self.call_solutions.extend(extra.call_solutions.iter()); } } } @@ -903,6 +914,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } self.fluid_adoptions.extend(extra.fluid_adoptions.iter()); + + self.call_solutions.extend(extra.call_solutions.iter()); } } @@ -947,6 +960,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { self.fluid_adoptions .extend(inference.fluid_adoptions.iter()); + self.call_solutions.extend(inference.call_solutions.iter()); if !matches!(self.region, InferenceRegion::Scope(..)) { self.bindings.extend( @@ -983,6 +997,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } self.fluid_adoptions.extend(extra.fluid_adoptions.iter()); + + self.call_solutions.extend(extra.call_solutions.iter()); } } @@ -2332,6 +2348,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { fn infer_module(&mut self, module: &ast::ModModule) { self.infer_body(&module.body); + // basedpython: a `private` symbol has no place in the module's interface + crate::types::visibility::check_private_exports(&self.context, &module.body); + // basedpython: a trailing-lambda block in a module-level loop that // captures a loop variable is a late-binding trap unless its callee // confines it (`local` / `once`) — the type-aware complement to `B023` @@ -3444,6 +3463,12 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { fn validate_class_pattern(&mut self, pattern: &ast::PatternMatchClass, cls_ty: Type<'db>) { let db = self.db(); let env = self.program_environment(); + // basedpython: `case A(x=...)` reads `x` off the subject exactly as `a.x` does + if self.context.is_lint_enabled(&INACCESSIBLE_MEMBER) { + for keyword in &pattern.arguments.keywords { + self.check_member_reach(&keyword.attr, cls_ty, keyword.attr.as_str()); + } + } // basedpython `case A(x, *_, y)`: the starred wildcard is not a subpattern of its own, // it only says that what follows it is counted back from the end of `__match_args__` let (starred, positional_patterns): (Vec<_>, Vec<_>) = pattern @@ -5553,7 +5578,10 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // class body a bare `final` assignment is a plain attribute, matching // `let`-in-class, so restrict this to non-class scopes. if let ast::Expr::Name(ann_name) = annotation - && ann_name.id.as_str() == "__modifier_assign__" + && ruff_python_ast::helpers::DeclarationMarker::from_id(ann_name.id.as_str()) + .is_some_and(|marker| { + marker.kind == ruff_python_ast::helpers::DeclarationMarkerKind::Assign + }) && let ast::Expr::Name(target_name) = target && self .index @@ -5633,9 +5661,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // exempts a `let` from the override-of-final check. no `Final` is emitted in // the lowered python either — read-only-ness is a type-checker-only marker let is_let_marker = match annotation { - ast::Expr::Name(n) => n.id.as_str() == "__let__", + ast::Expr::Name(n) => is_let_marker_id(n.id.as_str()), ast::Expr::Subscript(s) => { - matches!(s.value.as_ref(), ast::Expr::Name(n) if n.id.as_str() == "__let__") + matches!(s.value.as_ref(), ast::Expr::Name(n) if is_let_marker_id(n.id.as_str())) } _ => false, }; @@ -5950,6 +5978,12 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { if let Some(name_expr) = target.as_name_expr() && !name_expr.id.starts_with("__") && !matches!(name_expr.id.as_str(), "_ignore_" | "_value_" | "_name_") + // basedpython: a declaration marker with no type under it stands + // for a declaration that wrote none (`private A = 1`) + && !matches!( + ruff_python_ast::helpers::DeclarationMarker::of(annotation), + Some((_, None)) + ) && ( // Not bare Final (bare Final is allowed on enum members) !(declared.qualifiers.contains(TypeQualifiers::FINAL) @@ -12464,17 +12498,28 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } // basedpython: a `private` constructor may only be called from - // inside the body of the class that declares it. `type[A]` is left - // alone for the same reason a protocol is: the class it stands for - // may be a subclass that declares a constructor of its own. + // inside the body of the class that declares it, a `protected` one + // from a subclass's body as well. `type[A]` is left alone for the + // same reason a protocol is: the class it stands for may be a + // subclass that declares a constructor of its own. // the lint is asked first because answering the question at all // costs a `__init__` lookup on every construction in the program if !callable_type.is_subclass_of() && self.context.is_lint_enabled(&PRIVATE_CONSTRUCTOR) - && let Some(constructor) = private_constructor(db, class) - && !scope_is_within_class(db, self.index, self.scope(), constructor.owner) + && let Some(constructor) = restricted_constructor(db, class) + && !match constructor.visibility { + MemberVisibility::Protected => { + crate::types::visibility::scope_is_within_subclass_of( + db, + self.index, + self.scope(), + constructor.owner, + ) + } + _ => scope_is_within_class(db, self.index, self.scope(), constructor.owner), + } { - report_private_constructor(&self.context, call_expression, class, constructor); + report_restricted_constructor(&self.context, call_expression, class, constructor); } // Inference of correctly-placed `TypeVar`, `ParamSpec`, `NewType`, and @@ -12694,11 +12739,13 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { Ok(()) => bindings, Err(_) => { bindings.report_diagnostics(&self.context, call_expression.into()); + self.record_call_solution(call_expression, callable_type, None); let return_ty = bindings.return_type(self.db(), env); self.record_unsolved_typevar_call(call_expression, return_ty, &bindings); return return_ty; } }; + self.record_call_solution(call_expression, callable_type, Some(&bindings)); // Explicit function references already report implementation deprecations. // Other calls reference an object or class, not the implicitly invoked method. @@ -12996,6 +13043,56 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } } + /// basedpython: record what `call` solved its callee's own type parameters to, for the + /// exception analysis to specialize the callee's `raises` clause by — see + /// [`CallSolution`]. `bindings` is `None` for a call that did not bind. + fn record_call_solution( + &mut self, + call: &ast::ExprCall, + callable_type: Type<'db>, + bindings: Option<&Bindings<'db>>, + ) { + // exceptions are only followed through calls to functions and methods, and only + // in a `.by` body; nothing else ever reads this + if !self.is_basedpython_file() + || !matches!( + callable_type, + Type::FunctionLiteral(_) | Type::BoundMethod(_) + ) + { + return; + } + let solution = match bindings { + None => CallSolution::Unbound, + Some(bindings) => { + let env = self.context.program_environment(); + let Some(specialization) = bindings + .single_element() + .and_then(|callable| callable.matching_overloads().exactly_one().ok()) + .and_then(|(_, binding)| binding.merged_specialization(self.db(), env)) + else { + return; + }; + // a solution that substitutes nothing is left out, which is also what + // tells a recursive call that changes nothing from one that does + if specialization.substitutes_nothing(self.db()) { + return; + } + CallSolution::Solved(specialization) + } + }; + self.call_solutions + .insert(ExpressionNodeKey::from(call), solution); + } + + /// basedpython: what `call` solved its callee's own type parameters to, as recorded so + /// far in this region. + fn call_solution(&self, call: &ast::ExprCall) -> Option> { + self.call_solutions + .get(&ExpressionNodeKey::from(call)) + .copied() + } + /// basedpython: remember a call that only returns `Never` because it left a type variable /// unsolved, so that reachability analysis does not read it as a call that never returns. fn record_unsolved_typevar_call( @@ -13751,6 +13848,14 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { .info("the block is called as `fn()`, which leaves `it` at its `None` default"); } + // basedpython: a class body read that may reach a module's `private` name + crate::types::visibility::check_private_class_read( + &self.context, + self.index, + self.scope(), + name_node, + ); + let expr = PlaceExpr::from_expr_name(name_node); let (resolved, _) = self.infer_place_load(expr, ast::ExprRef::Name(name_node)); @@ -15196,11 +15301,16 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } = attribute; match ctx { - ExprContext::Load => self - .infer_attribute_load(attribute) - .unwrap_or_else(|recovery_ty| recovery_ty), + ExprContext::Load => { + let member_type = self + .infer_attribute_load(attribute) + .unwrap_or_else(|recovery_ty| recovery_ty); + self.validate_member_visibility(attribute, self.expression_type(value)); + member_type + } ExprContext::Store => { self.infer_expression(value, TypeContext::default()); + self.validate_member_visibility(attribute, self.expression_type(value)); Type::Never } ExprContext::Del => { @@ -15211,6 +15321,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { attr.as_str(), true, ); + self.validate_member_visibility(attribute, self.expression_type(value)); Type::Never } ExprContext::Invalid => { @@ -15220,6 +15331,112 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } } + /// basedpython: `a.x` where `x` carries a visibility keyword and the access is + /// written outside the code that keyword admits — anywhere but the declaring + /// class's own body for `private`, anywhere but that and a subclass's body for + /// `protected` — or where the classes `a` can be disagree about `x` + fn validate_member_visibility(&self, attribute: &ast::ExprAttribute, receiver: Type<'db>) { + // answering costs a member lookup at every attribute access in the + // program, so the lint is asked first + if !self.context.is_lint_enabled(&INACCESSIBLE_MEMBER) { + return; + } + let db = self.db(); + // a module's `private` symbol reached as an attribute of the module from + // another one. `from m import x` is `private-import`; this is the same + // boundary crossed without an import statement, and the lowering has + // renamed the symbol, so the attribute is not there at runtime either + if let Type::ModuleLiteral(module) = receiver + && let Some(module_file) = module.module(db).file(db) + && module_file != self.file() + && crate::types::visibility::private_symbols(db, module_file) + .contains(&attribute.attr.id) + { + if let Some(builder) = self.context.report_lint(&INACCESSIBLE_MEMBER, attribute) { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`{name}` is private to module `{module}`", + name = attribute.attr.id, + module = module.module(db).name(db), + )); + diagnostic.info( + "the lowering renames it with a leading underscore, which is not the name \ + written here", + ); + } + return; + } + self.check_member_reach(attribute, receiver, attribute.attr.as_str()); + } + + /// basedpython: reports `name`, reached through `receiver` at `node`, when a + /// visibility keyword keeps it from being reached there — the one check an + /// attribute access and a class pattern's keyword share + fn check_member_reach( + &self, + node: impl ruff_text_size::Ranged, + receiver: Type<'db>, + name: &str, + ) { + let db = self.db(); + let env = self.program_environment(); + let accesses = match crate::types::visibility::member_access(db, env, receiver, name) { + Ok(accesses) => accesses, + Err(AmbiguousAccess) => { + if let Some(builder) = self.context.report_lint(&INACCESSIBLE_MEMBER, node) { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`{name}` is emitted under a different name on the classes `{receiver}` \ + can be", + receiver = receiver.display(db, env), + )); + diagnostic.info("no single access reaches all of them"); + } + return; + } + }; + for access in accesses { + let MemberAccess::Restricted { visibility, owner } = access else { + continue; + }; + let reachable = match visibility { + MemberVisibility::Public => true, + MemberVisibility::Private => { + scope_is_within_class(db, self.index, self.scope(), owner) + } + MemberVisibility::Protected => { + crate::types::visibility::scope_is_within_subclass_of( + db, + self.index, + self.scope(), + owner, + ) + } + }; + if reachable { + continue; + } + let Some(builder) = self.context.report_lint(&INACCESSIBLE_MEMBER, node) else { + return; + }; + let owner_name = owner.name(db); + let mut diagnostic = match visibility { + MemberVisibility::Private => { + builder.into_diagnostic(format_args!("`{name}` is private to `{owner_name}`")) + } + _ => builder.into_diagnostic(format_args!( + "`{name}` is protected: only `{owner_name}` and its subclasses may reach it" + )), + }; + if let Some(emitted) = + crate::types::visibility::emitted_member_name(db, visibility, owner, name) + { + diagnostic.info(format_args!( + "the lowering emits it as `{emitted}`, which is not the name written here" + )); + } + return; + } + } + fn report_unsupported_unary_operator( &self, unary: &ast::ExprUnaryOp, @@ -16275,6 +16492,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { type_expression_flags, collection_use_constraints, fluid_adoptions, + call_solutions, fluid_creation, fluid_timeline, string_annotations, @@ -16325,6 +16543,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { type_expression_flags, collection_use_constraints, fluid_adoptions, + call_solutions, fluid_creation, fluid_timeline, string_annotations, @@ -16351,6 +16570,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { fluid_creation: _, fluid_timeline: _, mut fluid_adoptions, + mut call_solutions, mut collection_use_constraints, string_annotations, expected_types, @@ -16395,14 +16615,17 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { || !qualifiers.is_empty() || !type_expression_flags.is_empty() || !collection_use_constraints.is_empty() - || !fluid_adoptions.is_empty()) + || !fluid_adoptions.is_empty() + || !call_solutions.is_empty()) .then(|| { collection_use_constraints.shrink_to_fit(); fluid_adoptions.shrink_to_fit(); + call_solutions.shrink_to_fit(); return_types_and_ranges.shrink_to_fit(); Box::new(StatementInferenceInnerExtra { string_annotations: FrozenSet::from(string_annotations), fluid_adoptions, + call_solutions, expected_types: FrozenMap::from(expected_types), called_functions: called_functions .into_iter() @@ -16494,6 +16717,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { fluid_creation: _, fluid_timeline: _, fluid_adoptions: _, + call_solutions: _, collection_use_constraints: _, dataclass_field_specifiers: _, slice_materialization: _, @@ -16541,6 +16765,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { fluid_creation, fluid_timeline, mut fluid_adoptions, + mut call_solutions, mut collection_use_constraints, string_annotations, expected_types, @@ -16577,6 +16802,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { + usize::from(!expected_types.is_empty()) + usize::from(!collection_use_constraints.is_empty()) + usize::from(!fluid_adoptions.is_empty()) + + usize::from(!call_solutions.is_empty()) + usize::from(fluid_creation.is_some()) + usize::from(fluid_timeline.is_some()) + usize::from(!called_functions.is_empty()) @@ -16630,10 +16856,12 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { (_, undecorated_type) => { collection_use_constraints.shrink_to_fit(); fluid_adoptions.shrink_to_fit(); + call_solutions.shrink_to_fit(); let extra = OtherDefinitionInferenceExtra { string_annotations: FrozenSet::from(string_annotations), expected_types: FrozenMap::from(expected_types), fluid_adoptions, + call_solutions, collection_use_constraints, fluid_creation, fluid_timeline, @@ -16698,6 +16926,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { fluid_creation: _, fluid_timeline: _, mut fluid_adoptions, + mut call_solutions, mut collection_use_constraints, expressions, comparison_truthiness: _, @@ -16740,10 +16969,12 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { || !type_expression_flags.is_empty() || !collection_use_constraints.is_empty() || !qualifiers.is_empty() - || !fluid_adoptions.is_empty()) + || !fluid_adoptions.is_empty() + || !call_solutions.is_empty()) .then(|| { collection_use_constraints.shrink_to_fit(); fluid_adoptions.shrink_to_fit(); + call_solutions.shrink_to_fit(); Box::new(ScopeInferenceExtra { string_annotations: FrozenSet::from(string_annotations), qualifiers: FrozenMap::from(qualifiers), @@ -16751,6 +16982,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { type_expression_flags: FrozenMap::from(type_expression_flags), collection_use_constraints, fluid_adoptions, + call_solutions, cycle_recovery, diagnostics, }) @@ -16791,6 +17023,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { fluid_creation: _, fluid_timeline: _, fluid_adoptions: _, + call_solutions: _, basedpython_chain_present: _, basedpython_statement_expression_values: _, collection_use_constraints: _, @@ -16863,6 +17096,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { fluid_creation: _, fluid_timeline: _, fluid_adoptions, + call_solutions, collection_use_constraints, string_annotations, unsolved_typevar_calls, @@ -16926,6 +17160,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } self.fluid_adoptions.extend(fluid_adoptions); + self.call_solutions.extend(call_solutions); // adopting the speculative builder's expression types means adopting the optional-chain // provenance of those same expressions, or a chain this region goes on to extend would @@ -17049,6 +17284,7 @@ struct FullExpressionCacheEntry<'db> { type_expression_flags: FxHashMap, collection_use_constraints: CollectionUseConstraints<'db>, fluid_adoptions: FxHashMap>, + call_solutions: FxHashMap>, fluid_creation: Option>, fluid_timeline: Option>, string_annotations: FxHashSet, @@ -17078,6 +17314,7 @@ impl<'db> FullExpressionCacheEntry<'db> { && self.type_expression_flags.is_empty() && self.collection_use_constraints.is_empty() && self.fluid_adoptions.is_empty() + && self.call_solutions.is_empty() && self.fluid_creation.is_none() && self.fluid_timeline.is_none() && self.string_annotations.is_empty() @@ -17099,6 +17336,7 @@ impl<'db> FullExpressionCacheEntry<'db> { || !self.type_expression_flags.is_empty() || !self.collection_use_constraints.is_empty() || !self.fluid_adoptions.is_empty() + || !self.call_solutions.is_empty() || self.fluid_creation.is_some() || self.fluid_timeline.is_some() || !self.expected_types.is_empty() @@ -17118,11 +17356,13 @@ impl<'db> FullExpressionCacheEntry<'db> { self.collection_use_constraints.shrink_to_fit(); self.fluid_adoptions.shrink_to_fit(); + self.call_solutions.shrink_to_fit(); self.diagnostics.shrink_to_fit(); Box::new(ExpressionInferenceExtra { string_annotations: FrozenSet::from(self.string_annotations), unsolved_typevar_calls: FrozenSet::from(self.unsolved_typevar_calls), fluid_adoptions: self.fluid_adoptions, + call_solutions: self.call_solutions, fluid_creation: self.fluid_creation, fluid_timeline: self.fluid_timeline, comparison_truthiness: FrozenMap::from(self.comparison_truthiness), diff --git a/crates/ty_python_semantic/src/types/infer/builder/annotation_expression.rs b/crates/ty_python_semantic/src/types/infer/builder/annotation_expression.rs index 54385802bd..3232f4f8f3 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/annotation_expression.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/annotation_expression.rs @@ -1,5 +1,8 @@ use ruff_python_ast as ast; -use ruff_python_ast::helpers::{is_dotted_name, type_modifier_marker}; +use ruff_python_ast::helpers::{ + DeclarationMarker, DeclarationMarkerKind, MemberVisibility, is_dotted_name, + type_modifier_marker, +}; use ty_python_core::scope::ScopeKind; use super::{DeferredExpressionState, TypeInferenceBuilder}; @@ -13,6 +16,16 @@ use crate::types::{ SpecialFormType, Type, TypeAndQualifiers, TypeContext, TypeQualifier, TypeQualifiers, todo_type, }; +/// basedpython: the qualifier a declaration marker's visibility contributes, +/// empty for a marker that records none +fn visibility_qualifiers(visibility: MemberVisibility) -> TypeQualifiers { + match visibility { + MemberVisibility::Public => TypeQualifiers::empty(), + MemberVisibility::Protected => TypeQualifiers::PROTECTED, + MemberVisibility::Private => TypeQualifiers::PRIVATE, + } +} + #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(super) enum PEP613Policy { Allowed, @@ -472,73 +485,77 @@ impl<'db> TypeInferenceBuilder<'db, '_> { == ScopeKind::Class; match annotation { - ast::Expr::Name(name) => match name.id.as_str() { - "__let__" => { - let qualifiers = if in_class_scope { - TypeQualifiers::empty() - } else { - TypeQualifiers::FINAL - }; - Some(TypeAndQualifiers::new( - Type::unknown(), - TypeOrigin::Declared, - qualifiers, - )) + ast::Expr::Name(name) => { + if name.id.as_str() == "__sentinel__" { + return Some(TypeAndQualifiers::declared(Type::unknown())); } - "__classvar__" => Some(TypeAndQualifiers::new( + let marker = DeclarationMarker::from_id(name.id.as_str())?; + let visibility = visibility_qualifiers(marker.visibility); + let qualifiers = match marker.kind { + // `let x = v` — read-only, which python spells `Final` outside a + // class body + DeclarationMarkerKind::Let if in_class_scope => TypeQualifiers::empty(), + DeclarationMarkerKind::Let => TypeQualifiers::FINAL, + // `class x = v` + DeclarationMarkerKind::ClassVar => TypeQualifiers::CLASS_VAR, + // `[modifiers] x = v`, and the `init(private var a)` parameter + // that declares nothing but the attribute's visibility: the type + // is left where a bare `let a` leaves it + DeclarationMarkerKind::Annot | DeclarationMarkerKind::Assign => { + TypeQualifiers::empty() + } + // these always wrap a declared type + DeclarationMarkerKind::Final | DeclarationMarkerKind::ClassVarAnnot => { + return None; + } + }; + Some(TypeAndQualifiers::new( Type::unknown(), TypeOrigin::Declared, - TypeQualifiers::CLASS_VAR, - )), - "__modifier_assign__" => Some(TypeAndQualifiers::declared(Type::unknown())), - "__sentinel__" => Some(TypeAndQualifiers::declared(Type::unknown())), - _ => None, - }, + qualifiers | visibility, + )) + } ast::Expr::Subscript(ast::ExprSubscript { value, slice, .. }) => { let ast::Expr::Name(value_name) = value.as_ref() else { return None; }; // every modifier declaration keeps the declared type as the slice - let always_final = match value_name.id.as_str() { - // typed `let x: T = v` / `final x: T = v`. - // `final` is `Final` everywhere; `let` only at module scope - "__let__" => false, - "__final__" => true, - // modifiers ty places no meaning on (`override x: T`, - // `abstract x: T`, `private x: T`): the declaration is just `x: T` - // `class var x: T` — a class variable whose type is - // declared rather than read off a value - "__classvar_annot__" => { + let marker = DeclarationMarker::from_id(value_name.id.as_str())?; + let visibility = visibility_qualifiers(marker.visibility); + let always_final = match marker.kind { + // typed `let x: T = v` / `final x: T = v`. `final` is `Final` + // everywhere; `let` only outside a class body + DeclarationMarkerKind::Let => false, + DeclarationMarkerKind::Final => true, + // `class var x: T` — a class variable whose type is declared + // rather than read off a value + DeclarationMarkerKind::ClassVarAnnot => { return Some(TypeAndQualifiers::new( self.infer_type_expression(slice), TypeOrigin::Declared, - TypeQualifiers::CLASS_VAR, - )); - } - "__modifier_annot__" | "__abstract_annot__" | "__visibility_annot__" => { - return Some(TypeAndQualifiers::declared( - self.infer_type_expression(slice), + TypeQualifiers::CLASS_VAR | visibility, )); } - // `private x: T` — the declaration is `x: T`, but the privacy - // rides along as a qualifier: a private member is invisible to - // a widened view of its class, which is what makes it sound - // under a covariant type parameter - "__private_annot__" => { + // modifiers ty places no meaning on (`override x: T`, `abstract + // x: T`): the declaration is just `x: T`. a visibility keyword + // rides along as a qualifier — it decides who may reach the + // member + DeclarationMarkerKind::Annot => { return Some(TypeAndQualifiers::new( self.infer_type_expression(slice), TypeOrigin::Declared, - TypeQualifiers::PRIVATE, + visibility, )); } - _ => return None, + DeclarationMarkerKind::ClassVar | DeclarationMarkerKind::Assign => return None, }; let inner = self.infer_type_expression(slice); - let qualifiers = if always_final || !in_class_scope { + let mut qualifiers = if always_final || !in_class_scope { TypeQualifiers::FINAL } else { TypeQualifiers::empty() }; + qualifiers |= visibility; Some(TypeAndQualifiers::new( inner, TypeOrigin::Declared, diff --git a/crates/ty_python_semantic/src/types/infer/builder/attribute_assignment.rs b/crates/ty_python_semantic/src/types/infer/builder/attribute_assignment.rs index 1458db9e56..220d6a65c0 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/attribute_assignment.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/attribute_assignment.rs @@ -41,6 +41,12 @@ impl<'db> TypeInferenceBuilder<'db, '_> { let env = self.program_environment(); let db = self.db(); + // basedpython: a visibility keyword draws the same boundary around a + // write as around a read + if emit_diagnostics { + self.validate_member_visibility(target, object_ty); + } + // basedpython use-site variance: writes to an attribute typed with a // covariantly-projected (`out`) typevar are statically rejected. The // attribute's declared type on the unspecialized class is inspected diff --git a/crates/ty_python_semantic/src/types/infer/builder/function.rs b/crates/ty_python_semantic/src/types/infer/builder/function.rs index 08601f8f2c..b5e0ff9e2c 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/function.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/function.rs @@ -15,7 +15,8 @@ use crate::{ UNSOUND_RETURN_STATEMENT, USELESS_OVERLOAD_BODY, add_type_expression_reference_link, is_invalid_typed_dict_literal, report_bool_as_int, report_implicit_return_type, report_invalid_generator_function_return_type, report_invalid_return_type, - report_shadowed_type_variable, report_unsound_return_statement, + report_missing_function_body, report_shadowed_type_variable, + report_unsound_return_statement, }, extensions, function::{ @@ -57,7 +58,7 @@ use ty_python_core::{ use ruff_db::diagnostic::{Annotation, Span}; use ruff_db::parsed::parsed_module; use ruff_python_ast as ast; -use ruff_python_ast::helpers::{ReturnGuardForm, return_guards}; +use ruff_python_ast::helpers::{MemberVisibility, ReturnGuardForm, return_guards}; use ruff_text_size::Ranged; use rustc_hash::FxHashSet; @@ -67,6 +68,24 @@ fn parameters_have_defaults(parameters: &ast::Parameters) -> bool { .any(|param| param.default.is_some()) } +/// basedpython: whether the parser built this function node out of a construct that is not a +/// `def` at all — an `init(...)`, a property accessor block, or a trailing-lambda block. Each of +/// those records the form it was written as with a synthetic marker decorator. +fn is_synthesized_from_another_construct(function: &ast::StmtFunctionDef) -> bool { + function.is_trailing_lambda + || function.decorator_list.iter().any(|decorator| { + matches!( + &decorator.expression, + ast::Expr::Name(marker) + if matches!(marker.ctx, ast::ExprContext::Invalid) + && matches!( + marker.id.as_str(), + "__init_method__" | "__property__" | "__static_property__" + ) + ) + }) +} + fn function_has_deferred_annotations(function: &ast::StmtFunctionDef) -> bool { function.type_params.is_none() && (function.returns.is_some() @@ -298,6 +317,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { self.scope(), self.index.expect_single_definition(function), |expr| self.try_expression_type(expr).unwrap_or_else(Type::unknown), + |call| self.call_solution(call), ); // basedpython: a trailing-lambda block in a loop that captures a loop @@ -314,6 +334,10 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // with nothing behind it self.check_redundant_return_annotation(function); + // basedpython: a `def` written with no body at all declares a signature. where the + // position asks for an implementation, the missing body is the whole of the problem + self.check_missing_function_body(function); + let enclosing_function_for_return_check = nearest_enclosing_function(db, self.index, self.scope()); @@ -327,6 +351,18 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let inherited_return_range = function.returns.as_ref().map_or_else( || { let enclosing = enclosing_function_for_return_check?; + // basedpython: the getter of an untyped property with an initialiser is held to + // the type that initialiser declares, as it would be to a written one. the + // property's name is where that type was stated + if function.property_construct_range().is_some() + && enclosing + .literal(db) + .last_definition + .property_initialiser_type(db) + .is_some() + { + return Some(function.name.range()); + } let (overloads, implementation) = enclosing.overloads_and_implementation(db); if overloads.is_empty() || implementation.is_none() { return None; @@ -349,6 +385,15 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let mut enclosing_class_context = None; if has_empty_body { + // basedpython: a `def` written with no body at all is + // `missing-function-body`'s to report, whatever it declares it returns. it + // has no body to check a return type against, and where the position does + // not ask for a declaration the body is what went missing — saying instead + // that the function returns `None` describes the consequence, and says + // nothing at all when the return type is `None` already + if function.body.is_empty() { + return; + } if self.in_stub() { return; } @@ -358,19 +403,6 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { if self.is_in_type_checking_block(self.scope(), function) { return; } - // basedpython: a bodyless `def f(...) -> T` in a run of same-name defs is - // an implicit overload stub — the lowering writes the `@overload` - // decorators the source leaves out — so it is exempt for the same reason a - // written `@overload` is. one that is *not* in such a run declares an - // ordinary function, and the `: ...` the lowering fills in returns `None` - // exactly as a written one would - if function.body.is_empty() - && enclosing_function_for_return_check.is_some_and(|enclosing| { - enclosing.literal(db).last_definition.is_overload(db) - }) - { - return; - } if let Some(class) = self.class_context_of_current_method() { enclosing_class_context = Some(class); if class.is_protocol(db) { @@ -588,6 +620,61 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } } + /// basedpython: report a `def` written with no body at all, where the position asks for an + /// implementation. + /// + /// The lowering fills the missing body in with `: ...`, so a `def` that declares nothing but + /// a signature still produces a function — one that returns `None`. That is what a stub + /// file, a protocol, an `abstract def`, an overload group and an `if TYPE_CHECKING` block + /// each ask for. Everywhere else the implementation the signature promises was never + /// written, and the `: ...` stands in for it silently. + fn check_missing_function_body(&self, function: &ast::StmtFunctionDef) { + if !function.body.is_empty() || !self.is_basedpython_file() { + return; + } + // only a `def` the source wrote can be missing a body. every other construct the + // parser builds a function out of is given the body its own form means — an + // `init(...)` gets one built from its attribute parameters, an accessor block gets + // the accessors — so an empty one reaching here failed to parse, which has been + // reported already. a `decorator def` is not one of them: it is a `def` with a + // keyword in front, and the dispatcher its lowering writes calls the body the source + // is supposed to supply + if is_synthesized_from_another_construct(function) { + return; + } + if self.bodyless_def_declares_a_signature(function) { + return; + } + report_missing_function_body(&self.context, function); + } + + /// basedpython: whether a `def` written with no body at all declares a signature in this + /// position, rather than leaving out an implementation. + fn bodyless_def_declares_a_signature(&self, function: &ast::StmtFunctionDef) -> bool { + let db = self.db(); + + if self.in_stub() || self.is_in_type_checking_block(self.scope(), function) { + return true; + } + + // `@overload` and `@abstractmethod`, which the `abstract` modifier resolves to + if self.in_function_overload_or_abstractmethod() { + return true; + } + + // a bodyless `def` in a run of same-name defs is an overload declaration: the lowering + // writes the `@overload` decorator the source leaves out, so it declares a signature + // for the same reason a written one does + if nearest_enclosing_function(db, self.index, self.scope()) + .is_some_and(|enclosing| enclosing.literal(db).last_definition.is_overload(db)) + { + return true; + } + + self.class_context_of_current_method() + .is_some_and(|class| class.is_protocol(db)) + } + /// basedpython: report an explicit `-> None` that leaves the function's type exactly where /// deleting it would. /// @@ -614,6 +701,12 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { return; } + // basedpython: a property getter's return annotation is the type its declaration wrote. + // `let n: None` states the property's type, and there is no `-> None` to remove + if function.property_construct_range().is_some() { + return; + } + // everything below reads the class MRO and the enclosing overload chain, so don't // pay for it when nothing will be reported if !self.context.is_lint_enabled(&REDUNDANT_RETURN_ANNOTATION) { @@ -857,7 +950,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let mut function_decorators = FunctionDecorators::empty(); let mut dataclass_transformer_params = None; let mut final_decorator = None; - let mut private_modifier = None; + let mut visibility_modifier = None; for decorator in decorator_list { // basedpython: a trailing lambda block's synthetic decorator holds the @@ -877,20 +970,33 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // effect — and would otherwise resolve to `Unknown` and poison the // function type. (`final`/`abstract`/`static`/… map to real stdlib // decorators via `synthetic_decorator_target_type` and are kept.) - // `private` has no decorator either, but it *is* recorded: privacy is - // what makes a class's variance safe, so ty has to know about it + // `private` and `protected` have no decorator either, but they *are* + // recorded: they decide who may reach the method, and a member no + // widened view can reach is what makes a class's variance safe if let ast::Expr::Name(n) = &decorator.expression && matches!(n.ctx, ast::ExprContext::Invalid) && matches!( n.id.as_str(), // `__init_method__` marks the `init(...)` shorthand — a plain // `__init__`, so the synthetic marker is dropped too - "decorator_keyword" | "private" | "export" | "open" | "__init_method__" + "decorator_keyword" + | "private" + | "protected" + | "export" + | "open" + | "__init_method__" ) { - if n.id.as_str() == "private" { - function_decorators |= FunctionDecorators::PRIVATE; - private_modifier = Some(decorator); + match n.id.as_str() { + "private" => { + function_decorators |= FunctionDecorators::PRIVATE; + visibility_modifier = Some((MemberVisibility::Private, decorator)); + } + "protected" => { + function_decorators |= FunctionDecorators::PROTECTED; + visibility_modifier = Some((MemberVisibility::Protected, decorator)); + } + _ => {} } continue; } @@ -979,10 +1085,10 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { diagnostic.info("`@final` is only meaningful on methods and classes"); } - // basedpython: a `private` the lowering cannot act on hides nothing. it is - // reported here rather than left to the lowering, which can only silently - // do nothing with it - if let Some(private_modifier) = private_modifier + // basedpython: a visibility keyword the lowering cannot act on hides + // nothing. it is reported here rather than left to the lowering, which can + // only silently do nothing with it + if let Some((visibility, visibility_modifier)) = visibility_modifier && !ruff_python_stdlib::basedpython::private_mangles(&name.id) && name.id != "__init__" && self @@ -992,17 +1098,19 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { .is_class() && let Some(builder) = self .context - .report_lint(&INEFFECTIVE_PRIVATE, private_modifier) + .report_lint(&INEFFECTIVE_PRIVATE, visibility_modifier) { + let keyword = visibility.keyword(); let mut diagnostic = builder.into_diagnostic(format_args!( - "`private` has no effect on `{name}`", + "`{keyword}` has no effect on `{name}`", name = name.id )); - diagnostic.info( - "`private` renames a member to `__` so python's name-mangling hides it, \ - and python mangles only a name with at most one trailing underscore", - ); - diagnostic.info("`init` is the one dunder `private` says something about"); + diagnostic.info(format_args!( + "`{keyword}` renames a member to `{prefix}`, and a dunder is called by \ + the exact name python knows it under", + prefix = visibility.name_prefix(), + )); + diagnostic.info("`init` is the one dunder a visibility keyword says something about"); } // basedpython: a classmethod cannot have reified type parameters — the diff --git a/crates/ty_python_semantic/src/types/infer/builder/imports.rs b/crates/ty_python_semantic/src/types/infer/builder/imports.rs index df38d09ef5..69c7e7c00a 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/imports.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/imports.rs @@ -242,6 +242,30 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { return; } + // basedpython: `import a.b` binds its top-level package `a`, and no alias + // keeps that binding, so it cannot rebind a module-level `private a` under + // the underscored name the lowering gives it + if asname.is_none() + && let Some((top, _)) = name.as_str().split_once('.') + && definition + .scope(self.db()) + .file_scope_id(self.db()) + .is_global() + && crate::types::visibility::private_symbols(self.db(), self.file()) + .contains(&ruff_python_ast::name::Name::new(top)) + && let Some(builder) = self + .context + .report_lint(&crate::types::diagnostic::INVALID_VISIBILITY, alias) + { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`import {name}` rebinds `{top}`, which this module declares `private`" + )); + diagnostic.info(format_args!( + "a dotted import binds its top-level package, which no alias can keep under \ + `_{top}`: write `import {name} as ...` instead" + )); + } + // The name of the module being imported let Some(full_module_name) = ModuleName::new(name) else { tracing::debug!("Failed to resolve import due to invalid syntax"); diff --git a/crates/ty_python_semantic/src/types/lifetimes.rs b/crates/ty_python_semantic/src/types/lifetimes.rs index 09d13e6a19..a932dfd64b 100644 --- a/crates/ty_python_semantic/src/types/lifetimes.rs +++ b/crates/ty_python_semantic/src/types/lifetimes.rs @@ -24,7 +24,7 @@ use ruff_db::parsed::ParsedModuleRef; use ruff_db::source::source_text; -use ruff_python_ast::helpers::parameter_modifiers; +use ruff_python_ast::helpers::{is_final_marker_id, is_let_marker_id, parameter_modifiers}; use ruff_python_ast::statement_visitor::{StatementVisitor, walk_stmt}; use ruff_python_ast::visitor::{Visitor, walk_expr}; use ruff_python_ast::{self as ast, Expr, ExprContext, ExprName, ParameterBorrow, Stmt}; @@ -1038,7 +1038,8 @@ fn final_declaration_range(body: &[ast::Stmt], name: &str) -> Option return None; }; (target.id.as_str() == name - && matches!(marker(&ann.annotation), Some("__let__" | "__final__"))) + && marker(&ann.annotation) + .is_some_and(|id| is_let_marker_id(id) || is_final_marker_id(id))) .then(|| target.range()) }) } diff --git a/crates/ty_python_semantic/src/types/overrides.rs b/crates/ty_python_semantic/src/types/overrides.rs index b06406ecf0..0b5ff15687 100644 --- a/crates/ty_python_semantic/src/types/overrides.rs +++ b/crates/ty_python_semantic/src/types/overrides.rs @@ -9,6 +9,7 @@ use ruff_db::{ files::FileRange, parsed::{ParsedModuleRef, parsed_module}, }; +use ruff_python_ast::helpers::{MemberVisibility, is_let_marker_id}; use ruff_python_ast::{self as ast, PythonVersion, name::Name}; use ruff_python_stdlib::identifiers::is_mangled_private; use rustc_hash::FxHashSet; @@ -29,8 +30,9 @@ use crate::{ INVALID_EXPLICIT_OVERRIDE, INVALID_METHOD_OVERRIDE, INVALID_NAMED_TUPLE, INVALID_NAMED_TUPLE_OVERRIDE, MISSING_OVERRIDE_DECORATOR, OVERRIDE_OF_FINAL_METHOD, OVERRIDE_OF_FINAL_VARIABLE, report_incompatible_base_method, - report_invalid_method_override, report_invalid_reified_override, - report_overridden_final_method, report_overridden_final_variable, + report_invalid_method_override, report_invalid_override_visibility, + report_invalid_reified_override, report_overridden_final_method, + report_overridden_final_variable, }, enums::{EnumMetadata, enum_metadata, is_enum_class_by_inheritance}, function::{FunctionDecorators, FunctionType, KnownFunction, OverloadLiteral}, @@ -451,6 +453,25 @@ fn check_class_declaration<'db>( let instance_of_class = Type::instance(db, env, class); let subclass_instance_member = instance_of_class.member(db, env, &member.name); + + let Some((literal, _)) = class.static_class_literal(db) else { + return; + }; + let class_kind = CodeGeneratorKind::from_class(db, literal.into()); + + // basedpython: a visibility keyword where the member's name is part of how + // the class works at runtime, or where it cannot hide the name. asked before + // the instance lookup below, which a typed dict's key does not answer + check_member_visibility( + context, + class, + class_kind, + enum_info, + member, + *first_reachable_definition, + subclass_instance_member.qualifiers, + ); + let Place::Defined(DefinedPlace { ty: type_on_subclass_instance, .. @@ -459,11 +480,6 @@ fn check_class_declaration<'db>( return; }; - let Some((literal, _)) = class.static_class_literal(db) else { - return; - }; - let class_kind = CodeGeneratorKind::from_class(db, literal.into()); - // Check for prohibited `NamedTuple` attribute overrides. // // `NamedTuple` classes have certain synthesized attributes (like `_asdict`, `_make`, etc.) @@ -626,7 +642,38 @@ fn check_class_declaration<'db>( let mut missing_override_target: Option> = None; let mut overridden_final_method = None; let mut overridden_final_variable: Option<(ClassType<'db>, Option>)> = None; - let is_private_member = is_mangled_private(member.name.as_str()); + // basedpython: a `private` member is emitted under a name python mangles per + // class, so it is a member of its own rather than an override of anything. a + // `protected` one keeps one name across the hierarchy, so it overrides like + // any other member + let is_private_member = is_private_to(db, class, &member.name); + + // basedpython: a member emitted under a different name from the member it + // inherits under the same written name does not override it — it sits + // beside it, and the inherited one still answers. that is a keyword declaring + // it narrower than what it inherits, and a plain declaration over an + // inherited `protected` one. reported before the walk, because a private + // member's walk does not happen + let changed_visibility = inherited_member_visibility(db, env, class, &member.name).and_then( + |(superclass, inherited)| { + let declared = own_member_visibility(db, class, &member.name); + (inherited != MemberVisibility::Private + && emitted_name(&member.name, declared) != emitted_name(&member.name, inherited)) + .then_some((superclass, declared, inherited)) + }, + ); + if let Some((superclass, declared, inherited)) = changed_visibility { + report_invalid_override_visibility( + context, + &member.name, + *first_reachable_definition, + class, + superclass, + declared, + inherited, + ); + } + let mut subclass_variable_kind: Option> = None; // Track the first superclass that defines this method (the "immediate parent" for this method). @@ -654,6 +701,14 @@ fn check_class_declaration<'db>( ClassBase::Class(class) => class, }; + // basedpython: a `private` member of a superclass is emitted under a + // name mangled with that superclass's own, so nothing a subclass + // declares can override it. it is not a contract for this member to + // meet, and the subclass's member is not missing an `@override` + if is_private_to(db, superclass, &member.name) { + continue; + } + let Some((superclass_literal, superclass_specialization)) = superclass.static_class_literal(db) else { @@ -1006,12 +1061,15 @@ fn check_class_declaration<'db>( if !subclass_overrides_superclass_declaration && !has_dynamic_superclass + && changed_visibility.is_none() && ( // accessing `.kind()` here is fine as `definition` // will always be a definition in the file currently being checked first_reachable_definition.kind(db).is_function_def() ) { + // a member whose visibility was already reported as narrowed is not also + // told that it overrides nothing: it is the narrowing that stopped it check_explicit_overrides(context, member, class_scope, class); } @@ -1314,9 +1372,9 @@ fn is_let_declaration<'db>(db: &'db dyn Db, scope: ScopeId<'db>, symbol: ScopedS /// `__let__[T]`), which the forward transform emits for a `let` declaration. fn is_let_marker(annotation: &ast::Expr) -> bool { match annotation { - ast::Expr::Name(name) => name.id.as_str() == "__let__", + ast::Expr::Name(name) => is_let_marker_id(name.id.as_str()), ast::Expr::Subscript(subscript) => { - matches!(subscript.value.as_ref(), ast::Expr::Name(name) if name.id.as_str() == "__let__") + matches!(subscript.value.as_ref(), ast::Expr::Name(name) if is_let_marker_id(name.id.as_str())) } _ => false, } @@ -1453,6 +1511,179 @@ pub(super) enum MethodKind<'db> { NotSynthesized, } +/// basedpython: the declaration `class` makes of `name` in its own body, if it +/// makes one — its qualifiers and the type it declares. +fn own_member<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + class: ClassType<'db>, + name: &str, +) -> Option<(TypeQualifiers, Type<'db>)> { + let mut member = class.own_instance_member(db, env, name); + if member.is_undefined() { + member = class.own_class_member(db, env, None, name); + } + Some((member.qualifiers(), member.ignore_possibly_undefined()?)) +} + +/// basedpython: the visibility keyword `class`'s own body declares `name` with, +/// `Public` when it writes none or does not declare the member at all +fn own_member_visibility(db: &dyn Db, class: ClassType<'_>, name: &str) -> MemberVisibility { + class + .class_literal(db) + .as_static() + .and_then(|literal| literal.member_visibilities(db).get(name).copied()) + .unwrap_or_default() +} + +/// basedpython: whether `class`'s member `name` is private to it — declared +/// `private`, or spelled with the leading double underscore python mangles +fn is_private_to(db: &dyn Db, class: ClassType<'_>, name: &str) -> bool { + own_member_visibility(db, class, name) == MemberVisibility::Private || is_mangled_private(name) +} + +/// basedpython: the name a member written `name` and declared with `visibility` +/// is emitted under +fn emitted_name(name: &str, visibility: MemberVisibility) -> String { + ruff_python_stdlib::basedpython::visibility_rename(name, visibility.name_prefix()) + .unwrap_or_else(|| name.to_owned()) +} + +/// basedpython: the nearest superclass of `class` that declares `name`, and the +/// visibility keyword it declares it with +fn inherited_member_visibility<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + class: ClassType<'db>, + name: &str, +) -> Option<(ClassType<'db>, MemberVisibility)> { + class + .iter_mro(db) + .skip(1) + .filter_map(ClassBase::into_class) + .find(|superclass| own_member(db, env, *superclass, name).is_some()) + .map(|superclass| (superclass, own_member_visibility(db, superclass, name))) +} + +/// basedpython: reports a visibility keyword written on a member whose name the +/// class relies on at runtime — a field, an enum member — on an abstract method, +/// which a subclass could then never override, and on a name the keyword cannot +/// act on +fn check_member_visibility<'db>( + context: &InferContext<'db, '_>, + class: ClassType<'db>, + class_kind: Option>, + enum_info: Option<&EnumMetadata<'db>>, + member: &Member<'db>, + definition: Definition<'db>, + qualifiers: TypeQualifiers, +) { + let db = context.db(); + let name = member.name.as_str(); + let visibility = own_member_visibility(db, class, name); + if visibility == MemberVisibility::Public { + return; + } + let renamed = + ruff_python_stdlib::basedpython::visibility_rename(name, visibility.name_prefix()); + let focus = definition.focus_range(db, context.module()); + let class_name = class.name(db); + + // a keyword that cannot act on the name. a dunder is reported where the + // `def` is inferred + let ineffective = if visibility == MemberVisibility::Protected + && name.starts_with("__") + && !name.ends_with("__") + { + Some(format!( + "`protected` has no effect on `{name}`: python name-mangles it, which makes it private" + )) + } else if visibility == MemberVisibility::Private + && renamed.is_some() + && class_name.trim_start_matches('_').is_empty() + { + Some(format!( + "`private` cannot hide `{name}` in `{class_name}`: python does not mangle names in a \ + class named only with underscores" + )) + } else { + None + }; + if let Some(message) = ineffective { + if let Some(builder) = + context.report_lint(&crate::types::diagnostic::INEFFECTIVE_PRIVATE, focus) + { + builder.into_diagnostic(message); + } + return; + } + if renamed.is_none() { + return; + } + + // a field is an annotated class-body attribute the lowering keeps annotated: + // an untyped `private x = v` is emitted without one, which no field-collecting + // construct reads + let is_field = !qualifiers.contains(TypeQualifiers::CLASS_VAR) + && match definition.kind(db) { + DefinitionKind::AnnotatedAssignment(assignment) => { + ruff_python_ast::helpers::DeclarationMarker::of( + assignment.annotation(context.module()), + ) + .is_none_or(|(marker, _)| { + !matches!( + marker.kind, + ruff_python_ast::helpers::DeclarationMarkerKind::Assign + | ruff_python_ast::helpers::DeclarationMarkerKind::ClassVar + | ruff_python_ast::helpers::DeclarationMarkerKind::ClassVarAnnot + ) + }) + } + _ => false, + }; + let reason = if enum_info.is_some_and(|info| info.members.contains_key(&member.name)) { + Some("an enum member's name is how the enum looks it up") + } else if is_field { + match class_kind { + Some(CodeGeneratorKind::NamedTuple) => { + Some("a named tuple's field cannot start with an underscore") + } + Some(CodeGeneratorKind::TypedDict) => { + Some("a typed dict's key is the name it is written with") + } + Some(_) => Some("a field's name is its constructor's keyword"), + None => None, + } + } else if visibility == MemberVisibility::Private + && match member.ty { + Type::FunctionLiteral(function) => { + function.has_known_decorator(db, FunctionDecorators::ABSTRACT_METHOD) + } + _ => false, + } + { + Some("a `private` member is renamed per class, so no subclass could ever override it") + } else { + None + }; + let Some(reason) = reason else { + return; + }; + let Some(builder) = context.report_lint(&crate::types::diagnostic::INVALID_VISIBILITY, focus) + else { + return; + }; + let mut diagnostic = builder.into_diagnostic(format_args!( + "`{name}` cannot be `{keyword}` here", + keyword = visibility.keyword(), + )); + diagnostic.info(format_args!( + "{reason}, and `{keyword}` renames it to `{renamed}`", + keyword = visibility.keyword(), + renamed = renamed.unwrap_or_default(), + )); +} + pub(crate) fn is_constructor_like_method(name: &str) -> bool { matches!( name, diff --git a/crates/ty_python_semantic/src/types/reified_infer.rs b/crates/ty_python_semantic/src/types/reified_infer.rs index 577693b7b5..d6d1d79bda 100644 --- a/crates/ty_python_semantic/src/types/reified_infer.rs +++ b/crates/ty_python_semantic/src/types/reified_infer.rs @@ -1750,8 +1750,8 @@ pub(crate) fn protocol_structural_members<'db>( /// A literal has no bare runtime spelling, so it is rendered as a call to the /// structural check's own `_by_lit` helper, which rebuilds `typing.Literal[…]`. /// That keeps the check exact — an invariant member typed `Literal[True]` must -/// not match a `bool` annotation — and, because the helper ships with the -/// protocol runtime, needs no import at the use site. +/// not match a `bool` annotation — and, because the helper travels with the +/// protocol runtime, the use site never has to ask for it separately. /// /// This deliberately does *not* widen [`runtime_spelling`] itself: that spelling /// is also injected into reified calls (`f[int](…)`) and constructor diff --git a/crates/ty_python_semantic/src/types/signatures.rs b/crates/ty_python_semantic/src/types/signatures.rs index 5af9e22702..8c9f0841f2 100644 --- a/crates/ty_python_semantic/src/types/signatures.rs +++ b/crates/ty_python_semantic/src/types/signatures.rs @@ -1339,6 +1339,20 @@ impl<'db> Signature<'db> { .any(|param| param.inferred_annotation && param.annotated_type.is_unknown()) } + /// basedpython: give the parameter `name` the type `ty` as though it had been written, when + /// the source left it unannotated. every source consulted after this one sees it as declared + pub(crate) fn declare_unannotated_parameter(&mut self, name: &Name, ty: Type<'db>) { + for parameter in &mut Arc::make_mut(&mut self.parameters.data).value { + if parameter.inferred_annotation + && parameter.annotated_type.is_unknown() + && parameter.name() == Some(name) + { + parameter.annotated_type = ty; + parameter.inferred_annotation = false; + } + } + } + pub(crate) fn inherit_unannotated_from_overloads( &mut self, db: &'db dyn Db, diff --git a/crates/ty_python_semantic/src/types/typevar.rs b/crates/ty_python_semantic/src/types/typevar.rs index dec033b372..5412de4769 100644 --- a/crates/ty_python_semantic/src/types/typevar.rs +++ b/crates/ty_python_semantic/src/types/typevar.rs @@ -1727,6 +1727,19 @@ impl<'db> BoundTypeVarInstance<'db> { self.identity(db) == other.identity(db) } + /// basedpython: returns whether two bound typevars are occurrences of the same parameter — + /// the same declaration in the same binding context — whatever freshness each carries. + /// + /// A call into a generic function binds a fresh occurrence of its type parameters, so what + /// the call solves is keyed on that occurrence rather than on the source-level one the + /// function's own types are written in. + pub(crate) fn is_occurrence_of_same_parameter(self, db: &'db dyn Db, other: Self) -> bool { + let (this, other) = (self.identity(db), other.identity(db)); + this.identity == other.identity + && this.binding_context == other.binding_context + && this.paramspec_attr == other.paramspec_attr + } + /// Create a new PEP 695 type variable that can be used in signatures /// of synthetic generic functions. pub(crate) fn synthetic( diff --git a/crates/ty_python_semantic/src/types/visibility.rs b/crates/ty_python_semantic/src/types/visibility.rs index dbe57663ec..dfe5dc858d 100644 --- a/crates/ty_python_semantic/src/types/visibility.rs +++ b/crates/ty_python_semantic/src/types/visibility.rs @@ -7,15 +7,17 @@ //! not import it. [`private_symbols`] collects the marked names so //! `infer_import_from_definition` can report [`PRIVATE_IMPORT`]. //! -//! Only module-level declarations are collected. A `private` member of a class -//! is name-mangled rather than renamed, and is unreachable through an import -//! anyway. +//! Only module-level declarations are collected — functions, classes, type +//! aliases and variables. A `private` member of a class is name-mangled rather +//! than renamed, and is unreachable through an import anyway. The same set also +//! answers a private symbol reached as an attribute of its module (`m.x`), and +//! the lowering renames every reference to one from it. //! //! A dunder is the exception, and the rest of this module is about it. Python //! mangles only a name with at most one trailing underscore, so a `private` //! dunder keeps the name it was written with. For every dunder but one that //! makes `private` a no-op, which the parser reports. The one it does not is -//! `__init__`: [`private_constructor`] answers which class declared a private +//! `__init__`: [`restricted_constructor`] answers which class declared a private //! one, so that construction can be refused wherever the declaring class's own //! body does not reach — see [`PRIVATE_CONSTRUCTOR`]. //! @@ -25,11 +27,18 @@ use ruff_db::files::File; use ruff_db::parsed::parsed_module; use ruff_db::source::source_text; +use ruff_python_ast::helpers::MemberVisibility; use ruff_python_ast::name::Name; use ruff_python_ast::{self as ast, Stmt}; use ruff_text_size::Ranged; use rustc_hash::FxHashSet; +use ruff_python_ast::statement_visitor::{StatementVisitor, walk_stmt}; +use ty_python_core::ProgramFile; +use ty_python_core::ast_ids::HasScopedUseId; +use ty_python_core::definition::DefinitionState; +use ty_python_core::scope::{ScopeId, ScopeKind}; + use crate::Db; /// The module-level names `file` declares `private`. @@ -40,73 +49,530 @@ pub fn private_symbols(db: &dyn Db, file: File) -> FxHashSet { let parsed = parsed_module(db, db.program_file(file).python_file(db)).load(db); let source = source_text(db, file); - let mut names = FxHashSet::default(); - for stmt in parsed.suite() { + let mut collector = PrivateSymbolCollector { + source: &source, + names: FxHashSet::default(), + }; + collector.visit_body(parsed.suite()); + let mut names = collector.names; + names.shrink_to_fit(); + names +} + +/// basedpython: collects [`private_symbols`] — the module-level declarations, +/// including those written inside a module-level `if` or `try`, but none from a +/// function's or a class's body +struct PrivateSymbolCollector<'src> { + source: &'src str, + names: FxHashSet, +} + +impl<'a> StatementVisitor<'a> for PrivateSymbolCollector<'_> { + fn visit_stmt(&mut self, stmt: &'a Stmt) { match stmt { Stmt::TypeAlias(alias) if alias.is_private => { if let ast::Expr::Name(name) = alias.name.as_ref() { - names.insert(name.id.clone()); + self.names.insert(name.id.clone()); } } Stmt::FunctionDef(function) => { - if has_private_marker(&source, &function.decorator_list) { - names.insert(Name::new(function.name.as_str())); + if has_private_marker(self.source, &function.decorator_list) { + self.names.insert(Name::new(function.name.as_str())); } } Stmt::ClassDef(class) => { - if has_private_marker(&source, &class.decorator_list) { - names.insert(Name::new(class.name.as_str())); + if has_private_marker(self.source, &class.decorator_list) { + self.names.insert(Name::new(class.name.as_str())); } } - _ => {} + // `private count: int = 0`, `private count = 0`, `private let count = 0` + // — a variable's visibility rides in its declaration marker + Stmt::AnnAssign(assign) + if ruff_python_ast::helpers::declaration_marker_visibility(&assign.annotation) + == MemberVisibility::Private => + { + if let ast::Expr::Name(name) = assign.target.as_ref() { + self.names.insert(name.id.clone()); + } + } + _ => walk_stmt(self, stmt), } } - names.shrink_to_fit(); - names } -/// basedpython: the name a `private` method is reached by in the emitted -/// python, for an attribute whose inferred type is `member_type`. +/// basedpython: reports each `private` symbol the module lists in `__all__` — +/// in every idiom `__all__` is built with: assigned, annotated, `+=`, +/// `.append(...)`, `.extend([...])` +pub(crate) fn check_private_exports(context: &super::context::InferContext<'_, '_>, body: &[Stmt]) { + let db = context.db(); + let private = private_symbols(db, context.file()); + if private.is_empty() { + return; + } + let mut strings = DunderAllStrings::default(); + strings.visit_body(body); + for string in strings.found { + let value = string.value.to_str(); + if !private.contains(&Name::new(value)) { + continue; + } + let Some(builder) = context.report_lint(&super::diagnostic::PRIVATE_EXPORT, string) else { + continue; + }; + let mut diagnostic = builder.into_diagnostic(format_args!( + "`{value}` is private, so it cannot be in `__all__`" + )); + diagnostic.info( + "the lowering renames it with a leading underscore, so `from ... import *` would not find it", + ); + } +} + +/// basedpython: the string literals a module writes into `__all__` at module level +#[derive(Default)] +struct DunderAllStrings<'a> { + found: Vec<&'a ast::ExprStringLiteral>, +} + +impl<'a> DunderAllStrings<'a> { + fn collect(&mut self, value: &'a ast::Expr) { + let elements = match value { + ast::Expr::List(list) => &list.elts, + ast::Expr::Tuple(tuple) => &tuple.elts, + ast::Expr::StringLiteral(string) => { + self.found.push(string); + return; + } + _ => return, + }; + for element in elements { + if let ast::Expr::StringLiteral(string) = element { + self.found.push(string); + } + } + } +} + +fn is_dunder_all(expr: &ast::Expr) -> bool { + matches!(expr, ast::Expr::Name(name) if name.id.as_str() == "__all__") +} + +impl<'a> StatementVisitor<'a> for DunderAllStrings<'a> { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + match stmt { + Stmt::Assign(assign) if assign.targets.iter().any(is_dunder_all) => { + self.collect(&assign.value); + } + Stmt::AnnAssign(assign) if is_dunder_all(&assign.target) => { + if let Some(value) = &assign.value { + self.collect(value); + } + } + Stmt::AugAssign(assign) if is_dunder_all(&assign.target) => self.collect(&assign.value), + Stmt::Expr(expr) => { + if let ast::Expr::Call(call) = expr.value.as_ref() + && let ast::Expr::Attribute(attribute) = call.func.as_ref() + && is_dunder_all(&attribute.value) + && matches!(attribute.attr.as_str(), "append" | "extend") + { + for argument in &call.arguments.args { + self.collect(argument); + } + } + } + Stmt::FunctionDef(_) | Stmt::ClassDef(_) => {} + _ => walk_stmt(self, stmt), + } + } +} + +/// basedpython: a class body read of a module-level `private` name that the body +/// binds on some paths to the read and not on others. python reads the class's +/// own binding where it exists and the module's otherwise, and the lowering +/// renames the module's, so no one emitted name reads both +pub(crate) fn check_private_class_read<'db>( + context: &super::context::InferContext<'db, '_>, + index: &ty_python_core::SemanticIndex<'db>, + scope: ScopeId<'db>, + name: &ast::ExprName, +) { + let db = context.db(); + if index.scope(scope.file_scope_id(db)).kind() != ScopeKind::Class + || !private_symbols(db, context.file()).contains(&name.id) + || class_scope_binding(db, scope, name) != ClassBinding::Maybe + { + return; + } + let Some(builder) = context.report_lint(&super::diagnostic::INVALID_VISIBILITY, name) else { + return; + }; + let mut diagnostic = builder.into_diagnostic(format_args!( + "`{name}` may read this class body's own `{name}` or the module's private one", + name = name.id, + )); + diagnostic.info( + "the lowering renames the module's with a leading underscore, so no one name reads both: \ + bind it on every path through the class body, or on none", + ); +} + +/// basedpython: whether a class body has bound a name by the point a read of it +/// is written. python reads a class's own binding only once the body has made +/// it; before that the read resolves outward, exactly as it would from a +/// function +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ClassBinding { + Bound, + Unbound, + /// bound on some paths to the read and not on others + Maybe, +} + +/// basedpython: see [`ClassBinding`]. `reference` must be a read in `scope` +fn class_scope_binding<'db>( + db: &'db dyn Db, + scope: ScopeId<'db>, + reference: &ast::ExprName, +) -> ClassBinding { + let use_id = reference.scoped_use_id(db, scope.program_file(db)); + let (mut defined, mut undefined) = (false, false); + for binding in ty_python_core::use_def_map(db, scope).bindings_at_use(use_id) { + match binding.binding { + DefinitionState::Defined(_) => defined = true, + DefinitionState::Undefined | DefinitionState::Deleted => undefined = true, + } + } + match (defined, undefined) { + (true, false) => ClassBinding::Bound, + (false, _) => ClassBinding::Unbound, + (true, true) => ClassBinding::Maybe, + } +} + +/// basedpython: how a bare name read or written in a class body is emitted, when +/// it names a member the class declares with a visibility keyword — `y = x + 1` +/// after `private x = 1`, `alias = helper`, `@size.setter` over a protected +/// property. `None` for any other name /// -/// `None` when the attribute is not a private method. -pub(crate) fn private_method_name<'db>( +/// the class body spells the member as it is declared (`__x`, `_x`): python +/// mangles a class body's names lexically, so `__x` there reaches the same +/// attribute the declaration made +pub(crate) fn class_body_member_name<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + reference: &ast::ExprName, +) -> Option { + let index = crate::semantic_index(db, file); + let file_scope = index.try_expression_scope_id(&ast::ExprRef::from(reference))?; + let class_node = index.scope(file_scope).node().as_class()?; + if reference.ctx.is_load() + && class_scope_binding(db, file_scope.to_scope_id(db, file), reference) + == ClassBinding::Unbound + { + return None; + } + let definition = index.expect_single_definition(class_node); + let class = super::infer::original_class_type(db, definition)?.as_static()?; + let visibility = *class.member_visibilities(db).get(reference.id.as_str())?; + ruff_python_stdlib::basedpython::visibility_rename( + reference.id.as_str(), + visibility.name_prefix(), + ) +} + +/// basedpython: whether the name `reference` reads or writes resolves to the +/// module scope — no scope between it and the module binds it, or one declares +/// it `global`. `None` when ty did not index the reference +/// +/// a class body is flow-sensitive here as python is: a read the body has not yet +/// bound resolves outward. a read that is bound on some paths and not on others +/// answers `false`, and the checker reports it where the name is private +pub(crate) fn resolves_to_module_scope<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + reference: &ast::ExprName, +) -> Option { + let index = crate::semantic_index(db, file); + let reference_scope = index.try_expression_scope_id(&ast::ExprRef::from(reference))?; + for (ancestor_id, scope) in index.ancestor_scopes(reference_scope) { + let is_own_scope = ancestor_id == reference_scope; + // a class body is skipped by name resolution from a scope nested in it + if scope.kind() == ScopeKind::Class && !is_own_scope { + continue; + } + let scope_id = ancestor_id.to_scope_id(db, file); + let table = ty_python_core::place_table(db, scope_id); + let Some(symbol) = table.symbol_by_name(&reference.id) else { + continue; + }; + // `global count` hands the name to the module, whatever this scope does + if symbol.is_global() { + return Some(true); + } + // a binding, or a bare annotation, makes the name local to its scope + if !(symbol.is_bound() || symbol.is_declared()) { + continue; + } + if scope.kind() == ScopeKind::Class && reference.ctx.is_load() { + match class_scope_binding(db, scope_id, reference) { + ClassBinding::Unbound => continue, + ClassBinding::Bound | ClassBinding::Maybe => return Some(false), + } + } + return Some(scope.kind() == ScopeKind::Module); + } + Some(false) +} + +/// basedpython: how a member reached through a receiver is restricted on one +/// class the receiver can be +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum MemberAccess<'db> { + /// declared with no visibility keyword, or by a class whose body cannot be read + Unrestricted, + /// declared with `visibility` by `owner` + Restricted { + visibility: MemberVisibility, + owner: super::class::StaticClassLiteral<'db>, + }, +} + +impl MemberAccess<'_> { + /// the name an access is emitted under, `None` when it keeps the written one + fn emitted_name(self, db: &dyn Db, member: &str) -> Option { + match self { + MemberAccess::Unrestricted => None, + MemberAccess::Restricted { visibility, owner } => { + emitted_member_name(db, visibility, owner, member) + } + } + } +} + +/// basedpython: the classes a receiver can be emit a member under different +/// names — one restricts it and another does not, or two restrict it to names +/// of their own — so no single access reaches all of them +#[derive(Debug)] +pub(crate) struct AmbiguousAccess; + +/// basedpython: how `member` is restricted on each class `receiver` can be that +/// defines it — the one answer both the access check and the lowering's rename +/// read, so the two cannot disagree about a shape. every access returned is +/// emitted under the same name, and each has to be reachable from the access. +/// a class that has no such member at all is left to the checker's own +/// unresolved-attribute diagnostic, so `A | None` is not ambiguous +pub(crate) fn member_access<'db>( + db: &'db dyn Db, + env: &crate::types::ProgramEnvironment<'db>, + receiver: crate::types::Type<'db>, + member: &str, +) -> Result; 2]>, AmbiguousAccess> { + let mut sources = smallvec::SmallVec::<[MroSource<'db>; 2]>::new(); + receiver_sources(db, env, receiver, &mut sources); + let mut accesses = smallvec::SmallVec::<[MemberAccess<'db>; 2]>::new(); + // the common case, answered without a member lookup: nothing the receiver can + // be declares any member with a visibility keyword + if !sources.iter().any(|source| source.declares_any(db, env)) { + return Ok(accesses); + } + for source in sources { + let Some(access) = source.access(db, env, member) else { + continue; + }; + if let Some(first) = accesses.first() + && first.emitted_name(db, member) != access.emitted_name(db, member) + { + return Err(AmbiguousAccess); + } + if !accesses.contains(&access) { + accesses.push(access); + } + } + Ok(accesses) +} + +/// basedpython: the name a class member declared with a visibility keyword is +/// reached by in the emitted python. `None` when the access is not renamed +pub(crate) fn restricted_member_name<'db>( db: &'db dyn Db, env: &crate::types::ProgramEnvironment<'db>, receiver: crate::types::Type<'db>, - member_type: crate::types::Type<'db>, member: &str, ) -> Option { - // a method reaches the access site as the bound method it produced, so its - // own type answers. a property hands back whatever its getter returns - // instead — an `int` says nothing about the declaration — so the class is - // asked for the member it actually holds - let function = declared_function(db, member_type) - .or_else(|| declared_function(db, class_member(db, env, receiver, member)?))?; - if !function.has_known_decorator(db, super::function::FunctionDecorators::PRIVATE) { - return None; + // every access emits the same name, so the first one speaks for all of them + member_access(db, env, receiver, member) + .ok()? + .first()? + .emitted_name(db, member) +} + +/// basedpython: how an access to `owner`'s `member`, declared with `visibility`, +/// is spelled in the emitted python. `None` for a name the keyword cannot rename +pub(crate) fn emitted_member_name( + db: &dyn Db, + visibility: MemberVisibility, + owner: super::class::StaticClassLiteral<'_>, + member: &str, +) -> Option { + let renamed = + ruff_python_stdlib::basedpython::visibility_rename(member, visibility.name_prefix())?; + Some(match visibility { + // `__name` is what python mangles, and it mangles lexically: written in a + // subclass's body it would name `_Subclass__name`, and outside a class body + // nothing at all. the mangled name is spelled out so the same attribute is + // reached from all of them + MemberVisibility::Private => mangled_private_name(owner.name(db).as_str(), member), + _ => renamed, + }) +} + +/// basedpython: how `class`'s own member `name`, declared with a visibility +/// keyword, is spelled — in the class body, where python mangles `__name` +/// lexically, and anywhere else. `None` for a member no keyword renames +pub(crate) fn class_member_spellings<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + class: &ast::StmtClassDef, + name: &str, +) -> Option<(String, String)> { + let index = crate::semantic_index(db, file); + let definition = index.expect_single_definition(class); + let literal = super::infer::original_class_type(db, definition)?.as_static()?; + let visibility = *literal.member_visibilities(db).get(name)?; + let in_body = + ruff_python_stdlib::basedpython::visibility_rename(name, visibility.name_prefix())?; + let anywhere = emitted_member_name(db, visibility, literal, name)?; + Some((in_body, anywhere)) +} + +/// basedpython: where a member reached through a receiver is looked up +#[derive(Clone, Copy)] +enum MroSource<'db> { + /// a class's MRO, from the class itself + Class(crate::types::ClassType<'db>), + /// what `super()` reads: the owner's MRO after the pivot + AfterPivot(super::bound_super::BoundSuperType<'db>), +} + +impl<'db> MroSource<'db> { + fn declares_any(self, db: &'db dyn Db, env: &crate::types::ProgramEnvironment<'db>) -> bool { + match self { + MroSource::Class(class) => declares_any(db, class.iter_mro(db)), + MroSource::AfterPivot(bound) => bound + .lookup_mro_after_pivot(db, env) + .is_some_and(|mro| declares_any(db, mro)), + } } - let scope = function.definition(db).scope(db); - let index = crate::semantic_index(db, scope.program_file(db)); - let class = super::infer::nearest_enclosing_class(db, index, scope)?; - Some(mangled_private_name(class.name(db).as_str(), member)) + + fn access( + self, + db: &'db dyn Db, + env: &crate::types::ProgramEnvironment<'db>, + member: &str, + ) -> Option> { + match self { + MroSource::Class(class) => access_in(db, env, class.iter_mro(db), member), + MroSource::AfterPivot(bound) => bound + .lookup_mro_after_pivot(db, env) + .and_then(|mro| access_in(db, env, mro, member)), + } + } +} + +/// basedpython: every class a receiver can be, as the MRO an attribute on it is +/// looked up in. a class object is asked about its instances: `cls.made` in a +/// classmethod and `Registry.made` reach the class variable `self.made` does +fn receiver_sources<'db>( + db: &'db dyn Db, + env: &crate::types::ProgramEnvironment<'db>, + receiver: crate::types::Type<'db>, + sources: &mut smallvec::SmallVec<[MroSource<'db>; 2]>, +) { + use crate::types::Type; + match receiver.erase_restriction(db) { + Type::Union(union) => { + for element in union.elements(db) { + receiver_sources(db, env, *element, sources); + } + } + Type::Intersection(intersection) => { + for element in intersection.positive(db) { + receiver_sources(db, env, *element, sources); + } + } + Type::BoundSuper(bound) => sources.push(MroSource::AfterPivot(bound)), + receiver @ (Type::ClassLiteral(_) | Type::GenericAlias(_)) => { + if let Some(class) = receiver.to_class_type(db) { + sources.push(MroSource::Class(class)); + } + } + Type::SubclassOf(subclass_of) => { + receiver_sources(db, env, subclass_of.to_instance(db, env), sources); + } + receiver => { + if let Some(class) = receiver.nominal_class(db, env) { + sources.push(MroSource::Class(class)); + } + } + } +} + +/// basedpython: whether any class in `mro` declares a member with a visibility +/// keyword. a class whose body cannot be read is assumed to +fn declares_any<'db>(db: &'db dyn Db, mro: impl Iterator>) -> bool { + mro.filter_map(super::ClassBase::into_class).any(|base| { + base.class_literal(db) + .as_static() + .is_none_or(|literal| !literal.member_visibilities(db).is_empty()) + }) } -/// basedpython: the class that declares `class`'s `private` constructor — the -/// only class whose body may construct it. `None` when the constructor is not -/// private. +/// basedpython: the access to `member` the first class in `mro` that defines it +/// declares. `None` when no class in it defines the member +fn access_in<'db>( + db: &'db dyn Db, + env: &crate::types::ProgramEnvironment<'db>, + mro: impl Iterator>, + member: &str, +) -> Option> { + for base in mro.filter_map(super::ClassBase::into_class) { + if base.own_instance_member(db, env, member).is_undefined() + && base.own_class_member(db, env, None, member).is_undefined() + { + continue; + } + let Some(literal) = base.class_literal(db).as_static() else { + return Some(MemberAccess::Unrestricted); + }; + return Some(match literal.member_visibilities(db).get(member) { + Some(&visibility) if visibility != MemberVisibility::Public => { + MemberAccess::Restricted { + visibility, + owner: literal, + } + } + _ => MemberAccess::Unrestricted, + }); + } + None +} + +/// basedpython: the class that declares `class`'s `private` or `protected` +/// constructor — the class whose body may construct it. `None` when the +/// constructor carries no visibility keyword. /// /// The declaring class answers rather than `class` itself, so a subclass that /// inherits a private `__init__` is reported at its own construction sites: the /// constructor is the base's implementation detail, and a subclass is outside /// the base's body like any other caller. -pub(crate) fn private_constructor<'db>( +pub(crate) fn restricted_constructor<'db>( db: &'db dyn Db, class: crate::types::ClassType<'db>, -) -> Option> { +) -> Option> { // the specialization says nothing about which `__init__` is found or how it // was declared, so the question is asked of the class itself and the answer // is shared by every specialization of it - *private_constructor_of(db, class.class_literal(db).as_static()?) + *restricted_constructor_of(db, class.class_literal(db).as_static()?) } /// Tracked for two reasons. It is asked at every construction site in the @@ -116,10 +582,10 @@ pub(crate) fn private_constructor<'db>( /// one module would depend on the index of every module it constructs /// something from. #[salsa::tracked(returns(ref), heap_size = ruff_memory_usage::heap_size)] -fn private_constructor_of<'db>( +fn restricted_constructor_of<'db>( db: &'db dyn Db, class: super::class::StaticClassLiteral<'db>, -) -> Option> { +) -> Option> { let env = &crate::types::ProgramEnvironment::from_file(class.program_file(db)); let init = class .identity_specialization(db) @@ -127,20 +593,29 @@ fn private_constructor_of<'db>( .place .ignore_possibly_undefined()?; let function = declared_function(db, init)?; - if !function.has_known_decorator(db, super::function::FunctionDecorators::PRIVATE) { - return None; - } + let visibility = crate::types::declared_visibility( + db, + crate::types::TypeQualifiers::empty(), + crate::types::Type::FunctionLiteral(function), + )?; let scope = function.definition(db).scope(db); let index = crate::semantic_index(db, scope.program_file(db)); let owner = super::infer::nearest_enclosing_class(db, index, scope)?; - Some(PrivateConstructor { owner, function }) + Some(RestrictedConstructor { + owner, + function, + visibility, + }) } /// A `private` constructor, and the class that declares it. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, get_size2::GetSize, salsa::SalsaValue)] -pub(crate) struct PrivateConstructor<'db> { +pub(crate) struct RestrictedConstructor<'db> { pub(crate) owner: super::class::StaticClassLiteral<'db>, pub(crate) function: super::function::FunctionType<'db>, + /// `Private` when only `owner`'s own body may construct it, `Protected` when + /// a subclass's body may too. + pub(crate) visibility: MemberVisibility, } /// Whether `scope` lies within the body of `class` — the test for "this code is @@ -162,6 +637,30 @@ pub(crate) fn scope_is_within_class<'db>( }) } +/// basedpython: whether `scope` sits inside `class`'s body or inside the body of +/// a subclass of it — the code a `protected` member is declared for. +pub(crate) fn scope_is_within_subclass_of<'db>( + db: &'db dyn Db, + index: &ty_python_core::SemanticIndex<'db>, + scope: ty_python_core::scope::ScopeId<'db>, + class: super::class::StaticClassLiteral<'db>, +) -> bool { + index + .ancestor_scopes(scope.file_scope_id(db)) + .filter_map(|(_, ancestor)| ancestor.node().as_class()) + .any(|ancestor| { + let definition = index.expect_single_definition(ancestor); + let Some(literal) = super::infer::original_class_type(db, definition) else { + return false; + }; + literal + .identity_specialization(db) + .iter_mro(db) + .filter_map(super::ClassBase::into_class) + .any(|base| base.class_literal(db).as_static() == Some(class)) + }) +} + /// The function a member's type stands for — the member itself, or the getter /// of the property wrapping it. fn declared_function<'db>( @@ -178,22 +677,6 @@ fn declared_function<'db>( } } -/// The member a class holds under `name`, read off the class rather than -/// through an instance, so a descriptor is not resolved on the way. -fn class_member<'db>( - db: &'db dyn Db, - env: &crate::types::ProgramEnvironment<'db>, - receiver: crate::types::Type<'db>, - name: &str, -) -> Option> { - receiver - .erase_restriction(db) - .nominal_class(db, env)? - .class_member(db, env, name, super::MemberLookupPolicy::default()) - .place - .ignore_possibly_undefined() -} - /// basedpython: the attribute name a `private` class member is reached by in /// the emitted python — python's own name mangling, written out in full. /// diff --git a/crates/ty_server/src/session/options.rs b/crates/ty_server/src/session/options.rs index fa44835af8..11d545de86 100644 --- a/crates/ty_server/src/session/options.rs +++ b/crates/ty_server/src/session/options.rs @@ -422,6 +422,7 @@ pub struct InlayHintOptions { inherited_parameter_types: Option, inherited_parameter_defaults: Option, inferred_return_types: Option, + property_types: Option, implicit_arguments: Option, enum_values: Option, template_binding_types: Option, @@ -451,6 +452,7 @@ impl InlayHintOptions { inherited_parameter_types: self.inherited_parameter_types.unwrap_or(true), inherited_parameter_defaults: self.inherited_parameter_defaults.unwrap_or(true), inferred_return_types: self.inferred_return_types.unwrap_or(true), + property_types: self.property_types.unwrap_or(true), implicit_arguments: self.implicit_arguments.unwrap_or(true), enum_values: self.enum_values.unwrap_or(true), template_binding_types: self.template_binding_types.unwrap_or(true), diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap b/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap index b5b2e8c0c1..0f428473f7 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap @@ -37,6 +37,7 @@ Settings: WorkspaceSettings { inherited_parameter_types: true, inherited_parameter_defaults: true, inferred_return_types: true, + property_types: true, implicit_arguments: true, enum_values: true, template_binding_types: true, diff --git a/docs/basedpython/development/how-transpilation-works.md b/docs/basedpython/development/how-transpilation-works.md index 967210800c..9d13ddbe48 100644 --- a/docs/basedpython/development/how-transpilation-works.md +++ b/docs/basedpython/development/how-transpilation-works.md @@ -22,7 +22,8 @@ source (.by) │ │ (intersection, callable, generics, literal-types, anon-NT, …) │ └─ splice it together: re-render changed statements, apply text edits │ (ruff-style first-wins overlap skip), emit hoisted class defs, prepend - │ required imports, append `__all__` epilogue + │ required imports and the runtime helpers the emitted code calls + │ (see "the runtime" below), append `__all__` epilogue │ ├─ phase 1 lowering preamble │ └─ optionally prepend `from __future__ import annotations` @@ -54,6 +55,11 @@ source (.by) │ (`is_anon_named_tuple`, `is_anon_named_tuple_value`, `is_typeof`). │ a leftover flag means a transform failed to lower its construct; the │ pipeline aborts rather than emit syntactically-valid-but-wrong Python + ├─ reject a call to one of the transpiler's own runtime helpers that the + │ module never got. a transform emits the call and records the need in + │ two different places, and forgetting the second half produces python + │ that parses, checks, and raises `NameError` the first time the lowered + │ line runs └─ parse it again *as the target version* and report any construct that version cannot parse. the first check asks whether the output is python at all; this one asks whether it is python the declared floor @@ -71,6 +77,43 @@ entry points in `crates/by_transforms/src/lib.rs`: - `transpile_typed_with_map(db, file, config, rebuild)` — also returns a line table for traceback rewriting and diagnostic mapping +## the runtime + +the emitted python calls helpers of its own: `_lazy_module` for a lowered +import, `_parametric_is` for a runtime type test, `_by_loop_bind` for a closure +that captures a loop binding by value. they live in +`crates/by_transforms/src/runtime/_by_runtime.py`, and `runtime.rs` slices them +out by name + +a transform names the helpers the code it emits calls, through the typed +constants in `runtime.rs`. what each of those calls in turn is read out of its +body, so a helper brings the rest of what it needs along + +`Config::runtime_module` decides how a module gets them: + +- `by build`, `by run` and `by compile` stage a tree of their own. they write + `_by_runtime.py` into each package they stage — a directory with an + `__init__` — and each module imports what it calls from its package's copy +- a module at a module root gets the definitions pasted in. a copy at the root + would be a top-level module, which a second basedpython wheel built by + another version overwrites on install +- so does a module in a directory that is not a package, a `scripts/` folder + say, which has no import that works both when it is run and when it is + imported +- `by transpile ` and the language server's `by/transpile` answer with one + module's text, and `by transpile ` writes into the source tree itself, + where a file with no `.by` beside it would read as a module the author wrote. + all three paste the definitions in + +both renderings are slices of the same text. the lazy-import pass leaves the +runtime import eager, since a helper reached through its proxy would be a proxy +call on every use + +`_by_runtime.py` is excluded from this repository's own ruff configuration. +reformatting it rewrites transpiled output, and the isort rule would put a +`from __future__ import annotations` at the top of every module that gets a +helper pasted into it + ## the phase-0 database three pre-passes run before phase 0 and can rewrite the source: erased-union @@ -109,6 +152,20 @@ ordered lists live in `run_against_source`. a complete list of transforms is at `crates/by_transforms/src/transforms/mod.rs`; each module's `///` docs describe the rewrite it performs +## stubs + +a `.byi` transpiles to a `.pyi`, which a checker reads and python never runs. +whether a file is a stub is read off the file itself by `transpile_typed`, since +`by build` hands every source it stages the one config + +a pass whose output exists only for running — a runtime check, an entry point, +reification, a quoted forward reference — says so through `runtime_only`, and the +driver leaves it out of a stub. a pass that lowers syntax never does: left out, +its construct would reach the `.pyi` as something python cannot parse. the few +passes that do both keep the declaration and drop the rest themselves: a stub +keeps its imports eager, declares an enum's variants without attaching them, and +keeps a mutable default without its guard + ## splicing after the passes run, the driver assembles the output in one pass: diff --git a/docs/basedpython/features/decorated-parameters.md b/docs/basedpython/features/decorated-parameters.md index eb38e52472..2556e8c35c 100644 --- a/docs/basedpython/features/decorated-parameters.md +++ b/docs/basedpython/features/decorated-parameters.md @@ -4,7 +4,7 @@ a decoration hands the function to the decorator, so the callable the decorator accepts gives the decorated function's unannotated parameters their types ```by -decorator def route(fn: (int) -> None) +decorator def route(fn: (int) -> None): ... @route def home(request): diff --git a/docs/basedpython/features/decorator-keyword.md b/docs/basedpython/features/decorator-keyword.md index c56980d970..d4521fe228 100644 --- a/docs/basedpython/features/decorator-keyword.md +++ b/docs/basedpython/features/decorator-keyword.md @@ -39,13 +39,19 @@ keyword-only and have defaults — they are the decorator's options ## declaring one without a body -like any other `def`, a `decorator def` can be written with no body at all — a -declaration of the shape, with nothing to run +like any other `def`, a `decorator def` can be written with no body at all. what +that declares is the shape, with nothing to run, so it belongs where a +declaration does — a stub file, or an `if TYPE_CHECKING` block: -```by +```byi decorator def route(fn: (int) -> None) ``` +in a module that runs, the body is required +([`missing-function-body`](empty-declarations.md)): the dispatcher the lowering +writes is the machinery around the body, and calls it for every shape the +decorator is applied in + ## scope `decorator def` is **module-scope only**. inside a class body the keyword is diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 9f3aa76002..a53a774add 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -223,6 +223,7 @@ on | `inheritedParameterTypes` | the type a parameter takes from the method it overrides | | `inheritedParameterDefaults` | the [default](inherited-defaults.md) it takes from that method | | `inferredReturnTypes` | the return type of a `def` that leaves it out | +| `propertyTypes` | the type a [property](properties.md) leaves to its accessors | | `implicitArguments` | the [context arguments](context-parameters.md) a call fills | | `enumValues` | the value an [enum](enums.md) member takes implicitly | | `templateBindingTypes` | a django template `{% for %}` binding's element type | diff --git a/docs/basedpython/features/empty-declarations.md b/docs/basedpython/features/empty-declarations.md index 0314abe8f1..0bce151fc6 100644 --- a/docs/basedpython/features/empty-declarations.md +++ b/docs/basedpython/features/empty-declarations.md @@ -1,12 +1,14 @@ # empty declarations class and function declarations may be written without a body. basedpython -fills in `: ...` at transpile time so the output is valid Python: +fills in `: ...` at transpile time so the output is valid python: ```by class Empty class Stub(Base) -def stub(x: int) -> int + +protocol Reader: + def read(self) -> str ``` transpiles to: @@ -14,7 +16,9 @@ transpiles to: ```python class Empty: ... class Stub(Base): ... -def stub(x: int) -> int: ... + +class Reader(Protocol): + def read(self) -> str: ... ``` ## scope @@ -27,11 +31,11 @@ empty defs that *are* part of an overload run instead receive an `abstract def` with no body is given `: raise NotImplementedError` rather than `: ...` -the bodyless form is the same empty body written a shorter way, so it is -allowed in the same places: a stub file, a `Protocol` member, an abstract -method, an overload, or an `if TYPE_CHECKING` block. anywhere else a `def` -that declares a return type but no body is reported (`empty-body`) — the -`: ...` it lowers to returns `None`: +a `def` with no body is a declaration, so it is allowed where a declaration is +what the position asks for: a stub file, a `Protocol` member, an abstract +method, an overload, or an `if TYPE_CHECKING` block. anywhere else the +implementation the signature promises was never written — the `: ...` it lowers +to just returns `None` — and that is reported (`missing-function-body`): ```by def parse(s: str) -> int # ok — the run below makes this an overload @@ -39,9 +43,16 @@ def parse(s: bytes) -> int def parse(s): return int(s) -def lookup() -> int # error: implicitly returns `None` +def lookup() -> int # error: no body ``` +the return type makes no difference: `def lookup()` is reported the same way. +a function that is meant to do nothing says so with a body of its own, +`def ignore(event: str): ...` + +an empty `class` is reported nowhere: a class with no members is a whole class, +with nothing left out + ## interaction with modifiers modifiers stack as expected: diff --git a/docs/basedpython/features/exceptions.md b/docs/basedpython/features/exceptions.md index 87a7c1886b..4e3c0ba1f0 100644 --- a/docs/basedpython/features/exceptions.md +++ b/docs/basedpython/features/exceptions.md @@ -85,6 +85,43 @@ def f() raises not TypeError: the practical way to rule an exception out is to declare what the function does raise, or `raises Never` +### type parameters + +a clause may name a type parameter, and a call reads it as whatever it solved +that parameter to: + +```by +def rethrow[T: BaseException](error: T) raises T: + raise error + +def f() raises KeyError: + rethrow(KeyError()) # raises KeyError, not BaseException +``` + +an explicit specialization is read the same way, and so is a call back into the +same function that solves the parameter to something else: + +```by +def g() raises FileNotFoundError: + rethrow[FileNotFoundError](FileNotFoundError()) +``` + +a method may name its class's type parameter, and the receiver says which +exception that is — `Reader[KeyError].read` as much as `reader.read()`. a +function nested in a generic one can name the enclosing function's parameter, +which keeps meaning what the enclosing call was made with + +the parameter has to be declared an exception, since it stands for one type the +caller chooses: + +```by +def bad[T](error: T) raises T: # error: `T@bad` is not always an exception + ... +``` + +a parameter left unsolved that names nothing where the call is written stands for +everything it was declared to allow — its bound, or its set of constraints + ## what is inferred the analysis reports what it can see in the body: @@ -228,6 +265,48 @@ only a clause with a faithful runtime test is guarded: a gradual `raises ...` and any set with no runtime spelling are left alone, and `raises Never` becomes the empty tuple, which nothing is an instance of +a type parameter has no runtime spelling of its own — which exception it is was +chosen by the caller, and the guard runs inside the callee — so the guard tests +the parameter's **ceiling**, the bound or set of constraints it was declared +with. that never rejects an exception the clause allows, and still catches one +it does not: + +```by +def rethrow[T: OSError](error: T) raises T: + raise error +``` + +lowers to + +```py +@_by_raises(OSError, "rethrow") +def rethrow[T: OSError](error: T): + raise error +``` + +a [reified](reified-generics.md) parameter does carry the type the caller chose, +and where the guard can read it, it tests exactly that: + +```by +def rethrow[reified T: OSError](error: dynamic) raises T: + raise error + +rethrow[FileNotFoundError](PermissionError()) # AssertionError: not a FileNotFoundError +``` + +- a function's own parameter comes from the specialization it is called through +- a class's parameter comes from the instance a method is called on. a function + nested in a method has no receiver, so there it stays on the ceiling +- an enclosing function's parameter is already bound where the nested + function's guard is evaluated +- a subscripted argument, which `isinstance` refuses, is tested by its origin, + the way `list[str]` is tested as `list` + +the guard never asks for an argument a parameter does not already carry: making +one reified changes how the program is built, and turning a check on must not do +that. an unreified parameter stays on its ceiling, where a `PermissionError` +passes + a decorated function whose statement another lowering re-renders cannot carry the guard — the insertion sits inside the range being rebuilt — and that is a transpile error rather than a silently missing check diff --git a/docs/basedpython/features/forward-references.md b/docs/basedpython/features/forward-references.md index c4ff868d31..1787f74ea0 100644 --- a/docs/basedpython/features/forward-references.md +++ b/docs/basedpython/features/forward-references.md @@ -1,55 +1,85 @@ # automatic forward references -a class that refers to itself in a *subscript* base or in a method-signature -annotation triggers a `NameError` at class definition time in standard -Python, because the name is not yet bound. the conventional fix is to quote -the reference: `class Tree(list["Tree"])`. basedpython does the quoting -automatically: +an annotation can name a class defined further down, or the class it sits in: ```by +def root() -> Tree: + return Tree() + + class Tree(list[Tree]): children: list[Tree] - def add(self, child: Tree) -> Tree: ... + + def add(self, child: Tree) -> Tree: + self.append(child) + return child ``` -transpiles to: +before 3.14, python evaluates these annotations as the `def` or class body +runs, when `Tree` is not bound yet, and raises `NameError`. the conventional fix +is to quote the reference. basedpython does the quoting for you, so for a +target before 3.14 this transpiles to: ```python +def root() -> "Tree": + return Tree() + + class Tree(list["Tree"]): - children: list["Tree"] - def add(self, child: "Tree") -> "Tree": ... + children: "list[Tree]" + + def add(self, child: "Tree") -> "Tree": + self.append(child) + return child ``` ## scope -quoting fires in three positions inside a class definition: +an annotation python evaluates as its definition runs is quoted when it names +something that is not bound by then: a class defined further down, the class +the annotation sits in, or a name imported only under `if TYPE_CHECKING:`, +which never runs at all. that covers parameter and return annotations, and the +annotations of class-body and module-level variables. the whole annotation is +quoted, so a basedpython type inside it (`Tree?`, `(Tree) -> None`) is quoted +in its lowered form (`"Tree | None"`, `"Callable[[Tree], None]"`) -1. **subscript bases** — `class A(list[A])`, `class T(Node[T | None])`. the - entire subscript value is quoted (`"T | None"`), not just the - self-reference, so unions and nested generics work uniformly -1. **class-body annotations** — attribute and method-parameter / return - types whose textual form contains the class name -1. **inherited generic typevars** that name the class +a name that is already bound is left alone, as is a local variable's +annotation, which python never evaluates -direct base classes — `class A(A)` — are *not* quoted. that pattern is -always a runtime error and quoting it would only mask the bug. method -*bodies* are also not rewritten: by the time the body executes, the class -name is bound, and quoting would produce a string instead of a value +a class's *subscript bases* evaluate while the class is being built, and so do +value-position subscripts in its body (`list[Tree]()`). a self-reference there +is quoted where it stands: `class Tree(list["Tree"])`. a direct base — +`class A(A)` — is *not* quoted. that is always a runtime error, and quoting it +would only mask the bug. method *bodies* are not rewritten either: by the time +a body runs, the class is bound, and quoting would produce a string instead of +a value ## why automatic -forward-reference quoting is mechanical: the only information that controls -it is whether the name is the enclosing class. doing it at the transpiler -level lets ty type-check the *unquoted* form (which it already understands) -and emits the quoted form for the runtime +whether a name is bound by the time an annotation runs is a question about +the program's bindings, which the checker already answers. basedpython reads +every annotation as deferred, so you write the unquoted form everywhere and the +transpiler quotes exactly the references that need it + +## converting python + +in python a string annotation is a forward reference. in basedpython a string +in an annotation is a [literal type](literal-types.md), so +`by transpile --reverse` writes each one as the expression it spells: +`-> "Tree"` becomes `-> Tree`, and `"Tree | None"` becomes `Tree?`. the +arguments of `Literal[…]` and the metadata of `Annotated[…]` are values, and +stay strings ## when quoting is skipped -quoting is only emitted when the annotation would otherwise be evaluated -eagerly. it is skipped when: +annotation quoting is only emitted when the annotation would otherwise be +evaluated eagerly. it is skipped when: - the target is python 3.14 or newer — annotations are deferred natively (PEP 649), so the bare name resolves lazily - the file already defers every annotation through `from __future__ import annotations`, whether you wrote it yourself or opted into the blanket injection + +a subscript base evaluates eagerly on every target, so its self-reference is +quoted either way diff --git a/docs/basedpython/features/generics.md b/docs/basedpython/features/generics.md index dc36d3dfc7..d6613cb2e1 100644 --- a/docs/basedpython/features/generics.md +++ b/docs/basedpython/features/generics.md @@ -54,7 +54,7 @@ bound on a type parameter: ```by # `*` here means the projected top type, which differs from `*: object, **: object` -def f[P: (*: *, **: *)](fn: (**P) -> None) -> (**P) -> int +def f[P: (*: *, **: *)](fn: (**P) -> None) -> (**P) -> int: ... ``` call-site arguments for any type parameter can use an enhanced tuple type, @@ -73,7 +73,7 @@ A[(bool, a: str, b: str)] `Concatenate` is replaced with an unpack: ```by -def f[P: (*: *, **: *)](fn: (**P) -> None) -> (int, **P) -> None +def f[P: (*: *, **: *)](fn: (**P) -> None) -> (int, **P) -> None: ... ``` ## forwarding @@ -105,7 +105,7 @@ class Callable[Parameters: (*: *, **: *), Return]: returns: Return class A[Fn: (*: *, **: *) -> object]: - def f(self, *args: *Fn.parameters, **kwargs: **Fn.parameters) -> Fn.returns + def f(self, *args: *Fn.parameters, **kwargs: **Fn.parameters) -> Fn.returns: ... ``` ## a bound can name another type parameter diff --git a/docs/basedpython/features/init-method.md b/docs/basedpython/features/init-method.md index cdcd67cb6c..11b3a3b254 100644 --- a/docs/basedpython/features/init-method.md +++ b/docs/basedpython/features/init-method.md @@ -103,9 +103,10 @@ def _(box: Box[int]): `var` is the mutable counterpart of `let`; on an `init` parameter it self-assigns identically, but the attribute stays writable — and a class that -stores one is invariant in its type. a visibility modifier — `private` or `public` — may -precede `let` / `var`. `private` name-mangles the synthesised attribute to -`self.__name`, while the parameter itself keeps its declared name: +stores one is invariant in its type. a visibility modifier — `private`, +`protected` or `public` — may precede `let` / `var`. it decides who may reach the +attribute, exactly as it does on a class-body declaration, and the parameter +itself keeps the name it was declared with: ```by class A: @@ -120,6 +121,10 @@ class A: self.__a: int = a ``` +the class's own body reaches the attribute by the name the parameter wrote — +`self.a` — and the lowering spells out the mangled one. see +[modifiers](modifiers.md#private-and-protected) + a visibility modifier without `let` / `var` has no attribute to name, and any other modifier keyword (`final`, `abstract`, …) is meaningless in this position — both are reported as errors @@ -180,6 +185,22 @@ python calls a constructor by its exact name, so there is no spelling that would hide it and leave the class constructible. a private constructor is a static guarantee rather than a runtime one +`protected init` draws the boundary one step wider: the declaring class and the +classes that inherit it may construct it, and nothing else. that is what a base +class writes when only its subclasses should build one + +```by +class Shape: + protected init(let sides: int) + +class Square(Shape): + @classmethod + def make(cls) -> Square: + return Square(4) + +Shape(3) # rejected +``` + ## implicit `self` `self` may be omitted from the parameter list. it is implied, so it is diff --git a/docs/basedpython/features/lazy-imports.md b/docs/basedpython/features/lazy-imports.md index 7f5d2166b1..79faf48c83 100644 --- a/docs/basedpython/features/lazy-imports.md +++ b/docs/basedpython/features/lazy-imports.md @@ -39,6 +39,9 @@ what actually loads the module `import a.b` without an alias stays eager (write `import a.b as ab` to opt in). `from __future__ import …` and `from x import *` are always eager +a stub is never executed, so it has nothing to defer: a `.byi` keeps every import +as written, less any `lazy` + ## target version on python 3.15 and later, the PEP 810 `lazy` keyword is used directly. diff --git a/docs/basedpython/features/modifiers.md b/docs/basedpython/features/modifiers.md index bb1d47cff9..301c5fabb1 100644 --- a/docs/basedpython/features/modifiers.md +++ b/docs/basedpython/features/modifiers.md @@ -173,7 +173,7 @@ rejects it with `final-on-variable` and points you to `let`, which lowers to `Final`. inside a class body it is a plain attribute, matching `let` there, and is not flagged -## export / public / private +## export / public basedpython infers `__all__` from explicit visibility keywords: @@ -193,31 +193,174 @@ def also_exported(): ... def _helper(): ... ``` -- `export` and `public` are aliases. each marked symbol is added to a synthesized - `__all__` list at module level -- `private` strips the keyword and gives the symbol a leading underscore at the - definition site *and* every same-module call site. a name that already has - one keeps it — a second would make it a `__name`, which python name-mangles - wherever a class body reads it. it is excluded from `__all__` even when no - `export`/`public` declarations exist -- inside a class body only `private` means anything — `export`/`public` are - stripped. what `private` renames depends on the member: a `private def` is - name-mangled (`__helper`), a `private` [property](properties.md) becomes `_x` - with `__x` storage, and a `private` attribute keeps its name. either way the - member is private to the type checker, which is what - [safe variance](safe-variance.md) rests on -- a call to a `private def` is written with the mangled name spelled out — - `self.helper()` becomes `self._A__helper()`. python mangles lexically, so a - bare `self.__helper` would name a different attribute in a subclass's body - and none at all outside a class; the full spelling reaches the method from - all of them -- `private` on a name python looks up verbatim — a dunder, or `_` — is reported - as having no effect. mangling applies only to a name with at most one - trailing underscore, so renaming would change what the member *is* rather - than who can reach it, and leaving it alone would make the modifier do - nothing. the one dunder where `private` says something is - [`init`](init-method.md#private-constructors), which is checked at the - construction site instead +`export` and `public` are aliases. each marked symbol is added to a synthesized +`__all__` list at module level. inside a class body they are stripped: a class +member is not a module export + +a module-level `private` strips the keyword and gives the symbol a leading +underscore at the definition site *and* every same-module call site. a name that +already has one keeps it — a second would make it a `__name`, which python +name-mangles wherever a class body reads it. it is excluded from `__all__` even +when no `export`/`public` declarations exist, and another module that imports it +is reported by `private-import` + +a module-level `private` works on a variable exactly as it does on a function, +class or type alias — `private count: int = 0` is emitted as `_count`, and so is +every reference to it in the module. a parameter, local or class attribute that +merely shares the name is a different binding and keeps it. reaching a private +symbol as an attribute of its module from another one (`helpers.count`) is +`inaccessible-member`, for the same reason importing it is `private-import` + +the rename follows the symbol wherever the module binds it again — a `def` or +`class` of the same name, an import (`from m import count` becomes `from m import count as _count`), an `except ... as count`, a `match` capture, and a `global count` +in a function. a dotted `import count.sub` binds its top-level package, which no +alias can keep under `_count`, so it is an error. a private symbol is left out of +`from m import *`, and listing one in `__all__` is `private-export` + +## private and protected + +`private` and `protected` say who may reach a class member: + +```by +class Account: + protected rate: float = 0.05 + init(private let balance: int) + + def interest(self) -> float: + return self.balance * self.rate + + +class Savings(Account): + def bonus(self) -> float: + return self.rate * 2 # `protected` — a subclass may +``` + +- **`private`** — only the declaring class's own body +- **`protected`** — that, and the body of any subclass + +reaching one from anywhere else is `inaccessible-member`: + +```by +def audit(account: Account) -> int: + return account.balance # error: `balance` is private to `Account` +``` + +the keywords work on every kind of member — a `def`, an attribute, a nested +class, a [property](properties.md), an [`init`](init-method.md) parameter — and +the member is written by the name it was declared with wherever it may be +reached. what changes is the name it is *emitted* under, because that is the only +enforcement python itself offers: `private` becomes `__name`, which python +name-mangles per class, and `protected` becomes `_name`, the convention python +uses for the same thing. a member no widened view can reach is also what +[safe variance](safe-variance.md) rests on + +the mangled name is spelled out at the access site — `self.helper()` becomes +`self._A__helper()`. python mangles lexically, so a bare `self.__helper` written +in a nested scope would name whatever class encloses it, and the full spelling +names the same attribute from every one of them + +a visibility keyword on a name python looks up verbatim — a dunder, or `_` — is +reported as having no effect. mangling applies only to a name with at most one +trailing underscore, so renaming would change what the member *is* rather than +who can reach it, and leaving it alone would make the modifier do nothing. the +one dunder where the keyword says something is +[`init`](init-method.md#private-constructors), which is checked at the +construction site instead + +`protected` needs a class for the "and its subclasses" half to mean anything, so +it is only a modifier on a class member: + +```by +protected def helper(): ... # error: `protected` is only a modifier on a class member +``` + +a class variable takes one modifier besides its own `class` keyword — a +visibility keyword, which composes with any declaration. the class reaches it +through the class object as readily as through an instance: + +```by +class Counter: + private class var made: int = 0 + protected class let LIMIT: int = 3 + + @classmethod + def total(cls) -> int: + return cls.made + cls.LIMIT +``` + +### visibility and inheritance + +a visibility keyword decides the name the member is emitted under, so a member +emitted under a different name from the one it inherits does not override it. +it sits beside it, and the inherited member is still what a call finds — which is +never what the declaration looks like it does, so it is +`invalid-override-visibility`. that happens in two ways: a keyword declaring the +member narrower than what it inherits, and a plain declaration over an inherited +`protected` member + +```by +class A: + def f(self) -> int: + return 1 + + protected def g(self) -> int: + return 1 + + +class B(A): + private def f(self) -> int: # error: `f` is public on `A` + return 2 + + def g(self) -> int: # error: `g` is protected on `A` + return 2 +``` + +a `private` member is the exception on the other side. two `private` members that +share a name are mangled apart, so neither overrides the other and neither has to +match the other's signature, and a subclass is free to declare a public member +under a name its base kept private: + +```by +class Base: + private def step(self, n: int) -> int: + return n + + +class Derived(Base): + def step(self) -> str: # a new member, not a replacement + return "x" +``` + +a `protected` member keeps one name across the hierarchy, so it overrides like +any other member and is checked like one + +### names a member is looked up by + +a visibility keyword renames every place a member is named, not only attribute +accesses: a bare name in the class body (`y = x + 1`, `@size.setter`), a string in +`__slots__` or `__match_args__`, and a keyword in a class pattern (`case Point(x=0)`). each is spelled the way that position needs — `__slots__` takes the +class-body spelling python mangles, `__match_args__` and a pattern keyword the one +`getattr` reaches + +### where a visibility keyword cannot go + +some names are how the class works at runtime, and renaming them would change what +the class does rather than who can reach it. a visibility keyword on one is +`invalid-visibility`: + +- a field of a dataclass-like class, a named tuple or a typed dict — the field's + name is its constructor's keyword, or its key +- an enum member — its name is how the enum looks it up +- an abstract method — a `private` one is renamed per class, so no subclass could + ever override it + +a keyword that cannot act on its name is `ineffective-private`: a dunder, `protected` +on a name that already starts with `__` (python mangles it, which makes it private), +and `private` in a class named only with underscores (python mangles nothing there) + +a visibility keyword says who may reach a class member, or marks a module-level +declaration as the module's own. a declaration inside a function body is a local, +which nothing outside the function reaches anyway, so a keyword on one is an error ## inlay hints diff --git a/docs/basedpython/features/properties.md b/docs/basedpython/features/properties.md index 37694a78d6..9e9378ad8f 100644 --- a/docs/basedpython/features/properties.md +++ b/docs/basedpython/features/properties.md @@ -131,6 +131,38 @@ set(value): field = value ``` +## inferred type + +a property that carries an accessor block may leave its type out. with an +initialiser, the property's type is the one the initialiser declares, exactly as +if it were written: `var count = 0` is `var count: int = 0`, so `get` must return +an `int` and `set` accepts one + +```by +class Counter: + var count = 0 + get() = field + set(value): + assert value >= 0 + field = value +``` + +without an initialiser the type is whatever `get` returns. only a `let` can leave +both out — a `var` needs a type or an initialiser + +```by +class A: + let a + get() = 1 +``` + +`A().a` reads as `1`. the editor draws the recovered type where the declaration +left it out — `var count⟨: int⟩ = 0`, `let a⟨: 1⟩` — and +`ty.inlayHints.propertyTypes` turns that off; see [editor features](editor.md) + +an initialiser is stored in the backing field, so a computed property — one whose +accessors never mention `field` — takes none + ## `field` keyword inside `get`/`set` body, `field` refers to backing storage. lowers to @@ -277,12 +309,13 @@ declaration: a bodyless `init(...)`, whose body the property declarations compose with [modifier keywords](modifiers.md): -| basedpython | Python output | -| ------------------------- | ---------------------------------------------- | -| `override var x: int = 0` | `x` overrides parent; `@override` on accessors | -| `final var x: int = 0` | property marked `@final` | -| `abstract let x: int` | `@property` + `@abstractmethod`, no body | -| `private var x: int = 0` | property renamed `_x`, storage `__x` | +| basedpython | Python output | +| -------------------------- | ---------------------------------------------- | +| `override var x: int = 0` | `x` overrides parent; `@override` on accessors | +| `final var x: int = 0` | property marked `@final` | +| `abstract let x: int` | `@property` + `@abstractmethod`, no body | +| `private var x: int = 0` | property renamed `__x`, storage `__x_field` | +| `protected var x: int = 0` | property renamed `_x`, storage `__x` | `abstract let` / `abstract var` are bodyless. abstract `var` produces both abstract getter and abstract setter. the modifiers apply to a declaration that @@ -341,10 +374,12 @@ inside an [extension](extensions.md) the same declaration needs no descriptor at all — the access site is rewritten at transpile time, so the backing function just receives the class -## `private` +## `private` and `protected` -`private` shifts the whole construct one level of underscore deeper — the property -becomes `_x` and its storage `__x`: +a visibility keyword renames a property the way it renames any member: `private` +to `__x`, which python name-mangles per class, and `protected` to `_x`. the storage +of a `private` property takes a name of its own, `__x_field`, since `__x` is the +property: ```by class A: @@ -362,31 +397,29 @@ transpiles to: ```python class A: def __init__(self) -> None: - self.__x: int = 0 + self.__x_field: int = 0 @property - def _x(self) -> int: - return self.__x - @_x.setter - def _x(self, value: int) -> None: - self.__x = value + def __x(self) -> int: + return self.__x_field + @__x.setter + def __x(self, value: int) -> None: + self.__x_field = value def bump(self): - self._x = self._x + 1 + self._A__x = self._A__x + 1 ``` -accesses written inside the class under the public name are redirected, so the -declaration site is the only place the name changes. privacy is self-enforcing: -the property does not exist under its public name, so an access from outside the -class — or from a subclass — is an unresolved attribute, reported rather than -failing at runtime +accesses written inside the class under the declared name are renamed with it, so +the declaration is written once. an access from outside the class is reported as +`inaccessible-member`, and so is one from a subclass when the property is +`private` -a write is redirected to the property, not to its storage, so a validating setter -still runs +a write inside the class goes through the property, not its storage, so a +validating setter still runs -note this differs from a plain `private var x: int = 0` with no accessor block, -which is [stripped without renaming](modifiers.md) like any other class member -annotation — it is still private to the type checker, which is what -[safe variance](safe-variance.md) rests on, but nothing hides it at runtime +a plain `private var x: int = 0` with no accessor block is the attribute `__x` +itself — there is no property to keep separate from its storage. see +[modifiers](modifiers.md#private-and-protected) ## `late` diff --git a/docs/basedpython/getting-started.md b/docs/basedpython/getting-started.md index 0b19c34a85..c29dc49a50 100644 --- a/docs/basedpython/getting-started.md +++ b/docs/basedpython/getting-started.md @@ -144,19 +144,23 @@ class Node: def __eq__(self, other: object) -> bool: if other === self: return True - return other is Node and other.children == self.children + return isinstance(other, Node) and other.children == self.children - def find(self, key: str) -> Optional[Node] + def find(self, key: str) -> Optional[Node]: ... on_visit: (Node) -> None ``` -the identity fast path became `===` and the `isinstance` became -[`is`](features/identity-swap.md), the quotes came off the self-references, -`Callable[[Node], None]` became an [arrow type](features/callable.md), the -`: ...` body became an [empty declaration](features/empty-declarations.md), and -the now-unused `Callable` import was pruned +the identity fast path became [`===`](features/identity-swap.md). the +`isinstance` call stays a call: basedpython's `x is Node` is a type test that +the value's static type can settle, and a python check is there to run +whatever the annotations say. the quotes came off the +[forward references](features/forward-references.md), +`Callable[[Node], None]` became an [arrow type](features/callable.md), and the +now-unused `Callable` import was pruned. the `: ...` body stayed: it is a body +the method is entitled to, and only in a stub does dropping it leave a +[declaration](features/empty-declarations.md) that says the same thing point it at a directory to convert a whole tree in place, every `.py` to a `.by`: diff --git a/docs/basedpython/packaging.md b/docs/basedpython/packaging.md index dc80217a4c..4edb350506 100644 --- a/docs/basedpython/packaging.md +++ b/docs/basedpython/packaging.md @@ -56,7 +56,10 @@ module `app.main`, so it lands at `app/main.py` and not at `src/app/main.py` `build/` is a mirror, not a pile: what a previous build wrote and this one did not is deleted, so a module you renamed does not go on being importable -a stub stays a stub. `a.byi` builds to `a.pyi`, never to `a.py` +a stub stays a stub. `a.byi` builds to `a.pyi`, never to `a.py`, and it holds +declarations only: a `.pyi` is read by a type checker and never run, so nothing +that exists for running — a deferred import, the `main` entry point, a runtime +check, a registration — is written into it ### two sources, one module @@ -210,9 +213,11 @@ the declarations that have no python spelling survive the trip: ```by extension FlowContent: - def card(self) -> Div + def card(self) -> Div: + return Div(self) -def load(path: str) -> Config raises ParseError +def load(path: str) -> Config raises ParseError: + return Config.parse(read_text(path)) ``` a consumer reading only the transpiled python sees `load` returning a `Config`. diff --git a/pyproject.toml b/pyproject.toml index 5ad37daa95..a5b0f9ac20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -186,6 +186,12 @@ extend-exclude = [ # the isort rule alone would put `from __future__ import annotations` at the # top of every one of them "scripts/native-bench/programs/", + # the helpers the transpiler emits into the python it produces. reformatting + # them rewrites transpiled output, and the isort rule would put a + # `from __future__ import annotations` at the top of every module that gets + # a helper pasted into it. what shape they take is pinned by the + # transpiler's own tests + "crates/by_transforms/src/runtime/", "crates/ty_vendored/vendor/", "crates/ruff/resources/", "crates/ruff_linter/resources/", diff --git a/python/basedpython-pygments/basedpython_pygments/__init__.py b/python/basedpython-pygments/basedpython_pygments/__init__.py index 969aacf0c2..f9bc6565f5 100644 --- a/python/basedpython-pygments/basedpython_pygments/__init__.py +++ b/python/basedpython-pygments/basedpython_pygments/__init__.py @@ -52,6 +52,7 @@ "open", "override", "private", + "protected", "public", "sealed", "static", diff --git a/scripts/check_by_lexer.py b/scripts/check_by_lexer.py index 2005e5a0c5..56d612121c 100644 --- a/scripts/check_by_lexer.py +++ b/scripts/check_by_lexer.py @@ -77,6 +77,7 @@ ("override def f(): ...", "override"), ("static let x: int", "static"), ("private type X = int", "private"), + ("protected def f(): ...", "protected"), ("public let x = 1", "public"), ("late var x: int", "late"), ("class Mapping[out Key]: ...", "out"), diff --git a/ty.schema.json b/ty.schema.json index aeed621c23..db038b2b8e 100644 --- a/ty.schema.json +++ b/ty.schema.json @@ -1060,6 +1060,16 @@ } ] }, + "inaccessible-member": { + "title": "detects access to a `private` or `protected` member from outside where it may be reached", + "description": "## What it does\nChecks for reads and writes of a `private` or `protected` class member\nfrom outside the code allowed to reach it, and for a module's `private`\nsymbol reached as an attribute of the module from another one.\n\n## Why is this bad?\nA visibility keyword draws a boundary around a member: `private` says only\nthe declaring class's own body may use it, and `protected` extends that to\na subclass's body. Reaching past the boundary defeats the point of drawing\nit, and the member may be renamed or removed without notice.\n\nIt is also very likely to fail at runtime. The lowering spells the\nvisibility in the member's name, and for `private` that name is one python\nmangles: an access written outside the class names a different attribute,\nor none at all.\n\n## Example\n\n```by\nclass Account:\n init(private let balance: int)\n\ndef audit(account: Account) -> int:\n return account.balance # error: `balance` is private to `Account`\n```", + "default": "error", + "oneOf": [ + { + "$ref": "#/definitions/Level" + } + ] + }, "inconsistent-mro": { "title": "detects class definitions with an inconsistent MRO", "description": "## What it does\n\nChecks for classes with an inconsistent [method resolution order] (MRO).\n\n## Why is this bad?\n\nClasses with an inconsistent MRO will raise a `TypeError` at runtime.\n\n## Examples\n\n```python\nclass A: ...\n\n\nclass B(A): ...\n\n\n# TypeError: Cannot create a consistent method resolution order\nclass C(A, B): ... # error\n```\n\n[method resolution order]: https://docs.python.org/3/glossary.html#term-method-resolution-order", @@ -1091,7 +1101,7 @@ ] }, "ineffective-private": { - "title": "detects a `private` modifier on a name it cannot hide", + "title": "detects a visibility keyword on a name it cannot act on", "description": "## What it does\nChecks for the `private` modifier on a class member whose name it cannot\nhide.\n\n## Why is this bad?\n`private` hides a class member by renaming it so python's name-mangling\napplies, and python mangles only a name with at most one trailing\nunderscore. A dunder is therefore left with the name it was written\nwith, and the modifier does nothing — which is worse than an error,\nbecause the declaration reads as though the member were hidden.\n\n`init` is the exception. It is the one dunder `private` says something\nabout, and it is enforced at the construction site rather than by hiding\na name: see [`private-constructor`](private-constructor.md).\n\n## Example\n\n```by\nclass Point:\n private def __repr__(self) -> str: # error: `private` does nothing here\n return \"Point()\"\n```", "default": "error", "oneOf": [ @@ -1470,6 +1480,16 @@ } ] }, + "invalid-override-visibility": { + "title": "detects a member declared less visible than the one it overrides", + "description": "## What it does\nChecks for a class member declared less visible than the one it inherits\nunder the same name.\n\n## Why is this bad?\nA visibility keyword decides the name the member is emitted under, so a\nmember declared less visible than the one it inherits does not override\nit. It sits beside it under a different name, and the inherited member is\nstill what a call finds — which is never what the declaration looks like\nit does.\n\n## Example\n\n```by\nclass A:\n def f(self) -> int:\n return 1\n\nclass B(A):\n private def f(self) -> int: # error: `f` is public on `A`\n return 2\n\nB().f() # 1, not 2\n```", + "default": "error", + "oneOf": [ + { + "$ref": "#/definitions/Level" + } + ] + }, "invalid-parameter-default": { "title": "detects default values that can't be assigned to the parameter's annotated type", "description": "## What it does\n\nChecks for default values that can't be assigned to the parameter's annotated type.\n\n## Why is this bad?\n\nThis breaks the rules of the type system and weakens a type checker's ability to accurately reason\nabout your code.\n\n## Examples\n\n```python\ndef f(a: int = \"\"): ... # error\n```", @@ -1522,7 +1542,7 @@ }, "invalid-raises-clause": { "title": "detects a `raises` clause that is not a set of exceptions", - "description": "## What it does\nChecks for a basedpython `raises` clause that does not describe a set of\nexceptions.\n\n## Why is this bad?\nOnly a `BaseException` subclass can be raised, so a clause with no\nexception in it can never be satisfied by anything the function does.\n\n## Example\n\n```by\ndef f() raises int: # error: `int` is not an exception\n ...\n```", + "description": "## What it does\nChecks for a basedpython `raises` clause that does not describe a set of\nexceptions.\n\n## Why is this bad?\nOnly a `BaseException` subclass can be raised, so a clause with no\nexception in it can never be satisfied by anything the function does.\n\nA type parameter in a clause stands for one type the caller chooses, so\nit has to be declared an exception as well — a parameter with no bound\ncan be `int` as easily as `OSError`.\n\n## Example\n\n```by\ndef f() raises int: # error: `int` is not an exception\n ...\n\ndef g[T](value: T) raises T: # error: `T@g` is not always an exception\n ...\n```", "default": "error", "oneOf": [ { @@ -1750,6 +1770,16 @@ } ] }, + "invalid-visibility": { + "title": "detects a visibility keyword where it cannot do what it says", + "description": "## What it does\nChecks for a visibility keyword written where it cannot do what it says.\n\n## Why is this bad?\nA visibility keyword renames the member or symbol it is written on, since\nthat is the only enforcement python offers. Some names cannot be renamed\nwithout changing what they mean: a dataclass field's name is its\nconstructor's keyword, an enum member's name is how the enum is looked up,\na `private` abstract method can never be overridden, and a dotted import\nbinds a package no rename can keep. The declaration would read as\nrestricted while doing something else.\n\n## Example\n\n```by\nfrom dataclasses import dataclass\n\n@dataclass\nclass Point:\n private x: int # error: a dataclass field's name is its constructor's keyword\n```", + "default": "error", + "oneOf": [ + { + "$ref": "#/definitions/Level" + } + ] + }, "invalid-yield": { "title": "detects yield expressions where the \"yield\" or \"send\" type is incompatible with the annotated return type", "description": "## What it does\n\nDetects `yield` and `yield from` expressions where the \"yield\" or \"send\" type is incompatible with\nthe generator function's annotated return type.\n\n## Why is this bad?\n\nYielding a value of a type that doesn't match the generator's declared yield type, or using\n`yield from` with a sub-iterator whose yield or send type is incompatible, is a type error that may\ncause downstream consumers of the generator to receive values of an unexpected type.\n\n## Examples\n\n```python\nfrom typing import Iterator\n\n\ndef gen() -> Iterator[int]:\n yield \"not an int\" # error: [invalid-yield]\n```", @@ -1850,6 +1880,16 @@ } ] }, + "missing-function-body": { + "title": "detects a `def` written with no body in a position that needs an implementation", + "description": "## What it does\n\nChecks for a `def` written with no body at all, in a position that needs an implementation.\n\n## Why is this bad?\n\nA `def` with no body declares a signature. The lowering fills in `: ...`, so the function exists and\nreturns `None`. That is what a declaration means in a stub file; anywhere else it is an\nimplementation that was never written, silently stood in for by one that does nothing.\n\nA body may be left out where a declaration is what the position asks for:\n\n- in a stub file\n- in an `if TYPE_CHECKING` block\n- as a member of a protocol class\n- as an `abstract def`, or an `@abstractmethod`-decorated method\n- as an overload declaration, written `@overload` or as a run of same-name `def`s\n\nAn `init(...)` may also be written without a body: what it does is store the attribute parameters it\ndeclares, and that body is built for it.\n\n## Examples\n\n```by\ndef parse(s: str) -> int # ok: the run below makes this an overload declaration\ndef parse(s: bytes) -> int\ndef parse(s):\n return int(s)\n\n# error: [missing-function-body]\ndef lookup() -> int\n```\n\nA function that is meant to do nothing says so with a body of its own:\n\n```by\ndef ignore(event: str): ...\n```", + "default": "error", + "oneOf": [ + { + "$ref": "#/definitions/Level" + } + ] + }, "missing-override-decorator": { "title": "detects methods that override a superclass member without an `@override` annotation", "description": "## What it does\n\nChecks for methods that override a method or attribute in a superclass but are not decorated with\n`@override`.\n\nThis rule is disabled by default. Enable it to opt in to strict `@override` enforcement for a\nproject.\n\n## Exemptions\n\nOverriding `__init__`, `__new__`, `__init_subclass__`, or `__post_init__` does not require\n`@override`, even if the method is explicitly declared by a superclass.\n\n## Why is this bad?\n\nWithout an `@override` annotation, refactors can silently change whether a method is an override.\nRequiring `@override` on every override lets ty report when an intended override stops overriding\nanything, and when a method unexpectedly starts overriding a superclass member.\n\n## Example\n\n```toml\n[environment]\npython-version = \"3.12\"\n```\n\n```python\nfrom typing import override\n\n\nclass Parent:\n def method(self) -> int:\n return 1\n\n\nclass Child(Parent):\n # when the rule is enabled\n def method(self) -> int: # error\n return 2\n\n\nclass ExplicitChild(Parent):\n @override\n def method(self) -> int: # fine\n return 2\n```", @@ -2130,6 +2170,16 @@ } ] }, + "private-export": { + "title": "detects a `private` symbol listed in `__all__`", + "description": "## What it does\nChecks for a `private` symbol listed in the module's `__all__`.\n\n## Why is this bad?\n`__all__` lists the module's interface, and a `private` symbol is declared\nnot to be part of it. The lowering renames the symbol with a leading\nunderscore, so `from m import *` would look up a name the module does not\nhave, and raise.\n\n## Example\n\n```by\nprivate def helper() -> int:\n return 1\n\n__all__ = [\"helper\"] # error: `helper` is private\n```", + "default": "error", + "oneOf": [ + { + "$ref": "#/definitions/Level" + } + ] + }, "private-import": { "title": "detects imports of another module's `private` symbols", "description": "## What it does\nChecks for imports of a symbol another module declared `private`.\n\n## Why is this bad?\nA `private` declaration is part of its module's implementation, not its\ninterface. It is renamed with a leading underscore by the lowering, so an\nimporting module is reaching past a boundary the author drew explicitly,\nand the symbol may be renamed or removed without notice.\n\n## Example\n\n```by\n# helpers.by\nprivate type Key = str | int\n\n# main.by\nfrom helpers import Key # error: `Key` is private to `helpers`\n```",