diff --git a/crates/by_build/src/annotate.rs b/crates/by_build/src/annotate.rs index 16bda2514b..dea938c24f 100644 --- a/crates/by_build/src/annotate.rs +++ b/crates/by_build/src/annotate.rs @@ -66,11 +66,26 @@ pub(crate) fn report(module: &ModuleIr) -> String { .map(|field| format!("{}: {}", field.name, field.ty)) .collect::>() .join(", "); + // a closure's environment and a generator's state are real emitted classes with + // real layouts, and nothing can name either: they are never bound in the module + // namespace. saying so is what lets a build compare this report against what an + // import actually stood a type under — see `BY_INSTALL_CENSUS` in the runtime + // header. left unsaid, the two sets could never be equal and the comparison that + // holds the report to what ran had nothing to compare + let notes = [ + class.immutable.then_some("frozen"), + (!class.exported).then_some("not published"), + ]; + let notes = notes.into_iter().flatten().collect::>(); let _ = writeln!( out, "## class {}{}\n\nfixed layout: {{{fields}}}\n", class.name, - if class.immutable { " (frozen)" } else { "" } + if notes.is_empty() { + String::new() + } else { + format!(" ({})", notes.join(", ")) + } ); // a property is not in the layout — it is a pair of compiled bodies behind one // attribute — so it would otherwise be invisible here @@ -176,6 +191,25 @@ frozen data class Point: assert!(text.contains("### Point.total"), "{text}"); } + #[test] + fn a_class_no_name_can_reach_says_so() { + // a closure's environment is a real emitted class with a real layout, and the + // module namespace never holds it. left unmarked it counted towards a figure + // that was being read as "classes an import stands a type under", which no + // import ever does for one of these + let module = lowered( + "\ +def make(n: int) -> (int) -> int: + def add(k: int) -> int: + return k + n + + return add +", + ); + let text = report(&module); + assert!(text.contains("## class make$env (not published)"), "{text}"); + } + #[test] fn a_borrowed_register_is_called_out() { let module = lowered( diff --git a/crates/by_build/src/lib.rs b/crates/by_build/src/lib.rs index 23ac643d2c..badf621531 100644 --- a/crates/by_build/src/lib.rs +++ b/crates/by_build/src/lib.rs @@ -154,7 +154,16 @@ fn create_parent(path: &Path) -> Result<()> { } /// what a build is allowed to leave interpreted -#[derive(Debug, Clone, Default)] +/// +/// [`Default`] is written out rather than derived because one of these is on by +/// default: a derived `false` for [`Self::verify_install`] would take the check out of +/// every build that did not name it, which is every build in the workspace +#[derive(Debug, Clone)] +#[expect( + clippy::struct_excessive_bools, + reason = "each is one independent `by compile` switch, and grouping them would only \ + put a name between the flag and the field it sets" +)] pub struct Options { /// reject a function declined because a type was gradual, instead of /// quietly leaving it interpreted. @@ -184,6 +193,16 @@ pub struct Options { /// trip through a different program — the transpiler inserts soundness checks /// and sentinels of its own — so it is used verbatim pub language: by_irbuild::Language, + /// have each licensed direct call re-ask, at runtime, the lookup it skips, and + /// abort where the two disagree. + /// + /// a *licence* is the compiler's decision that a call may go straight to a + /// compiled body because nothing can have put something else under the name. it + /// is a claim nothing checks, and one that is wrong is a wrong answer with + /// nothing to report it. this is that claim asked out loud, at every call it was + /// taken at — which costs the lookup the licence exists to avoid, so it is a + /// mode rather than the default + pub recheck_licences: bool, /// the transpiler configuration for the interpreted fallback, when there is /// one to transpile /// @@ -191,6 +210,39 @@ pub struct Options { /// from this source, so a build that means to insert extra soundness checks /// has to insert them here too or the two halves of the module disagree pub fallback: Option, + /// have module init end by checking that every class it reported as compiled is + /// the class standing under its own name when the import returns. + /// + /// on by default. a class that quietly leaves its interpreted definition standing + /// answers exactly as the twin does, so it agrees with every differential rung at + /// once while `--annotate` goes on reporting it compiled — which is how a coverage + /// figure becomes an upper bound without anyone noticing. the check runs once per + /// module at import rather than per call + pub verify_install: bool, +} + +impl Default for Options { + fn default() -> Self { + Self { + no_any: false, + require_native: false, + annotate: false, + language: by_irbuild::Language::default(), + fallback: None, + recheck_licences: false, + verify_install: true, + } + } +} + +impl Options { + /// what these options ask the lowering itself for + pub fn lowering(&self) -> by_irbuild::LowerOptions { + by_irbuild::LowerOptions { + language: self.language, + recheck_licences: self.recheck_licences, + } + } } /// write the `--annotate` report, when one was asked for @@ -225,7 +277,7 @@ fn lower( toolchain: Option<&Toolchain>, ) -> Result { finish( - by_irbuild::module_from_source(source, module_name, options.language), + by_irbuild::module_from_source(source, module_name, options.lowering()), source, options, toolchain, @@ -255,6 +307,8 @@ fn finish( )); } + module.verify_install = options.verify_install; + if options.require_native && !module.declined.is_empty() { bail!( "`require-native` is on and {} function(s) were left interpreted:\n{}", @@ -535,6 +589,7 @@ mod tests { fallback_source: None, fallback_code: None, shims: None, + verify_install: true, }; let dir = std::env::temp_dir().join("by_build_refuses_test"); let _ = fs::remove_dir_all(&dir); diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index 2ab15e8cb5..cac4d15f88 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -756,6 +756,53 @@ def _run_nested(m): a, b = _Recording(), _Recording() return (m.nested(a, b), a.seen, b.seen) +# a `with` site memoises what `__enter__` and `__exit__` resolved to, keyed on the +# manager's type and that type's version tag. every way the pair can move after the +# memo has been armed is asked here, and asked after enough passes to have armed it +def _rebound(m): + class Base: + def __enter__(self): return 'base enter' + def __exit__(self, *a): return False + + class Mgr(Base): + pass + + class Other: + def __enter__(self): return 'other enter' + def __exit__(self, *a): return False + + out = [] + mgr = Mgr() + for _ in range(20): + out.append(m.held(mgr)) + # a site that has settled on one class still has to notice the compiled class + # arriving at it, and then notice the first one arriving back + out.append(m.held(m.Own())) + out.append(m.held(mgr)) + # what the memo holds is declared on the *base*, and rebinding it there is the + # invalidation that has to reach the subclass + Base.__enter__ = lambda self: 'rebound on base' + out.append(m.held(mgr)) + # then on the class itself, where it now shadows the base's + Mgr.__enter__ = lambda self: 'rebound on class' + out.append(m.held(mgr)) + # `__exit__`'s answer is what decides whether the exception leaves the block, so + # a rebinding that changes its truthiness changes the program's control flow + out.append(repr(_capture(m.swallowed, mgr))) + Base.__exit__ = lambda self, *a: True + out.append(m.swallowed(mgr)) + Base.__exit__ = lambda self, *a: False + out.append(repr(_capture(m.swallowed, mgr))) + # a manager whose class was reassigned reaches the same site as a different type + mgr.__class__ = Other + out.append(m.held(mgr)) + # past the miss cap a site gives up memoising and looks the name up every time, + # and that path owes the same answers + for i in range(12): + Other.__enter__ = (lambda n: lambda self: 'late %d' % n)(i) + out.append(m.held(mgr)) + return out + def _chain(e): out = [] while e is not None: @@ -901,6 +948,21 @@ fn agree_inner(tag: &str, source: &str, calls: &[&str], allow_declines: bool) { ); } +/// whether this run has every licensed call re-ask the lookup it skipped +/// +/// on unless `BY_LICENCE_RECHECK=0` says otherwise. every case in this file is a +/// program whose two legs have to agree, so it is also a program every licence the +/// compiler hands out can be re-asked on — which is coverage no other suite has, and +/// the reason the default is the mode rather than the shipping configuration. +/// +/// the escape hatch is what the shipping configuration is tested through: with the mode +/// on, a licensed call site has one more use of its receiver, and the borrow and +/// refcount passes decide by use. a run with `BY_LICENCE_RECHECK=0` is the same suite +/// over the C a real build writes +fn recheck_licences() -> bool { + !std::env::var("BY_LICENCE_RECHECK").is_ok_and(|value| value == "0") +} + fn agree_in( tag: &str, source: &str, @@ -945,6 +1007,7 @@ fn agree_in( // the compiled leg let options = Options { language, + recheck_licences: recheck_licences(), ..Options::default() }; let built = match build_source(source, module.as_str(), &toolchain, &compiled_dir, &options) { @@ -2748,6 +2811,131 @@ def digits(n: int) -> str: ); } +/// the source the three tests below share +/// +/// `Held` is what puts them on the road they are about to walk. a module with a class +/// carrying a class-level constant has its body run with each `class` statement captured, +/// and it is that run which decides the builtins every function the body defines will +/// resolve through for the rest of the process — so a module with no such class never +/// reaches the code these test and passes them whatever it does. +/// +/// `eval` is what keeps `scaled` and `newcomer` off the compiled leg. a declined function +/// *is* the definition that run produced, which is why they are the half that can differ; +/// `compiled_scaled` is beside them to say what the same module's other half answers +const LIVE_BUILTINS_SOURCE: &str = "\ +class Held: + tag = 'held' + + def read(self) -> str: + return self.tag + +def scaled(n: int) -> int: + eval('0') + return abs(n) + +def compiled_scaled(n: int) -> int: + return abs(n) + +def newcomer() -> object: + eval('0') + return arriving +"; + +/// a builtin rebound after import is seen by a declined function +/// +/// both halves of the module are asked, because the answer that matters is not only that +/// the declined one is wrong but that it disagrees with the compiled one standing next to +/// it. a test that called `scaled` alone would still fail, but it would not show that one +/// module was answering `abs` two ways at once +#[test] +fn a_declined_function_sees_a_builtin_rebound_after_its_module_was_imported() { + agree_python_with_declines( + "livebuiltinrebind", + LIVE_BUILTINS_SOURCE, + &[ + "(setattr(__import__('builtins'), 'abs', lambda n: n * 100), \ + [m.compiled_scaled(-3), m.scaled(-3)])[1]", + ], + ); +} + +/// a builtin deleted after import stops answering a declined function +/// +/// the deletion is the case a stale namespace cannot express at all: it still holds the +/// entry, so the read succeeds where python's own would raise. the exception is compared +/// rather than caught and counted, so a `NameError` naming something else would not pass +#[test] +fn a_declined_function_stops_seeing_a_builtin_deleted_after_its_module_was_imported() { + agree_python_with_declines( + "livebuiltindelete", + LIVE_BUILTINS_SOURCE, + &["(delattr(__import__('builtins'), 'abs'), _capture(m.scaled, -3))[1]"], + ); +} + +/// a name added to builtins after import is found by a declined function +/// +/// the other direction of the same property: a namespace fixed at import has no entry to +/// find, and a `NameError` is what a stale one answers with +#[test] +fn a_declined_function_finds_a_name_added_to_builtins_after_its_module_was_imported() { + agree_python_with_declines( + "livebuiltinadd", + LIVE_BUILTINS_SOURCE, + &["(setattr(__import__('builtins'), 'arriving', 'here'), m.newcomer())[1]"], + ); +} + +/// a module body writing through `__builtins__` reaches the interpreter's own +/// +/// the other side of the same mapping being live: the body does not merely *read* what +/// the process has done to `builtins`, it writes where the process will see it. a run +/// given a namespace of its own would take this write and drop it on the floor at the end +/// of the import, which is a change made and silently undone +#[test] +fn a_module_body_writing_through_builtins_reaches_the_interpreter_s_own() { + agree_python( + "bodywritesbuiltins", + "\ +class Held: + tag = 'held' + + def read(self) -> str: + return self.tag + +__builtins__['planted'] = 'by the body' +", + &["[getattr(__import__('builtins'), 'planted', ''), m.Held.tag]"], + ); +} + +/// capturing a module's class bodies leaves a `class` statement written elsewhere alone +/// +/// the capture stands in the interpreter's own builtins while a body runs, so every +/// `class` statement in the process reaches it and only this module's have anything to do +/// with it — what tells them apart is the namespace the statement was written in. the two +/// classes are named the same on purpose: a capture that recorded the other one would +/// overwrite what this module's own statement wrote, and the constant `Shared` ends up +/// carrying is what says which body was read +#[test] +fn a_capturing_module_body_records_only_the_classes_written_in_it() { + agree_python( + "foreigncapture", + "\ +class Shared: + origin = 'outer' + + def where(self) -> str: + return self.origin + +_elsewhere: dict = {} +exec(\"class Shared:\\n origin = 'inner'\\n\", _elsewhere) +held = _elsewhere['Shared'] +", + &["[m.Shared.origin, m.Shared().where(), m.held.origin]"], + ); +} + /// a builtin rebound while a call site is already holding it is seen at once /// /// the site remembers which namespace answered, so *that* namespace has to be one @@ -3895,6 +4083,59 @@ SNAPSHOT = list(REGISTRY) ); } +/// and so is an attribute that exists only on what the decorator handed back +/// +/// the two cases above are a decorated name the body reads and an effect the body sees +/// under another name. this is the third and it is the one the standard library actually +/// breaks on: `render` is a `Table` from the moment its `def` stands, and `@render.register` +/// below reads an attribute a plain function does not have. move `@dispatcher` to init and +/// that read happens while the name still holds the function, so the import stops rather +/// than answering wrongly — `pkgutil` writes this as `@simplegeneric` with +/// `@iter_importer_modules.register` under it, and with this gate lifted its import fails +/// with `AttributeError: 'function' object has no attribute 'register'`. +/// +/// the assertions are about *identity* rather than an answer, because that is the half a +/// decorator moved to init gets wrong: `render` is the decorator's object on both legs, and +/// `other` is what `register` gave back +#[test] +fn a_decorated_definition_the_body_reaches_through_the_decorators_answer_declines() { + agree_python_with_declines( + "decoratoranswer", + "\ +class Table: + def register(self, case): + self.cases.append(case.__name__) + return case + + def __init__(self): + self.cases = [] + + +def dispatcher(f): + return Table() + + +@dispatcher +def render() -> str: + return \"default\" + + +@render.register +def other() -> str: + return \"other\" + + +AT_IMPORT = list(render.cases) +", + &[ + "type(m.render).__name__", + "m.render.cases", + "m.AT_IMPORT", + "m.other()", + ], + ); +} + /// the source both method-decorator tests below compile /// /// `mark` hands back what it was given, so the binding is right however many times it ran @@ -7208,7 +7449,9 @@ frozen data class Fixed: \x20 except (TypeError, AttributeError) as e:\n print(type(e).__name__)\n\ \x20 else:\n print('accepted')\n", ); - assert_eq!(out, "5\nTypeError\nAttributeError\nAttributeError"); + // the frozen write is *not* part of that delta: `FrozenInstanceError` is what the + // interpreted twin raises, and the emitted type's `__setattr__` raises the same + assert_eq!(out, "5\nTypeError\nAttributeError\nFrozenInstanceError"); } #[test] @@ -8633,6 +8876,54 @@ def accumulate(values: list[int]) -> int: ); } +/// a cell is read again on every call, however often the call is repeated +/// +/// a closure called in a loop is the shape a compiled build has the most to gain from +/// and the most to get wrong: the cell is one location two frames share, and anything +/// that remembers what it last held answers the value before the rebinding rather than +/// the one after it. that is a silent wrong answer, so the interleavings here rebind +/// the cell from outside the closure that reads it, between calls to that closure, and +/// from a third frame that closes over the same cell +#[test] +fn a_capture_rebound_between_calls_is_read_again() { + agree( + "cellrebind", + "\ +def pair(start: int) -> list[object]: + total = start + def step(by: int) -> int: + nonlocal total + total = total + by + return total + def put(v: int) -> int: + nonlocal total + total = v + return total + return [step, put] + +def repeated(n: int) -> int: + total = 0 + def step(by: int) -> int: + nonlocal total + total = total + by + return total + last = 0 + i = 0 + while i < n: + last = step(i) + i = i + 1 + return last +", + &[ + // the same call twice running, then the cell put back from elsewhere, then + // the same call twice again — a remembered read shows on the fourth + "[(p := m.pair(0), p[0](1), p[0](1), p[1](100), p[0](1), p[0](1))[1:]]", + "[(p := m.pair(7), p[1](-7), p[0](0), p[1](10 ** 20), p[0](1))[1:]]", + "[m.repeated(n) for n in (0, 1, 2, 50)]", + ], + ); +} + #[test] fn reading_a_cell_before_it_is_written_raises_the_way_python_does() { // a cell starts unset, and NULL has to read back as an error rather than a zero @@ -10473,6 +10764,137 @@ def entered(mgr: object) -> object: ); } +#[test] +fn a_with_block_sees_its_manager_rebound_after_it_first_ran() { + // a `with` site remembers what the two protocol names resolved to, so the answer + // it holds can be made stale by a write it was not present for. a memo that + // missed one would go on calling the method that has been replaced — and for + // `__exit__` that is not a stale value but a live exception either swallowed or + // let out against the program's wishes + agree_with_declines( + "withrebound", + "\ +class Own: + def __enter__(self) -> str: + return \"own enter\" + + def __exit__(self, kind: object, value: object, tb: object) -> bool: + return False + +def held(mgr: object) -> object: + with mgr as value: + return value + return None + +def swallowed(mgr: object) -> str: + with mgr: + raise ValueError(\"boom\") + return \"suppressed\" +", + &["_rebound(m)"], + ); +} + +#[test] +fn a_with_block_over_a_class_the_module_declares_reaches_both_halves_of_it() { + // a manager the compiler knows the class of reaches `__enter__` and `__exit__` + // without asking the object protocol for either, the way `o.m()` already does. all + // three exits have to arrive at the same `__exit__`: falling off the end of the + // block, leaving it by `return`, and unwinding out of it + agree_python( + "ownctx", + "\ +class Guard: + def __init__(self) -> None: + self.log: list[str] = [] + + def __enter__(self) -> str: + self.log.append('enter') + return 'held' + + def __exit__(self, kind: object, value: object, tb: object) -> bool: + self.log.append('exit ' + str(kind is None)) + return False + + +class Swallow: + def __enter__(self) -> str: + return 'held' + + # a truthy answer suppresses an exception that is there, and suppresses nothing + # on the way out of a block that raised none + def __exit__(self, kind: object, value: object, tb: object) -> bool: + return True + + +def fallen_off() -> list[str]: + guard = Guard() + with guard as name: + guard.log.append('body ' + name) + return guard.log + + +def returned() -> list[str]: + guard = Guard() + with guard as name: + guard.log.append('body ' + name) + return guard.log + return [] + + +def broken(n: int) -> list[str]: + guard = Guard() + i = 0 + while i < n: + with guard: + if i == 1: + break + i = i + 1 + return guard.log + + +def unwound() -> list[str]: + guard = Guard() + try: + with guard: + raise ValueError('boom') + except ValueError: + guard.log.append('caught') + return guard.log + + +def nested() -> list[str]: + outer = Guard() + with outer: + inner = Guard() + with inner: + outer.log.append('inner ' + str(len(inner.log))) + return outer.log + + +def suppressed() -> str: + with Swallow(): + raise ValueError('boom') + return 'suppressed' + + +def not_suppressing() -> str: + with Swallow(): + pass + return 'fell off' +", + &[ + "m.fallen_off()", + "m.returned()", + "m.broken(4)", + "m.unwound()", + "m.nested()", + "m.suppressed()", + "m.not_suppressing()", + ], + ); +} + #[test] fn an_early_exit_runs_the_finally_it_is_leaving() { // this was a silent wrong answer in a shipped feature: a `return` or a `break` @@ -10782,6 +11204,49 @@ def calls_none() -> int: ); } +/// a `*args` is read as the tuple the calling convention built it as, which is a +/// guess about the container and not a licence to skip anything the general read +/// does. python's own answer to a position off either end of it, to a position +/// counted from the end, and to an index that is not a position at all has to be +/// the answer either way — wording included +#[test] +fn indexing_a_variadic_agrees() { + agree( + "varindex", + "\ +def at(*rest: int) -> int: + return rest[0] + +def from_the_end(*rest: int) -> int: + return rest[-1] + +def at_position(index: int, *rest: int) -> int: + return rest[index] + +def rebound(*rest: int) -> int: + rest = (7, 8) + return rest[1] + +def calls_at(a: int, b: int) -> int: + return at(a, b) +", + &[ + "m.at(1, 2)", + "m.from_the_end(1, 2)", + "m.at_position(1, 10, 20)", + "m.at_position(-2, 10, 20)", + "m.rebound()", + "m.calls_at(3, 4)", + // every way the position can be no position at all + "[(type(e).__name__, str(e)) for e in [_capture(m.at)]]", + "[(type(e).__name__, str(e)) for e in [_capture(m.from_the_end)]]", + "[(type(e).__name__, str(e)) for e in [_capture(m.at_position, 2, 10)]]", + "[(type(e).__name__, str(e)) for e in [_capture(m.at_position, -3, 10)]]", + "[(type(e).__name__, str(e)) for e in [_capture(m.at_position, 1 << 70, 10)]]", + ], + ); +} + #[test] fn a_variadic_argument_does_not_leak() { let Some((python, toolchain)) = environment() else { @@ -10950,6 +11415,89 @@ def refused(reading: Reading) -> str: ); } +/// a property on a class another class extends is still whatever the receiver's type +/// says it is +/// +/// `Cell` has an in-module subclass, so it is emitted as a heap type an interpreted class +/// may subclass and whose attributes may be rebound — and a compiled read or write of the +/// pair is a *tested* call to the compiled half rather than a direct one. these are the +/// receivers the test has to send back round the descriptor protocol, and each of them +/// arrives after this module has been imported and its licence taken out: a subclass +/// written in the interpreter that overrides a half, one that overrides the pair, and a +/// half rebound on the class itself +#[test] +fn a_property_on_an_extended_class_sees_a_later_override() { + agree_python( + "propopen", + "\ +class Cell: + def __init__(self, n: int) -> None: + self._n = n + + @property + def v(self) -> int: + return self._n + + @v.setter + def v(self, given: int) -> None: + self._n = given + + +class Sized(Cell): + def width(self) -> int: + return 1 + + +class Fixed: + def __init__(self, n: int) -> None: + self._n = n + + @property + def v(self) -> int: + return self._n + + +class Wider(Fixed): + def width(self) -> int: + return 2 + + +def read(cell: Cell) -> int: + return cell.v + + +def write(cell: Cell, given: int) -> None: + cell.v = given + + +def read_fixed(fixed: Fixed) -> int: + return fixed.v +", + &[ + // the shape the licence is for: an exact `Cell`, read and written + "m.read(m.Cell(3))", + "(lambda c: (m.write(c, 4), m.read(c)))(m.Cell(0))", + // an interpreted subclass overriding the getter, built after import + "m.read(type('Get', (m.Cell,), {'v': property(lambda self: 99)})(1))", + // one overriding both halves, so that the write lands where it says + "(lambda t: (lambda c: (m.write(c, 4), c.seen))(t(0)))\ + (type('Both', (m.Cell,), \ + {'v': property(lambda self: 0, lambda self, given: setattr(self, 'seen', given * 2))}))", + // the in-module subclass inherits the pair, and reaches it the same way + "m.read(m.Sized(5))", + // a group with no setter written under it is licensed on the same terms, and + // the receivers that have to go round the protocol are the same ones + "m.read_fixed(m.Fixed(6))", + "m.read_fixed(type('Given', (m.Fixed,), \ + {'v': property(lambda self: 0, lambda self, g: None)})(6))", + // and the pair rebound on the class itself, which no receiver's type shows + "(lambda _: m.read(m.Cell(3)))(setattr(m.Cell, 'v', property(lambda self: 77)))", + "(lambda _: m.read_fixed(m.Fixed(6)))\ + (setattr(m.Fixed, 'v', property(lambda self: 8, lambda self, g: None)))", + ], + ); +} + /// what the type publishes under a property's name is a `property` /// /// this is the half no behavioural comparison can reach. an attribute published through @@ -11087,34 +11635,282 @@ class Box: ); } -/// a `@property` with no setter answers and refuses exactly as the interpreted one does +/// a published property answers `__doc__` with what the getter's body opens with /// -/// the group of one is published rather than left to the class body's own object, so -/// every answer it gives is now the compiled type's: the read runs the getter's body, and -/// the write and the `del` are refused by the `property` itself — in python's own wording, -/// which names the property, so it is also what says `__set_name__` reached it. -/// -/// `refused` asks the same question from a *compiled* frame with a typed receiver, which -/// calls the half outright instead of going round the descriptor. that route has its own -/// way of getting a missing half wrong: a write with nowhere to go could land beside the -/// property instead of raising. +/// two surfaces, and one docstring behind both. `p.fget.__doc__` is read straight off the +/// `PyMethodDef` the half is built from, and `p.__doc__` is python's own: `property` takes +/// its documentation off the getter exactly when it is handed none, which is what +/// `By_PublishProperty` does. /// -/// none of it says *which* leg answered, and it cannot: an interpreted fallback answers a -/// property exactly as the published one does, so every line here passes with the group of -/// one left unlowered. the test above is what says the compiled body is the one running +/// this is the shape of divergence no comparison of *answers* reaches. the property reads, +/// writes and refuses identically with the entry left NULL — it simply says `None` where +/// the interpreted class says the text, and every other property test here passed with it #[test] -fn a_lone_property_getter_agrees() { - agree_python( - "proplonediff", - "\ +fn a_published_property_carries_the_getters_docstring() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_propdoc"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ class Box: def __init__(self, n: int) -> None: self._n = n @property def value(self) -> int: - return self._n * 10 - + \"what the box holds\" + return self._n + + @value.setter + def value(self, given: int) -> None: + \"put something else in it\" + self._n = given + + @property + def lone(self) -> int: + \"a group of one documents itself the same way\" + return self._n + 1 + + @property + def bare(self) -> int: + return 0 +"; + let built = match build_source( + source, + "by_diff_propdoc", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_propdoc as m\n\ + for name in ('value', 'lone', 'bare'):\n\ + \x20 p = m.Box.__dict__[name]\n\ + \x20 print(type(p.fget).__name__, repr(p.__doc__), repr(p.fget.__doc__))\n\ + print(repr(m.Box.__dict__['value'].fset.__doc__))\n", + ); + assert_eq!( + out, + "method_descriptor 'what the box holds' 'what the box holds'\n\ + method_descriptor 'a group of one documents itself the same way' 'a group of one documents itself the same way'\n\ + method_descriptor None None\n\ + 'put something else in it'" + ); +} + +/// a `@property` over an abstract one is not still abstract on the emitted class +/// +/// a class whose base carries `abc.ABCMeta` is built by *calling* that metaclass, because +/// a type spec gives what it builds `type` as its own and any other metaclass on a base is +/// a conflict. which of the two applies is a runtime answer — it depends on what the base +/// names resolved to — so nothing at build time can tell this class apart from one a spec +/// will build. +/// +/// `ABCMeta.__new__` decides what is still abstract from the namespace it is handed, and a +/// property's halves are in no method table, so the namespace has to carry the `property` +/// the interpreted body built or the name is simply not there. it was not: the emitted +/// class reported `value` abstract and refused to be instantiated at all, while the +/// interpreted twin of the same source built. nothing declined and nothing was reported +/// +/// what the class answers with here is that carried `property` — the interpreted body, +/// which is what a class statement would have left under the name +#[test] +fn a_property_over_an_abstract_base_is_not_left_abstract() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_propabstract"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +import abc + + +class Held(abc.ABC): + @property + @abc.abstractmethod + def value(self) -> int: ... + + @property + @abc.abstractmethod + def lone(self) -> int: ... + + +class Box(Held): + @property + def value(self) -> int: + return 5 + + @value.setter + def value(self, given: int) -> None: + raise RuntimeError(\"no\") + + @property + def lone(self) -> int: + return 6 +"; + let built = match build_source( + source, + "by_diff_propabstract", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + // `Held`'s two stubs decline, because a body of `...` reaches its end without + // returning what it says it returns. `Box` itself is emitted whole, which is the + // question here + assert!( + !built + .declined + .iter() + .any(|declined| declined.name.starts_with("Box")), + "declined: {:?}", + built.declined + ); + let out = run( + &python, + &dir, + "import by_diff_propabstract as m\n\ + print(sorted(m.Box.__abstractmethods__))\n\ + b = m.Box()\n\ + print(b.value, b.lone)\n\ + for name in ('value', 'lone'):\n\ + \x20 p = m.Box.__dict__[name]\n\ + \x20 print(type(p).__name__, type(p.fget).__name__, repr(p.__doc__))\n", + ); + assert_eq!( + out, + "[]\n\ + 5 6\n\ + property function None\n\ + property function None" + ); +} + +/// a group of one on a class with a base of this module's is published over the body +/// +/// such a class is built from a spec standing on the base's finished type, because a class +/// this module built from a spec has `type` for its own metaclass — so nothing was handed a +/// namespace and the published property is the only thing that ever reaches the name. +/// +/// a base used to be enough on its own to leave the group alone, which is what left 101 of +/// the standard library's lone getters running interpreted while their classes compiled. +/// what makes the base safe is not the base: it is that the two constructions now answer +/// the question separately at import +#[test] +fn a_lone_property_getter_over_an_emitted_base_is_published() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_propbase"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Held: + def held(self) -> int: + return 1 + + +class Box(Held): + @property + def value(self) -> int: + \"the value over a base\" + return self.held() * 10 +"; + let built = match build_source( + source, + "by_diff_propbase", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_propbase as m\n\ + p = m.Box.__dict__['value']\n\ + print(type(p).__name__, isinstance(p, property), p.fset, p.fdel)\n\ + print(type(p.fget).__name__, p.fget.__name__, p.fget.__qualname__, repr(p.__doc__))\n\ + b = m.Box()\n\ + print(b.value, p.__get__(b), hasattr(m.Box, 'value$get'))\n\ + for verb, run_it in (('set', lambda: setattr(b, 'value', 1)),\n\ + \x20 ('del', lambda: delattr(b, 'value'))):\n\ + \x20 try:\n\ + \x20 run_it()\n\ + \x20 except AttributeError as e:\n\ + \x20 print(verb, e)\n", + ); + assert_eq!( + out, + "property True None None\n\ + method_descriptor value Box.value 'the value over a base'\n\ + 10 10 False\n\ + set property 'value' of 'Box' object has no setter\n\ + del property 'value' of 'Box' object has no deleter" + ); +} + +/// a `@property` with no setter answers and refuses exactly as the interpreted one does +/// +/// the group of one is published rather than left to the class body's own object, so +/// every answer it gives is now the compiled type's: the read runs the getter's body, and +/// the write and the `del` are refused by the `property` itself — in python's own wording, +/// which names the property, so it is also what says `__set_name__` reached it. +/// +/// `refused` asks the same question from a *compiled* frame with a typed receiver, which +/// calls the half outright instead of going round the descriptor. that route has its own +/// way of getting a missing half wrong: a write with nowhere to go could land beside the +/// property instead of raising. +/// +/// none of it says *which* leg answered, and it cannot: an interpreted fallback answers a +/// property exactly as the published one does, so every line here passes with the group of +/// one left unlowered. the test above is what says the compiled body is the one running +#[test] +fn a_lone_property_getter_agrees() { + agree_python( + "proplonediff", + "\ +class Box: + def __init__(self, n: int) -> None: + self._n = n + + @property + def value(self) -> int: + return self._n * 10 + def read(box: Box) -> int: return box.value @@ -11439,64 +12235,377 @@ class Wide: assert_eq!(out, "2 3 5 property"); } -#[test] -fn a_decorated_method_agrees() { - agree_with_declines( - "methoddeco", +/// a property written as a `get`/`set` block answers exactly as the pair it lowers to +/// +/// the surface is the only difference: the transpiler emits the same `@property` and +/// `@value.setter` over the same backing storage, so the compiled class has to publish +/// the same object over the same halves. every reflective way of reaching a property is +/// asked here, because a construct the backend recognises through parser markers rather +/// than through written decorators could publish something that reads right and reflects +/// wrong +#[test] +fn an_accessor_block_agrees() { + agree( + "accessorblock", "\ -def doubling(fn: object) -> object: - def wrapper(self: object) -> object: - return fn(self) * 2 - return wrapper +class Cell: + var v: int + get(): + \"\"\"what the cell holds\"\"\" + return field + set(given): + field = given -data class Point: - x: int - y: int + let doubled: int + get() = self.v * 2 - @property - def total(self) -> int: - return self.x + self.y - @doubling - def raw(self) -> int: - return self.x +def store(cell: Cell, given: int) -> int: + cell.v = given + return cell.v + + +def raised(fn: object) -> str: + try: + fn() + except AttributeError as error: + return str(error) + return 'nothing raised' ", &[ - // a property is a descriptor on the type, reached without a call - "m.Point(3, 4).total", - "type(m.Point.total).__name__", - // and a user decorator wraps the native method - "m.Point(3, 4).raw()", + "m.store(m.Cell(), 5)", + "(lambda c: (m.store(c, 5), c.v, c.doubled))(m.Cell())", + // what the type holds under each name, and what the object it holds is made of + "type(vars(m.Cell)['v']).__name__", + "isinstance(vars(m.Cell)['v'], property)", + "(vars(m.Cell)['v'].fget.__name__, vars(m.Cell)['v'].fget.__qualname__)", + "vars(m.Cell)['v'].fget.__doc__", + "vars(m.Cell)['v'].__doc__", + "vars(m.Cell)['v'].fdel", + "vars(m.Cell)['doubled'].fset", + // reached as a descriptor rather than through the instance + "(lambda c: (m.store(c, 3), vars(m.Cell)['v'].__get__(c, m.Cell)))(m.Cell())", + // the refusals name the property, which is what says `__set_name__` reached it + "m.raised(lambda: delattr(m.Cell(), 'v'))", + "m.raised(lambda: setattr(m.Cell(), 'doubled', 1))", + // the storage is the property's own, and a read before the first write + // raises rather than answering some class-level value + "m.raised(lambda: m.Cell().v)", + // each verb builds a new property and leaves the class holding the old one + "(lambda p: (type(p.deleter(lambda self: None)).__name__, p is vars(m.Cell)['v']))(vars(m.Cell)['v'])", ], ); } -/// a decorator that *mutates* what it is handed, next to one that wraps +/// and the halves the published property holds are this module's own bodies /// -/// `abc.abstractmethod` writes `__isabstractmethod__` onto its argument and hands the -/// same object back, so it is the whole class of decorator a compiled method has to -/// stay writable for — a method descriptor takes no attributes at all. and the two -/// class constructions have to be covered separately: `Plain` is built from a spec, and -/// the decorated method is put onto the finished type, while `Shape`'s metaclass rules a -/// spec out and the same value goes into the namespace the metaclass is handed instead +/// [`an_accessor_block_agrees`] cannot say that: an interpreted fallback publishes a +/// `property` that answers every one of those questions the same way. what tells the two +/// apart is the half inside — a `method_descriptor` is compiled, a `function` is the +/// interpreted definition #[test] -fn a_mutating_method_decorator_agrees() { - agree_python( - "mutatingdeco", - "\ -from abc import ABC, abstractmethod - +fn an_accessor_blocks_halves_are_the_compiled_bodies() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_accessorhalves"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Cell: + var v: int + get() = field + set(given): + field = given +"; + let built = match build_source( + source, + "by_diff_accessorhalves", + &toolchain, + &dir, + &Options::default(), + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_accessorhalves as m\n\ + p = m.Cell.__dict__['v']\n\ + print(type(p).__name__, type(p.fget).__name__, type(p.fset).__name__)\n", + ); + assert_eq!(out, "property method_descriptor method_descriptor"); +} -def doubling(fn: object) -> object: - def wrapper(self: object) -> object: - return fn(self) * 2 - return wrapper +/// an accessor block whose storage carries an initialiser +/// +/// the transpiler moves that initialiser into an `__init__` it injects, so every instance +/// gets storage of its own rather than sharing one object the class holds. the emitted +/// class writes it at construction for the same reason — the value rides on the field as +/// the constructor's default rather than being bound to the class — and the two arrive at +/// the same answers, including the message a constructor that takes no arguments gives +/// when handed one — which is the injected `__init__`'s and not `object.__init__`'s. +/// +/// the two constructions are what say the storage is per-instance: a class-level value +/// would leave the second one reading what the first one wrote +#[test] +fn an_accessor_block_with_an_initialiser_agrees() { + agree( + "accessorinit", + "\ +class Cell: + var v: int = 4 + get() = field + set(given): + field = given -def tagging(fn: object) -> object: - def wrapper(self: object) -> object: - return str(fn(self)) + '!' - return wrapper +def store(cell: Cell, given: int) -> int: + cell.v = given + return cell.v + + +def raised(fn: object) -> str: + try: + fn() + except TypeError as error: + return str(error) + except AttributeError as error: + return str(error) + return 'nothing raised' +", + &[ + "m.Cell().v", + "m.store(m.Cell(), 5)", + "type(vars(m.Cell)['v']).__name__", + "vars(m.Cell)['v'].fget(m.Cell())", + // the storage each instance gets is its own: writing one leaves the next + // construction still answering with the initialiser + "(lambda c: (m.store(c, 5), m.Cell().v))(m.Cell())", + "m.Cell()._Cell__v", + "m.raised(lambda: m.Cell(1))", + "m.raised(lambda: m.Cell(v=1))", + "m.raised(lambda: delattr(m.Cell(), 'v'))", + "isinstance(vars(m.Cell)['v'], property)", + ], + ); +} + +/// three accessor blocks over one class, each with an initialiser of its own +/// +/// the constructor writes one value per field rather than one field, and a `str` is the +/// one of these whose value is refcounted — a construction that took it without a +/// reference of its own would hand back a string the next collection frees +#[test] +fn several_accessor_block_initialisers_agree() { + agree( + "accessorinits", + "\ +class Many: + var name: str = 'ada' + get() = field + set(given): + field = given + + var ratio: float = 1.5 + get() = field + set(given): + field = given + + var on: bool = True + get() = field + set(given): + field = given + + +def touch(m: Many) -> str: + m.name = m.name + '!' + return m.name + + +def many() -> str: + last = '' + i = 0 + while i < 5000: + last = Many().name + i = i + 1 + return last +", + &[ + "(m.Many().name, m.Many().ratio, m.Many().on)", + "m.touch(m.Many())", + "(lambda a: (m.touch(a), m.Many().name))(m.Many())", + "m.many()", + ], + ); +} + +/// a read-only accessor block, and one a class in the same module extends +/// +/// the read-only form is the shape with no assignment to the receiver anywhere, so the +/// class-body declaration is the only thing saying the storage exists at all. the subclass +/// is the other construction: a class another one extends is emitted as a mutable heap +/// type, and the subclass inherits both the layout and the value written into it +#[test] +fn an_accessor_blocks_initialiser_reaches_a_subclass() { + agree( + "accessorinitsub", + "\ +class Read: + let v: int = 7 + get() = field + + +class Cell: + var v: int = 4 + get() = field + set(given): + field = given + + +class Sized(Cell): + def width(self) -> int: + return 1 + + +def grow(cell: Cell) -> int: + cell.v = cell.v + 1 + return cell.v + + +def raised(fn: object) -> str: + try: + fn() + except TypeError as error: + return str(error) + return 'nothing raised' +", + &[ + "m.Read().v", + "m.Cell().v", + "m.Sized().v", + "m.grow(m.Cell())", + "m.grow(m.Sized())", + "m.Sized().width()", + // the value the subclass inherits is written into each of its instances too + "(lambda s: (m.grow(s), m.Sized().v))(m.Sized())", + "m.raised(lambda: m.Sized(1))", + "m.raised(lambda: m.Read(1))", + ], + ); +} + +/// and the halves of that pair are this module's own bodies, over a written field +/// +/// [`an_accessor_block_with_an_initialiser_agrees`] cannot say so: a declined class +/// answers every one of those questions through its interpreted definition. this is what +/// separates a class that reached the layout from one that only agrees with it +#[test] +fn an_initialised_accessor_blocks_halves_are_the_compiled_bodies() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_accessorinithalves"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Cell: + var v: int = 4 + get() = field + set(given): + field = given +"; + let built = match build_source( + source, + "by_diff_accessorinithalves", + &toolchain, + &dir, + &Options::default(), + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_accessorinithalves as m\n\ + p = m.Cell.__dict__['v']\n\ + print(type(p).__name__, type(p.fget).__name__, type(p.fset).__name__)\n\ + print(type(m.Cell.__dict__['_Cell__v']).__name__, m.Cell().v)\n", + ); + assert_eq!( + out, + "property method_descriptor method_descriptor\ngetset_descriptor 4" + ); +} + +#[test] +fn a_decorated_method_agrees() { + agree_with_declines( + "methoddeco", + "\ +def doubling(fn: object) -> object: + def wrapper(self: object) -> object: + return fn(self) * 2 + return wrapper + +data class Point: + x: int + y: int + + @property + def total(self) -> int: + return self.x + self.y + + @doubling + def raw(self) -> int: + return self.x +", + &[ + // a property is a descriptor on the type, reached without a call + "m.Point(3, 4).total", + "type(m.Point.total).__name__", + // and a user decorator wraps the native method + "m.Point(3, 4).raw()", + ], + ); +} + +/// a decorator that *mutates* what it is handed, next to one that wraps +/// +/// `abc.abstractmethod` writes `__isabstractmethod__` onto its argument and hands the +/// same object back, so it is the whole class of decorator a compiled method has to +/// stay writable for — a method descriptor takes no attributes at all. and the two +/// class constructions have to be covered separately: `Plain` is built from a spec, and +/// the decorated method is put onto the finished type, while `Shape`'s metaclass rules a +/// spec out and the same value goes into the namespace the metaclass is handed instead +#[test] +fn a_mutating_method_decorator_agrees() { + agree_python( + "mutatingdeco", + "\ +from abc import ABC, abstractmethod + + +def doubling(fn: object) -> object: + def wrapper(self: object) -> object: + return fn(self) * 2 + return wrapper + + +def tagging(fn: object) -> object: + def wrapper(self: object) -> object: + return str(fn(self)) + '!' + return wrapper class Plain: @@ -12076,6 +13185,92 @@ def declared_global(n: int) -> int: ); } +#[test] +fn a_module_binding_slice_or_ellipsis_does_not_change_what_the_punctuation_means() { + // `a[i:j]` and `...` are punctuation rather than names: python builds the slice + // from `BUILD_SLICE` and loads the singleton as a constant, and neither reads the + // module namespace. lowering them as a call to `slice` and a read of `Ellipsis` + // meant a module binding either name for itself was obeyed where python ignores + // it — `ast` binds both, so `self._source[i:]` inside its unparser built a + // deprecated AST node and raised `TypeError: slice() takes no arguments`. + // + // the two classes are what the names would otherwise resolve to, and reading them + // back is what says the module really does bind them + agree_python( + "shadowedslice", + "\ +class slice: + def __init__(self) -> None: + self.tag = 'not a slice' + +class Ellipsis: + def __init__(self) -> None: + self.tag = 'not the singleton' + +def sliced(xs: list[int]) -> str: + return str(xs[1:3]) + str(xs[::2]) + str(xs[2:]) + str(xs[::-1]) + +def assigned(xs: list[int]) -> str: + xs[1:3] = [9, 9, 9] + return str(xs) + +def dots() -> bool: + return ... is Ellipsis + +def bound() -> str: + return slice().tag + Ellipsis().tag +", + &[ + "m.sliced([1, 2, 3, 4])", + "m.assigned([1, 2, 3, 4])", + "m.dots()", + "m.bound()", + "type(m.slice).__name__", + "m.Ellipsis().tag", + ], + ); +} + +#[test] +fn a_call_handed_back_to_the_interpreted_definition_keeps_its_keywords() { + // a parameter whose default the compiler cannot inline — `-1` is a negation rather + // than a literal, so only the interpreted definition holds it — makes every call + // that omits it reach that definition instead of the native entry. that hand-back + // used to pass the positional arguments and nothing else, so a keyword the caller + // wrote was dropped and the callee fell back on its *own* default for it. + // + // `ast.literal_eval` is where this was found: it calls `parse(source, mode='eval')` + // and `parse` has such a default, so the parse ran in `exec` mode and every literal + // came back a `Module` the converter refused with `malformed node or string`. + // + // `named` is the same call shape with a default the compiler *can* inline, which + // reaches the native entry — the two together say the keyword survives both routes + agree_python( + "deferredkeyword", + "\ +def deferring(a: str, b: str = 'B', c: int = -1) -> str: + return a + b + str(c) + +def inlined(a: str, b: str = 'B', c: int = 1) -> str: + return a + b + str(c) + +def by_keyword(x: str) -> str: + return deferring(x, b='b') + +def by_keyword_out_of_order(x: str) -> str: + return deferring(x, c=7, b='b') + +def named(x: str) -> str: + return inlined(x, b='b') +", + &[ + "m.by_keyword('a')", + "m.by_keyword_out_of_order('a')", + "m.named('a')", + ], + ); +} + #[test] fn a_bad_first_argument_raises_rather_than_crashing() { // the wrapper releases every argument local on the error path, so one whose @@ -13066,6 +14261,127 @@ fn the_classes_beside_a_refused_one_lay_out_and_deallocate() { assert_eq!(out, "True\n(0, 0, 0) (0, 0, 0)"); } +/// a family standing on a heap base from another module, which is the shape the standard +/// library's handler and transport hierarchies have +/// +/// `Base` keeps a field past a `SubprocessError` instance, so it is built from a type +/// spec — and no spec can stand on a heap base, so it refuses. what that used to cost was +/// the whole module: `Rotating` names `Base` in its own header, a NULL is not something a +/// bases tuple can be packed from, and so a base another class stands on was always the +/// module's to refuse. `count` was left interpreted along with it. +/// +/// nothing outside the family reads either class, so the two stand down together instead. +/// they have to move as one in both directions: leaving `Rotating` interpreted while +/// `Base`'s emitted type took the base's name would leave it standing on an orphaned copy, +/// and `isinstance(m.Rotating(...), m.Base)` would answer False where python answers True. +/// +/// `asyncio.unix_events` is the module this is for. its four classes stand on heap bases +/// from other asyncio modules, only each other names them, and one refusal left every one +/// of them — and every function in the module — interpreted +const REFUSED_FAMILY: &str = "\ +from subprocess import SubprocessError + + +class Base(SubprocessError): + def __init__(self, tag): + SubprocessError.__init__(self, tag) + self.tag = tag + + def label(self): + return 'base:' + self.tag + + +class Rotating(Base): + def __init__(self, tag, limit): + Base.__init__(self, tag) + self.limit = limit + + def label(self): + return 'rotating:' + self.tag + ':' + str(self.limit) + + +def count(values): + total = 0 + for value in values: + total += value + return total +"; + +#[test] +fn a_family_no_spec_can_build_agrees_beside_a_module_function() { + agree_python( + "refusedfamily", + REFUSED_FAMILY, + &[ + "(m.Base('b').label(), m.Rotating('r', 2).label(), m.count([1, 2, 3]))", + "[(e.tag, e.args, str(e)) for e in (m.Base('b'), m.Rotating('r', 2))]", + "[c.__name__ for c in m.Rotating.__mro__]", + // the pair's identity, which is what parting them company would break + "(isinstance(m.Rotating('r', 2), m.Base),\n\ + \x20 issubclass(m.Rotating, m.Base), m.Rotating.__base__ is m.Base,\n\ + \x20 isinstance(m.Base('b'), __import__('subprocess').SubprocessError))", + // a field the base declared, read and written through the subclass + "[(e.tag, (setattr(e, 'tag', 'w'), e.tag)[1], e.label())\n\ + \x20 for e in [m.Rotating('r', 2)]]", + // raised and caught, which is what an exception family is for + "[(type(e).__name__, e.tag, str(e)) for e in\n\ + \x20 (_raised_and_caught(m.Base, 'boom'),\n\ + \x20 _raised_and_caught(m.Rotating, 'bang', 3))]", + ], + ); +} + +/// which build answered, which no comparison of the two legs can say +/// +/// the family is interpreted either way — `__firstlineno__` is in both class dicts, since +/// a spec has no code object to write one from — so the only thing that moves is whether +/// the module's own function was installed. before the family could stand down together, +/// `by_exec` gave up before `PyModule_AddFunctions` ran and `count` was an ordinary python +/// function +#[test] +fn a_module_function_stands_where_the_whole_family_stood_down() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_refusedfamily_t"); + let _ = std::fs::remove_dir_all(&dir); + let built = match build_source( + REFUSED_FAMILY, + "by_diff_refusedfamily_t", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_refusedfamily_t as m\n\ + _leg = lambda f: 'native' if f.__code__.co_filename == '' else type(f).__name__\n\ + print(_leg(m.count), m.count([1, 2, 3]))\n\ + print('__firstlineno__' in vars(m.Base), '__firstlineno__' in vars(m.Rotating))\n\ + # the pair kept the definitions the module body built, so they still agree on\n\ + # what a `Rotating` is an instance of\n\ + print(m.Rotating.__base__ is m.Base, isinstance(m.Rotating('r', 2), m.Base))\n", + ); + assert_eq!( + out, + "native 6\n\ + True True\n\ + True True" + ); +} + /// a chain where every rung keeps fields of its own past the one below, which is the /// stdlib's commonest exception family — `configparser` writes ten of them /// @@ -13544,8 +14860,13 @@ fn a_base_this_module_emits_beside_one_it_does_not_agrees() { // outside — the base of ours in the list lays nothing out, so it asks for no room — // and python works out the mro and which of the bases owns the instance. // - // the outside base may own a real one: `dict`, `int` and `Exception` each decide the - // instance, and getting that wrong writes this class's idea of a layout over theirs + // the outside base may own a real one: `Exception` decides the instance, and getting + // that wrong writes this class's idea of a layout over its. + // + // `dict` and `int` are the same idea and are *not* here: an outside base whose layout + // the emitted class of ours cannot be mixed with leaves the mixture standing on the + // interpreted definition of that class, which is a wrong answer rather than a slow + // one — see `a_mixture_left_on_a_base_the_module_replaced_refuses_the_import` agree_python( "mixed_bases", "\ @@ -13574,19 +14895,9 @@ class OutsideFirst(codecs.StreamWriter, Ours): return \"outside\" -class AsDict(dict, Ours): - def label(self) -> str: - return \"dict\" - - -class AsInt(int, Ours): - def label(self) -> str: - return \"int\" - - -class OurError(Exception): - def which(self) -> str: - return \"ourerror\" +class OurError(Exception): + def which(self) -> str: + return \"ourerror\" class Diamond(OurError, ValueError): @@ -13620,16 +14931,13 @@ def exactly(which: int) -> str: "[c.__name__ for c in m.OursFirst.__mro__]", "[c.__name__ for c in m.OutsideFirst.__mro__]", "[c.__name__ for c in m.Diamond.__mro__]", - "(m.OursFirst.__base__.__name__, m.AsDict.__base__.__name__, m.AsInt.__base__.__name__)", + "m.OursFirst.__base__.__name__", "(m.OursFirst().side(), m.OursFirst().label())", "(m.OutsideFirst(None).side(), m.OutsideFirst(None).label())", "[m.through_the_base(o) for o in (m.Ours(), m.OursFirst(), m.OutsideFirst(None), m.Under())]", "[m.exactly(n) for n in (0, 1)]", "[m.resetting(o) for o in (m.Ours(), m.OursFirst(), m.OutsideFirst(None))]", "([c.__name__ for c in m.Under.__mro__], m.Under().label(), m.Under().side())", - // the outside base still owns the instance it always owned - "(sorted(m.AsDict(a=1, b=2).items()), m.AsDict().label())", - "(int(m.AsInt(7)) + 1, m.AsInt(7).label(), m.AsInt(7).side())", "(m.Diamond('boom').args, str(m.Diamond('boom')), m.Diamond('x').which(), m.Diamond('x').label())", "(isinstance(m.Diamond('x'), ValueError), isinstance(m.Diamond('x'), m.OurError))", // an instance of the mixture carries whatever `__dict__` the outside base @@ -13811,6 +15119,109 @@ def widened(value: object) -> str: ); } +/// the source the two tests below compile +/// +/// `Marked` carries a class keyword *and* a base this module emits, which is what +/// `_collections_abc.ByteString` is: `class ByteString(Sequence, metaclass=...)` over a +/// `Sequence` the same module writes. the keyword is what names the construction such a +/// class has — calling the metaclass on that very base — so it is no reason to turn the +/// base down. +/// +/// each class is then registered against at module level, the way a body tells an abstract +/// base which built-in types satisfy it. that registration lands on the interpreted +/// definition, because the fallback source runs the whole body before module init builds +/// anything, so the type replacing it has to be given the registry the twin collected +const KEYED_OVER_AN_EMITTED_BASE: &str = "\ +from abc import ABCMeta + + +class Root(metaclass=ABCMeta): + def kind(self) -> str: + return \"root\" + + +class Marked(Root, metaclass=ABCMeta): + def kind(self) -> str: + return \"marked\" + + +Root.register(list) +Marked.register(tuple) +"; + +#[test] +fn a_class_keyed_over_a_base_this_module_emits_agrees() { + agree_python( + "keyedbase", + KEYED_OVER_AN_EMITTED_BASE, + &[ + // the base is the emitted one, not a second class left standing under the + // name — which is the whole of what the construction had to get right + "m.Marked.__bases__ == (m.Root,)", + "m.Marked.__bases__[0] is m.Root", + "[c.__name__ for c in m.Marked.__mro__]", + // the keyword reached the metaclass, so the class is `ABCMeta`'s and not + // `type`'s — a spec-built one would report `type` here + "(type(m.Root).__name__, type(m.Marked).__name__)", + // the two directions of each question, so a leg answering `True` to + // everything is not mistaken for agreement + "(issubclass(m.Marked, m.Root), issubclass(m.Root, m.Marked))", + "(isinstance(m.Marked(), m.Root), isinstance(m.Root(), m.Marked))", + // what the module body registered, which only the twin was ever told + "(issubclass(list, m.Root), issubclass(list, m.Marked))", + "(issubclass(tuple, m.Marked), isinstance((1,), m.Marked))", + // registering against a subclass reaches the base, which takes `__subclasses__` + // over the emitted pair rather than over the twins + "issubclass(tuple, m.Root)", + "(isinstance([], m.Root), isinstance([], m.Marked))", + "(m.Root().kind(), m.Marked().kind())", + "(m.Root.__module__, m.Marked.__module__)", + "(m.Root.__name__, m.Marked.__qualname__)", + ], + ); +} + +#[test] +fn a_class_keyed_over_a_base_this_module_emits_is_the_compiled_type() { + // the agreement above is answered exactly the same way by a pair that fell back to + // their interpreted definitions, so it cannot say which build answered. a + // `method_descriptor` can: a compiled type holds one where an interpreted class holds + // a plain function + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_keyedbase_t"); + let _ = std::fs::remove_dir_all(&dir); + let built = match build_source( + KEYED_OVER_AN_EMITTED_BASE, + "by_diff_keyedbase_t", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_keyedbase_t as m\n\ + print(type(m.Root.__dict__['kind']).__name__,\n\ + \x20 type(m.Marked.__dict__['kind']).__name__)\n\ + print(m.Marked.__bases__[0] is m.Root)\n\ + print(issubclass(list, m.Root), issubclass(tuple, m.Marked))\n", + ); + assert_eq!(out, "method_descriptor method_descriptor\nTrue\nTrue True"); +} + /// a decorated method reaches the namespace the metaclass reads, not the type it built /// /// `ABCMeta` decides `__abstractmethods__` by walking the namespace it is handed, so a @@ -14842,6 +16253,297 @@ class Caught: ); } +#[test] +fn a_class_written_in_a_class_body_is_carried_off_the_interpreted_definition() { + // a `class` in a class body binds a name there like any other statement, and the + // interpreted definition built the class already — bases read, decorator applied, + // inner body run, all where python runs them. so the object it left behind is copied + // across the way every class-level constant is, and the outer class is compiled + // around it. + // + // `error` / `abort` is `imaplib.IMAP4`'s shape, where the second inner class stands + // on the first. `__Private` is python's own name mangling: the binding lands in the + // namespace as `_Holder__Private`, so the copy has to look for it under that name. + // and a `class` under a conditional is the same binding a `def` under one makes + agree_python( + "nestedclass", + "\ +on = True + + +class Holder: + class error(Exception): + pass + + class abort(error): + pass + + class __Private: + n = 4 + + if on: + class Conditional: + n = 5 + + def raised(self) -> str: + try: + raise Holder.abort('gone') + except Holder.error as failure: + return type(failure).__name__ + + def total(self) -> int: + return Holder.__Private.n + Holder.Conditional.n +", + &[ + "m.Holder().raised()", + "m.Holder().total()", + "(m.Holder.error.__qualname__, m.Holder.abort.__qualname__)", + "issubclass(m.Holder.abort, m.Holder.error)", + "m.Holder._Holder__Private.__qualname__", + "hasattr(m.Holder, '__Private')", + "type(m.Holder.error('boom')).__name__", + // an interpreted class can still be derived from, which is the whole reason + // the inner one keeps its own definition rather than being emitted + "type('Derived', (m.Holder.abort,), {})('x').args", + ], + ); +} + +#[test] +fn the_outer_class_is_compiled_and_the_class_written_in_it_is_not() { + // the two legs of the test above answer alike whichever definition stands, so this + // asks which one did: `method_descriptor` is the emitted method table, `function` is + // the interpreted leg. the outer class is the whole point — `imaplib.IMAP4` has 79 + // methods behind one nested `class` statement + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_nested_class_legs"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +def tag(cls: type) -> type: + cls.tagged = True + return cls + + +class Holder: + @tag + class Inner: + def v(self) -> int: + return 7 + + def make(self) -> int: + return Holder.Inner().v() +"; + let built = match build_source( + source, + "by_diff_nested_class_legs", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!( + !built + .declined + .iter() + .any(|declined| declined.name == "Holder"), + "the outer class declined: {:?}", + built.declined + ); + let out = run( + &python, + &dir, + "import by_diff_nested_class_legs as m\n\ + print(m.Holder().make(), m.Holder.Inner.tagged)\n\ + print(type(m.Holder.make).__name__, type(m.Holder.Inner.v).__name__)\n", + ); + assert_eq!( + out, + "7 True\n\ + method_descriptor function" + ); +} + +#[test] +fn a_base_a_class_written_in_a_class_body_stands_on_gives_up_its_emission() { + // such a class is never emitted: it is copied off the interpreted definition whole. so + // it stands on whatever its base name held while that body ran, which is the + // interpreted definition's class — and if the module emitted a type under that name, + // the copy carries a second, orphaned copy of it and `isinstance` answers `False` + // where python answers `True`, from ordinary code, with nothing reported. + // + // the base gives up its emission instead, exactly as it does for a module-level class + // this module does not emit, and both types are then the interpreted ones. the outer + // class still compiles, which is the whole point of lowering the nested `class`. + // + // `twice()` answers `6` either way, which is why this asserts on the *types*: a + // behavioural check on the method alone passes with the divergence fully present + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_nested_class_emitted_base"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Emitted: + def __init__(self, v: int) -> None: + self.v = v + + def read(self) -> int: + return self.v + + +class Based: + class Inner(Emitted): + def twice(self) -> int: + return self.read() * 2 + + def held(self) -> object: + return Based.Inner + + +class Computed: + class Inner(*[Emitted]): + pass +"; + let built = match build_source( + source, + "by_diff_nested_class_emitted_base", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + let mut declined: Vec<(&str, &str)> = built + .declined + .iter() + .map(|declined| (declined.name.as_str(), declined.reason.as_str())) + .collect(); + declined.sort_unstable(); + assert_eq!( + declined, + vec![ + // a base the header does not *name* is one no collected list can hold, so the + // class holding it is turned down instead + ( + "Computed", + "a base of `Inner` is worked out rather than named, so whether it is a class this module emits cannot be told", + ), + ( + "Emitted", + "`Based.Inner` is written in a class body, so it stands on the interpreted definition rather than this type", + ), + ] + ); + let out = run( + &python, + &dir, + "import by_diff_nested_class_emitted_base as m\n\ + print(m.Based.Inner.__bases__[0] is m.Emitted, m.Computed.Inner.__bases__[0] is m.Emitted)\n\ + print(isinstance(m.Based.Inner(3), m.Emitted), m.Based().held() is m.Based.Inner)\n\ + print(m.Based.Inner(3).twice(), type(m.Based.held).__name__)\n", + ); + // `Based` is still compiled — `held` answers from the method table — while the base it + // holds a class over, and that class, are both the interpreted definitions + assert_eq!( + out, + "True True\n\ + True True\n\ + 6 method_descriptor" + ); +} + +#[test] +fn the_shapes_a_class_written_in_a_class_body_is_not_lowered_for_decline() { + // the two the copy cannot answer for. a dunder is settled from the body text — a + // type slot, an instance layout, what the class publishes — while the copy only + // knows what the name holds once the interpreter has run the body. and a `def` + // beside a `class` of the same name is two definitions of one attribute: the `def` + // would go into the method table while the copy carried whatever python kept + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_nested_class_declines"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Dunder: + n = 0 + + class __repr__: + pass + + +class Twice: + def load(self, n: int) -> int: + return n + + class load: + pass +"; + let built = match build_source( + source, + "by_diff_nested_class_declines", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + let mut declined: Vec<(&str, &str)> = built + .declined + .iter() + .map(|declined| (declined.name.as_str(), declined.reason.as_str())) + .filter(|(name, _)| matches!(*name, "Twice" | "Dunder")) + .collect(); + declined.sort_unstable(); + assert_eq!( + declined, + vec![ + ( + "Dunder", + "`__repr__` is written as a class in the class body, and a dunder is settled before one runs", + ), + ( + "Twice", + "`load` is both defined by this class body and written as a class in it", + ), + ] + ); + let out = run( + &python, + &dir, + "import by_diff_nested_class_declines as m\n\ + print(m.Dunder.n, type(m.Dunder.__repr__).__name__, type(m.Twice.load).__name__)\n", + ); + assert_eq!(out, "0 type type"); +} + #[test] fn a_slots_declaration_reaches_the_metaclass_rather_than_the_finished_type() { // `__slots__` is the constant that proves the namespace is where these have to go. @@ -14972,16 +16674,15 @@ class Below(metaclass=ABCMeta): #[test] fn a_class_the_module_pops_out_of_its_own_globals_stays_off_the_compiled_surface() { - // `ast` builds `Num` and then pops the name straight out of its own globals. that is - // a `del` whose target this cannot read — the name comes off a comprehension there — - // so every definition the module writes is treated as one the pop could have taken. - // installing a compiled `Gone` over a name the body removed would put a class on the - // surface python does not have there, and the interpreted definition the construction - // would otherwise fall back to is not there to be found either. + // `ast` builds `Num` and then pops the name straight out of its own globals, through + // a dict comprehension over a tuple of string literals. installing a compiled `Gone` + // over a name the body removed would put a class on the surface python does not have + // there, and the interpreted definition the construction would otherwise fall back to + // is not there to be found either. // - // the class-level-constant gate used to carry this, and this is what stayed behind - // when it went. `Kept` is no longer a boundary — the rule reaches the whole module, - // which is what its second decline says + // the tuple is what says which names went, so `Kept` is not one of them and compiles. + // the last line is the one that says so: a `method_descriptor` is the compiled leg + // answering, a `function` the interpreted one let Some((python, toolchain)) = environment() else { return; }; @@ -15029,16 +16730,10 @@ HIDDEN = {name: globals().pop(name) for name in (\"Gone\",)} .collect(); assert_eq!( declined, - vec![ - ( - "Gone", - "`Gone` is rebound at module level, so installing this over it would replace what the rebind produced" - ), - ( - "Kept", - "`Kept` is rebound at module level, so installing this over it would replace what the rebind produced" - ) - ] + vec![( + "Gone", + "`Gone` is rebound at module level, so installing this over it would replace what the rebind produced" + )] ); let out = run( &python, @@ -15052,20 +16747,90 @@ HIDDEN = {name: globals().pop(name) for name in (\"Gone\",)} out, "False True\n\ 1 gone kept\n\ - function function" + function method_descriptor" ); } #[test] -fn an_annotated_class_attribute_reaches_the_compiled_type() { - // the statement was skipped in both the layout pass and the constant pass, so an - // annotated class attribute was lost outright: `Tagged.KIND` raised where python - // answers `'tagged'`. it is the same binding a plain assignment makes — the - // annotation only adds an entry to `__annotations__` - // - // the other four are the other constructions, because the attribute was lost on all - // of them: `Held` owns a layout and is readied in place, `Root` is a base and comes - // from a type spec, `Leaf` is built on an in-module base and `OnExternal` on one from +fn a_class_the_module_pops_out_under_a_computed_name_takes_the_whole_module_with_it() { + // the other half of the rule above. a key worked out at runtime names nothing that + // can be read where the module is compiled, so every definition the body wrote is + // treated as the one that went — including `Kept`, which the pop never touches. + // both legs answer alike either way, and `function` twice is what says both fell + // back to the interpreted definitions + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_computedpop"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Gone: + def label(self) -> str: + return \"gone\" + + +class Kept: + def label(self) -> str: + return \"kept\" + + +def pick() -> str: + return \"Gone\" + + +HIDDEN = globals().pop(pick()) +"; + let built = match build_source( + source, + "by_diff_computedpop", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + let declined: Vec<&str> = built + .declined + .iter() + .map(|declined| declined.name.as_str()) + .collect(); + // `pick` too: the rule reaches every definition the module wrote, and a `def` is + // one of them + assert_eq!(declined, vec!["Gone", "Kept", "pick"]); + let out = run( + &python, + &dir, + "import by_diff_computedpop as m\n\ + print('Gone' in m.__dict__, 'Kept' in m.__dict__)\n\ + print(m.HIDDEN().label(), m.Kept().label())\n\ + print(type(m.HIDDEN.label).__name__, type(m.Kept.label).__name__)\n", + ); + assert_eq!( + out, + "False True\n\ + gone kept\n\ + function function" + ); +} + +#[test] +fn an_annotated_class_attribute_reaches_the_compiled_type() { + // the statement was skipped in both the layout pass and the constant pass, so an + // annotated class attribute was lost outright: `Tagged.KIND` raised where python + // answers `'tagged'`. it is the same binding a plain assignment makes — the + // annotation only adds an entry to `__annotations__` + // + // the other four are the other constructions, because the attribute was lost on all + // of them: `Held` owns a layout and is readied in place, `Root` is a base and comes + // from a type spec, `Leaf` is built on an in-module base and `OnExternal` on one from // outside. `method_descriptor` against `function` is what says the *compiled* type // answered — a class that fell back to its interpreted definition would agree on // every value here and say `function` @@ -15393,14 +17158,16 @@ Holder.wrapper = Holder(leaf) /// loud failure it already gave. #[test] fn an_instance_the_layout_cannot_hold_is_left_where_the_body_built_it() { - // three refusals, one for each thing the layout has no room for. + // two refusals, one for each thing the layout has no answer for. + // + // `bare` was built through `__new__` and never ran `__init__`, so the field the layout + // treats as always defined was never written and there is nothing to move onto it. + // `raised` is an instance of a class standing on a base python allocates, and whatever + // `Exception` keeps for it lives in a part of the object nothing here can read back. // - // `spare` carries an attribute nothing declared, so the emitted instance would answer - // the layout's fields and quietly lose `extra`. `bare` was built through `__new__` and - // never ran `__init__`, so the field the layout treats as always defined was never - // written and there is nothing to move onto it. `raised` is an instance of a class - // standing on a base python allocates, and whatever `Exception` keeps for it lives in - // a part of the object nothing here can read back. + // an attribute nothing declared is *not* among these — the emitted class keeps a dict + // beside its layout, and the name goes there; see + // `an_instance_carrying_a_name_the_layout_never_had_moves_with_it`. // // both classes still compile — the refusal is about one value, not about the class — // and `wrapper_descriptor`/`method_descriptor` is what says so @@ -15420,10 +17187,6 @@ class Tagged(Exception): return \"tagged\" -spare = Loose() -spare.extra = 2 -Loose.spare = spare - bare = Loose.__new__(Loose) Loose.bare = bare @@ -15452,59 +17215,65 @@ Tagged.raised = raised &python, &dir, "import by_diff_twinunmoved as m\n\ - print(hasattr(m.Loose, 'spare'), hasattr(m.Loose, 'bare'),\n\ - \x20 hasattr(m.Tagged, 'raised'))\n\ - print(type(m.spare) is m.Loose, type(m.bare) is m.Loose,\n\ - \x20 type(m.raised) is m.Tagged)\n\ + print(hasattr(m.Loose, 'bare'), hasattr(m.Tagged, 'raised'))\n\ + print(type(m.bare) is m.Loose, type(m.raised) is m.Tagged)\n\ print(type(m.Loose.__init__).__name__, type(m.Tagged.tag).__name__)\n", ); assert_eq!( out, - "False False False\n\ - False False False\n\ + "False False\n\ + False False\n\ wrapper_descriptor method_descriptor" ); } -/// a frozen class needs no rule of its own, because its own setter is the rule +/// an instance the move left behind is still a value the program holds, and compiled code +/// reading the name it is under has to answer for it /// -/// the move writes each field through the type's setter, so that a value takes the same -/// conversion an assignment from python would. a frozen class publishes none, and that is -/// the whole answer for it: one with fields refuses at the first of them, and one with no -/// fields has nothing to lose and moves. the alternative — turning every immutable class -/// down up front — would cost the second case for nothing. +/// the test above says such a value stays what the module body built. this says what +/// compiled code may then assume about it, which is nothing: `Form` extends a class python +/// allocates, so there is no field table to move `marker` through and the name goes on +/// holding an instance of the interpreted definition for the life of the module. narrowing +/// the global read to the emitted representation refused it — and refused it under a name +/// that prints the same on both sides, because the interpreted definition and the emitted +/// type share one +/// +/// `typing` is where this was found, and it broke the whole construct: `Annotated` is such +/// a value, `_get_typeddict_qualifiers` compares `annotation_origin is Annotated`, and so +/// every `TypedDict('T', {...})` against a compiled standard library raised +/// `expected _TypedCacheSpecialForm, got _TypedCacheSpecialForm` #[test] -fn a_frozen_instance_moves_only_where_it_has_no_field_to_fill() { - // `Fixed.origin` is absent because `n` has no setter to write it through, and absent is - // the right answer: an emitted instance with `n` unwritten would answer `0`, which is - // the quiet wrong answer the whole move is arranged to avoid. `Blank.nothing` moves, - // because a frozen class with no fields is entirely its type +fn a_global_holding_an_instance_the_move_left_behind_is_read_as_an_object() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_twinfrozen"); + let dir = diff_root().join("by_diff_globaltwin"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -frozen data class Fixed: - n: int +class Form(Exception): + def label(self) -> str: + return \"form\" -frozen data class Blank: - pass +marker = Form(\"m\") -origin = Fixed(0) -Fixed.origin = origin +def is_marker(value: object) -> bool: + return value is marker -nothing = Blank() -Blank.nothing = nothing + +def marker_label() -> str: + return marker.label() "; let built = match build_source( source, - "by_diff_twinfrozen", + "by_diff_globaltwin", &toolchain, &dir, - &Options::default(), + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, ) { Ok(built) => built, Err(error) => { @@ -15514,85 +17283,82 @@ Blank.nothing = nothing } }; assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + // `method_descriptor` is what says the compiled leg answered: both functions run + // natively, and both reach the global the emitted type never took over let out = run( &python, &dir, - "import by_diff_twinfrozen as m\n\ - print(hasattr(m.Fixed, 'origin'), type(m.origin) is m.Fixed)\n\ - print(type(m.Blank.nothing) is m.Blank, m.nothing is m.Blank.nothing)\n", + "import by_diff_globaltwin as m\n\ + print(type(m.marker) is m.Form)\n\ + print(m.is_marker(m.marker), m.is_marker(m.Form('other')))\n\ + print(m.marker_label())\n\ + print(type(m.Form.label).__name__)\n", ); assert_eq!( out, - "False False\n\ - True True" + "False\n\ + True False\n\ + form\n\ + method_descriptor" ); } -/// a module-level *function* has an interpreted twin too, and it is deliberately left -/// where it stands -/// -/// the same staleness a class has: the module body runs against the interpreted -/// definitions, so everything it captured holds the `def`'s own function object, while -/// `PyModule_AddFunctions` puts the compiled `PyCFunction` under the name at the end of -/// init. `ALIAS is fn` is then False where python says True. -/// -/// the class fix does not transfer, and this pins that it has not been made to. a class -/// twin is *incompatible* with the type that replaced it — `isinstance` denies it and a -/// compiled method refuses its instances — so a reference still holding one is already -/// broken, and moving it repairs damage. a function twin is **interchangeable** with the -/// compiled function for every use except identity: it computes the same answer, and -/// nothing rejects it. so moving one repairs nothing that was broken and breaks two -/// things that were not: +/// the source both halves of the extra-attribute move use /// -/// * a `function` in a class dict binds `self` and a `PyCFunction` does not, so the -/// moved reference stops being a method. `optparse` writes `class Option: __repr__ = -/// _repr` and `multiprocessing.reduction` writes `class AbstractReducer: dump = dump` -/// * `inspect.signature` works on a `function` and raises `ValueError` on a -/// `PyCFunction`, so a captured callback stops being introspectable +/// `bump` is here for the reason [`A_CLASS_WITH_A_FIELD`] has it: without a method that +/// answers differently on the two legs, a test about a move passes just as well with the +/// class never compiled at all +const AN_INSTANCE_GIVEN_A_NAME_ITS_CLASS_NEVER_MENTIONED: &str = "\ +class Holder: + def __init__(self, tag: str) -> None: + self.tag = tag + + def shout(self) -> str: + return self.tag.upper() + + +class Other: + def label(self) -> str: + return \"other\" + + +held = Holder(\"one\") +held.extra = 7 +held.owner = Other + +alias = held +Holder.standing = held +"; + +/// an instance carrying a name its class never mentioned still moves onto the emitted type /// -/// both turn a *right* answer into a raise, and a wrong answer is better than a crash. -/// over the stdlib corpus the captured references are overwhelmingly dispatch tables — -/// `copy._deepcopy_dispatch`, `shutil._ARCHIVE_FORMATS`, `xml.etree.ElementTree._serialize` -/// — whose entries are only ever *called*, so the divergence is unobservable there while -/// the repair would be plainly observable. +/// the move writes each of the layout's fields onto a fresh instance, and a name the +/// layout has none of used to refuse the whole move — so `held` went on standing on the +/// interpreted definition and `isinstance(held, Holder)` was False where python says True. +/// that is the silent wrong answer the move exists to prevent, and it was reachable from +/// two lines of ordinary python. /// -/// the two questions a remap would also have had to answer turn out to be answered -/// already, and neither needs runtime machinery: a function whose module-level name the -/// body rebinds is not `exported`, so an accelerator import (`asyncio.events` keeping -/// `_py_get_event_loop`, `operator`'s trailing `from _operator import *`) never produces -/// a twin at all; and a *decorated* module-level definition the module reads already -/// declines, because its decorator cannot run where the `def` stands and again over the -/// compiled one -#[test] -fn a_class_attribute_naming_a_module_function_keeps_the_definition_that_binds() { - // `multiprocessing.reduction` writes `class AbstractReducer: dump = dump`, and this is - // that: a class body binding a name to a module-level function that goes on to - // compile. the value the emitted type carries is the *twin*, and it has to be — a - // `PyCFunction` in a class dict is not a descriptor, so `Reducer().dump()` would call - // `_dump` with no `self` at all. +/// an emitted class keeps a dict beside its layout wherever the source did not declare +/// `__slots__` throughout, and that dict is where `o.brand_new = 7` on a freshly built +/// emitted instance already goes. so the name has somewhere to be written after all, and +/// the refusal now applies only to a class whose instances have no dict at all. +#[test] +fn an_instance_carrying_a_name_the_layout_never_had_moves_with_it() { + // `extra` is a plain value and `owner` is one of this module's own classes, which has + // to come across as the type that replaced it — an extra reaches a twin exactly as a + // field does. `alias` and `Holder.standing` are the other two holders of the same + // object, and they have to answer it too. // - // `function` against the forwarder is the whole assertion. the class answers the - // same *value* either way, so nothing but what sits in the slot says which - // definition is standing there + // `vars` is asked for as a list of names rather than as a mapping, because the values + // include a class and the two legs spell a class's repr differently let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_fntwinbind"); + let dir = diff_root().join("by_diff_twinextra"); let _ = std::fs::remove_dir_all(&dir); - let source = "\ -def _dump(this: object) -> str: - return \"dumped\" - - -class Reducer: - dump = _dump - - def kind(self) -> str: - return \"reducer\" -"; let built = match build_source( - source, - "by_diff_fntwinbind", + AN_INSTANCE_GIVEN_A_NAME_ITS_CLASS_NEVER_MENTIONED, + "by_diff_twinextra", &toolchain, &dir, &Options { @@ -15608,66 +17374,145 @@ class Reducer: } }; assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + // the compiled leg answered with the emitted type, which is what makes the agreement + // below mean anything: a class that fell back would hold the very object the body + // wrote and agree for the one reason that makes the comparison worthless let out = run( &python, &dir, - "import by_diff_fntwinbind as m\n\ - _leg = lambda f: 'native' if f.__code__.co_filename == '' else type(f).__name__\n\ - print(m.Reducer().dump(), m.Reducer().kind())\n\ - print(_leg(m._dump), type(m.Reducer.__dict__['dump']).__name__)\n\ - print(type(m.Reducer.kind).__name__)\n", + "import by_diff_twinextra as m\n\ + print(type(m.Holder.shout).__name__, type(m.held) is m.Holder)\n", ); - // `method_descriptor` says the emitted type answered rather than a class that fell - // back to its interpreted definition — which would carry the slot for the other reason - assert_eq!( - out, - "dumped reducer\n\ - native function\n\ - method_descriptor" + assert_eq!(out, "method_descriptor True"); + agree_python( + "twinextra2", + AN_INSTANCE_GIVEN_A_NAME_ITS_CLASS_NEVER_MENTIONED, + &[ + "isinstance(m.held, m.Holder)", + "type(m.held) is m.Holder", + "(m.held.tag, m.held.extra, m.held.shout())", + "m.held.owner is m.Other", + "list(vars(m.held))", + "vars(m.held)['extra']", + "(m.alias is m.held, m.Holder.standing is m.held)", + "isinstance(m.Holder.standing, m.Holder)", + ], ); } -/// the same, for the slot a *declined* class keeps and for a dunder +/// the standard library's own spelling of it: a class of constants, blanked one name at a +/// time on an instance the module body built /// -/// `optparse` writes `class Option: __repr__ = _repr`, and over the corpus that is a -/// captured twin — the compiled `optparse` keeps `Option` interpreted and its `__repr__` -/// holds the definition the module's own `_repr` no longer names. the alias remap walks -/// exactly that dict, so it is the second route a function substitution would take into a -/// descriptor position, and `repr()` is where it would show: python looks a dunder up on -/// the type and calls what it finds, and what it finds has to bind +/// `_colorize` writes this, and it is what the refusal cost in practice — every attribute +/// on `NoColors` is a name the class never assigned in a method, so the whole instance was +/// turned down and `isinstance(NoColors, ANSIColors)` answered False. +/// +/// the class has no fields at all, which is the shape the extras are the *whole* of: there +/// is nothing in the layout to write, and the move is entirely the dict beside it. #[test] -fn a_declined_class_keeps_the_dunder_slot_a_module_function_filled() { +fn an_instance_blanked_by_a_setattr_loop_still_answers_its_own_class() { + agree_python( + "twinblanked", + "\ +class Palette: + RED = \"r\" + GREEN = \"g\" + + +blank = Palette() + +for name in dir(blank): + if not name.startswith(\"__\"): + setattr(blank, name, \"\") +", + &[ + "(isinstance(m.blank, m.Palette), type(m.blank) is m.Palette)", + "(m.blank.RED, m.blank.GREEN)", + "(m.Palette().RED, m.Palette().GREEN)", + "sorted(vars(m.blank))", + ], + ); +} + +/// a class body's own constant is the earliest a move can be asked for, and it is in time +/// +/// the constants are copied while their class is being built, which is before the pass +/// that walks the module namespace — so `class Bag: first = Seed("body")` reaches the move +/// earlier than a `Bag.second = later` written after the statement does. what the move +/// needs by then is the emitted type of `Seed`, and the types are filled one class at a +/// time as each is built. +/// +/// it is in time because a class body can only name a class already defined above it: the +/// body runs where the `class` statement stands, so a constant naming an instance of a +/// class further down would have raised `NameError` at import in the first place. this +/// pins the tightest spelling of that — the class immediately above, read in the very +/// first statement of the next class's body. +#[test] +fn a_class_body_constant_holding_an_instance_moves_with_the_class_above_it() { + agree_python( + "twinbodyorder", + "\ +class Seed: + def __init__(self, tag: str) -> None: + self.tag = tag + + +class Bag: + first = Seed(\"body\") + + +later = Seed(\"gift\") +Bag.second = later +", + &[ + "(isinstance(m.Bag.first, m.Seed), type(m.Bag.first) is m.Seed)", + "m.Bag.first.tag", + "(isinstance(m.Bag.second, m.Seed), type(m.Bag.second) is m.Seed)", + "(m.Bag.second is m.later, m.Bag.second.tag)", + "vars(m.Bag.first)", + ], + ); +} + +/// a frozen class needs no rule of its own, because its own setter is the rule +/// +/// the move writes each field through the type's setter, so that a value takes the same +/// conversion an assignment from python would. a frozen class publishes none, and that is +/// the whole answer for it: one with fields refuses at the first of them, and one with no +/// fields has nothing to lose and moves. the alternative — turning every immutable class +/// down up front — would cost the second case for nothing. +#[test] +fn a_frozen_instance_moves_only_where_it_has_no_field_to_fill() { + // `Fixed.origin` is absent because `n` has no setter to write it through, and absent is + // the right answer: an emitted instance with `n` unwritten would answer `0`, which is + // the quiet wrong answer the whole move is arranged to avoid. `Blank.nothing` moves, + // because a frozen class with no fields is entirely its type let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_fntwindunder"); + let dir = diff_root().join("by_diff_twinfrozen"); let _ = std::fs::remove_dir_all(&dir); - // the late gift is the decline lever, and a *dunder* one is what turns the class down - // rather than having its attributes adopted — see the twin-shapes test above. it is - // here to put `Option` on the interpreted leg, which is where `optparse` has it let source = "\ -def _repr(this: object) -> str: - return \"