From 60375dd8c437ffff9be1324211427b64a7d1b7e9 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Sat, 5 Sep 2026 13:47:41 +1000 Subject: [PATCH 1/6] retain a reference at the width its release reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit python 3.12 split ob_refcnt so an immortal object is recognised from the sign of its low half, and Py_INCREF has written only that low half ever since while Py_DECREF tests and decrements the whole word. a 32-bit store followed by a 64-bit load of one address cannot be served from the store buffer, so every retain an emitted module makes stalls the release after it. one small module carried 86 of these. By_IncRef/By_XIncRef keep the immortality check — asking _Py_IsImmortal, the predicate Py_DECREF itself uses, so both halves of the pair compile to the same test — and do the increment at 64 bits. it is cpython's own 32-bit-host arm of Py_INCREF, taken on a 64-bit host because the split it avoids is what costs us. it skips the increment on a superset of what the split form skips, and every object in the difference is one Py_DECREF also leaves alone, so no reference is dropped early and no immortal sentinel is written. the fast path is opt-in behind a positive test of every name it uses: 3.12 and 3.13 only, GIL-enabled, 64-bit, no REF_DEBUG/TRACE_REFS/STATS/LIMITED_API, and _Py_IsImmortal actually defined. checked against 3.11 through 3.15, free-threaded builds included. inc_ref's Primitive and Instance arms are where every emitted retain of a PyObject * comes from, so the codegen takes the same store. on one small module the whole-artefact count of 32-bit refcount stores goes 86 -> 13 and the 64-bit ones 3 -> 78; what is left is by.h's operational helpers. --- crates/by_codegen_c/src/lib.rs | 16 +++--- crates/by_rt/include/by.h | 89 +++++++++++++++++++++++++++++++--- 2 files changed, 92 insertions(+), 13 deletions(-) diff --git a/crates/by_codegen_c/src/lib.rs b/crates/by_codegen_c/src/lib.rs index ba306bf690..38a751090f 100644 --- a/crates/by_codegen_c/src/lib.rs +++ b/crates/by_codegen_c/src/lib.rs @@ -4432,7 +4432,9 @@ fn inc_ref(ty: &RType, expr: &str) -> Option { .collect::>() .join(" "), RType::Array(_) => format!("By_ArrayIncRef((ByArrayHeader *){expr});"), - RType::Primitive(_) | RType::Instance { .. } => format!("Py_XINCREF({expr});"), + // `By_XIncRef` rather than `Py_XINCREF`: same immortality check, but the + // store is the width the matching release reads — see the comment on it + RType::Primitive(_) | RType::Instance { .. } => format!("By_XIncRef({expr});"), }) } @@ -9164,7 +9166,7 @@ mod tests { .and_then(|rest| rest.split("static PyObject *byw").next()) .expect("the body is emitted"); // the frame never owned it, so there is no reference of its own to hand on - assert!(body.contains("Py_XINCREF(by_ret);"), "{body}"); + assert!(body.contains("By_XIncRef(by_ret);"), "{body}"); } #[test] @@ -9177,7 +9179,7 @@ mod tests { // claim nothing is owned at the exit: there is no release to cancel against module.functions[0].blocks[0].owned_at_exit = Some(Vec::new()); let c = emit_module(&module); - assert!(c.contains("Py_XINCREF(by_ret);"), "{c}"); + assert!(c.contains("By_XIncRef(by_ret);"), "{c}"); } #[test] @@ -9925,10 +9927,10 @@ mod tests { let text = emit_function(&ModuleIr::new("app"), &function); assert!(text.contains("r1 = r0->by_f_inner;"), "{text}"); // the intermediate is neither retained nor released - assert!(!text.contains("Py_XINCREF(r0->by_f_inner)"), "{text}"); + assert!(!text.contains("By_XIncRef(r0->by_f_inner)"), "{text}"); assert!(!text.contains("Py_XDECREF(r1)"), "{text}"); // the value that leaves still is - assert!(text.contains("Py_XINCREF(r1->by_f_label)"), "{text}"); + assert!(text.contains("By_XIncRef(r1->by_f_label)"), "{text}"); } #[test] @@ -9950,7 +9952,7 @@ mod tests { let text = emit_function(&ModuleIr::new("app"), &function); assert!(text.contains(" r1 = r0;\n"), "{text}"); - assert!(!text.contains("Py_XINCREF"), "{text}"); + assert!(!text.contains("By_XIncRef"), "{text}"); // and the frame does not give back what it never took, on either way out assert!(!text.contains("Py_XDECREF(r1)"), "{text}"); } @@ -9985,7 +9987,7 @@ mod tests { let text = emit_function(&ModuleIr::new("app"), &function); assert!(text.contains(" r2 = r1.f0;\n"), "{text}"); - assert!(!text.contains("Py_XINCREF(r1.f0)"), "{text}"); + assert!(!text.contains("By_XIncRef(r1.f0)"), "{text}"); assert!(!text.contains("Py_XDECREF(r2)"), "{text}"); // the tuple itself still owns what it holds, on either way out assert!(text.contains("Py_XDECREF(r1.f0)"), "{text}"); diff --git a/crates/by_rt/include/by.h b/crates/by_rt/include/by.h index 739cbd001c..e075f83d3e 100644 --- a/crates/by_rt/include/by.h +++ b/crates/by_rt/include/by.h @@ -148,6 +148,83 @@ typedef size_t ByTagged; #define BY_COLD static #endif +/* ── retaining at the width the release reads ───────────────────────────────── + * + * python 3.12 split `ob_refcnt` in two so an immortal object could be recognised + * from the sign of its low half, and the two halves of a retain/release pair have + * disagreed on access width ever since. `Py_INCREF` reads and writes + * `ob_refcnt_split[PY_BIG_ENDIAN]`, a 32-bit field; `Py_DECREF` tests immortality + * over the full `Py_ssize_t` and then decrements it. a narrow store followed by a + * wider load of the same address cannot be served from the store buffer, so every + * retain a compiled function makes stalls the release after it until the store has + * reached L1. emitted modules are little else *but* retain/release pairs — one + * small module carried 82 of these 32-bit stores — so it is paid everywhere. + * + * so retain at 64 bits and keep the immortality check, asking for it with + * `_Py_IsImmortal`, which is the predicate `Py_DECREF` itself uses. that is the + * point: both halves of the pair now test the same bits the same way, and the two + * compile to the same instruction. this is not an invention but cpython's own + * `#else` arm of `Py_INCREF` — the one a 32-bit host takes — selected on a 64-bit + * host because the split it exists to avoid is exactly what costs us. + * + * that it is safe rests on one property rather than on the arithmetic matching: + * this skips the increment on a *superset* of the objects the split form skips, + * and every object in the difference is one `Py_DECREF` also declines to touch. so + * a reference can never be dropped that would otherwise have been held, and an + * immortal sentinel is never written. (an object past 2^31 live references is + * leaked instead of counted — but it already is upstream, by the same sign test.) + * + * mypyc goes further and drops the check outright, as `op->ob_refcnt++`. that is + * only sound for objects it has *proved* mortal, because a 64-bit increment of an + * immortal refcount corrupts the sentinel, and we have no such proof. + * + * the fast path is opt-in behind a positive test of every name it uses. a spelling + * missing in some configuration must not fail to compile *every* module, so + * anything unrecognised falls back to the ordinary macros, which are always right: + * + * - `Py_GIL_DISABLED` — a free-threaded build counts in `ob_ref_local` and + * `ob_ref_shared`, a different layout with no split to match + * - `Py_REF_DEBUG`, `Py_TRACE_REFS`, `Py_STATS` — the macros keep books here + * - `Py_LIMITED_API` — there `Py_INCREF` is a call, by design + * - `SIZEOF_VOID_P > 4` — a 32-bit host has no split, so no mismatch + * - 3.12 and 3.13 alone — 3.11 predates the split, and 3.14 removed it again: + * its `Py_INCREF` already stores the whole `ob_refcnt`, so there is nothing + * left to match and taking this path would only risk a layout we did not read + * - `_Py_IsImmortal`, which is not stable api and moved header between versions + * + * where the fast path is declined these are exactly `Py_INCREF`/`Py_XINCREF`, so no + * caller has to know which it got. the release side needs no counterpart: it reads + * and writes the full width already, which is the half of the pair that was right. + * + * defining `BY_NO_WIDE_INCREF` forces the fallback. it is the escape hatch for a + * build that meets something none of the tests above anticipated, and it is also + * how the two legs of the disassembly check are taken from one emitted module */ +#if !defined(BY_NO_WIDE_INCREF) \ + && PY_VERSION_HEX >= 0x030C0000 && PY_VERSION_HEX < 0x030E0000 \ + && defined(_Py_IsImmortal) \ + && !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) \ + && !defined(Py_REF_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_STATS) \ + && defined(SIZEOF_VOID_P) && SIZEOF_VOID_P > 4 +#define BY_WIDE_INCREF 1 +#endif + +BY_HOT void By_IncRefObject(PyObject *o) { +#ifdef BY_WIDE_INCREF + if (!_Py_IsImmortal(o)) o->ob_refcnt++; +#else + Py_INCREF(o); +#endif +} + +BY_HOT void By_XIncRefObject(PyObject *o) { + if (o != NULL) By_IncRefObject(o); +} + +/* an emitted register holding an instance is typed as that class's own struct, so + * these take the cast `Py_XINCREF` takes for the same reason */ +#define By_IncRef(op) By_IncRefObject((PyObject *)(op)) +#define By_XIncRef(op) By_XIncRefObject((PyObject *)(op)) + BY_HOT int By_IsShort(ByTagged x) { return (x & BY_INT_TAG) == 0; } BY_HOT Py_ssize_t By_ShortValue(ByTagged x) { return ((Py_ssize_t)x) >> 1; } @@ -179,7 +256,7 @@ static inline PyObject *By_BoxInt(ByTagged x) { return PyLong_FromSsize_t(By_ShortValue(x)); } PyObject *o = By_LongOf(x); - Py_INCREF(o); + By_IncRef(o); return o; } @@ -214,7 +291,7 @@ static inline ByTagged By_TaggedFromLong(PyObject *o) { return By_ShortFrom((Py_ssize_t)value); } PyErr_Clear(); - Py_INCREF(o); + By_IncRef(o); return ((ByTagged)(void *)o) | BY_INT_TAG; } } @@ -235,7 +312,7 @@ BY_HOT void By_DecRefTagged(ByTagged x) { BY_HOT void By_IncRefTagged(ByTagged x) { if (BY_UNLIKELY(!By_IsShort(x))) { - Py_XINCREF(By_LongOf(x)); + By_XIncRef(By_LongOf(x)); } } @@ -695,12 +772,12 @@ static inline PyObject *By_BoxFloat(double v) { return PyFloat_FromDouble(v); } static inline PyObject *By_BoxBool(char v) { PyObject *o = v ? Py_True : Py_False; - Py_INCREF(o); + By_IncRef(o); return o; } static inline PyObject *By_BoxNone(void) { - Py_INCREF(Py_None); + By_IncRef(Py_None); return Py_None; } @@ -835,7 +912,7 @@ static inline char By_UnboxNone(PyObject *o) { /* widen a known-class object to `object`: the pointer is unchanged, but the * destination register owns what it holds, so it needs its own reference */ static inline PyObject *By_NewRef(PyObject *o) { - Py_XINCREF(o); + By_XIncRef(o); return o; } From 0aac8ef38032a84ee02c047af9ddb094a1864954 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Sat, 5 Sep 2026 12:10:57 +1000 Subject: [PATCH 2/6] correct what a compiled module answers about its own definitions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit four families of silent wrong answer, all of them a compiled module disagreeing with the interpreted twin it was built from: about a class written in a class body, about the names its own namespace holds, about what a decorator, a calling frame or a nested scope leaves behind, and about which of its classes install. ## carry a class written in a class body off its interpreted definition three things a class statement nested in another class body got wrong. the outer class now compiles while the inner one keeps its interpreted definition, copied into the type dict as a class-level constant. a base such a class stands on gives up its emission. the nested class is never emitted, so it stands on whatever its base name held while the interpreted definition's body ran; an emitted type under that name left the copy on a second, orphaned copy of it, and isinstance answered False where python answers True. that is what a module-level class this module does not emit already does. and a module whose class bodies are captured ran its interpreted twin against a *copy* of the builtins mapping, so that `__build_class__` reached that body and nothing else. python gives a function the builtins its module dict held when the function was made, so every function and method the body defined kept that copy for life — and those are exactly the definitions a declined function runs from. the copy was a snapshot frozen at import: a builtin rebound afterwards still answered with the old one, a deleted one still answered, an added one was never found, and a body writing through `__builtins__` wrote where nobody would read, while the compiled half of the same module saw all four. the hook now displaces `__build_class__` in the real mapping, calls the entry it displaced, records only where the `class` statement was written in this module's own globals, and puts the entry back only if it is still the one standing. the globals filter closes a second silent wrong answer: a `class` exec'd into another namespace during a capturing body was recorded as if it were this module's, and the emitted class then carried the foreign body's constants. ## answer for what a module's own namespace holds a module-level definition is bound under many names — `ALIAS = fn`, a class body's `direct = fn`, a container the body built, a default argument. every one was bound to the *interpreted* definition while the name the definition came from answered the forwarder the module publishes. the two compute the same thing and are not the same object, so only an `is` sees it: a dispatch table keyed on a function, an unregister, a set of callbacks all silently took the wrong branch. five shapes now agree with python. this reverses a decision the tests recorded — that comment called the divergence tolerable because it "never reaches an answer", which is exactly the shape of a silent wrong answer, so the test is replaced. a decorated definition is left out: the twin's source has its decorators taken out of it, so pairing it with the undecorated forwarder swaps one wrong answer for another. the guard on a class the module body writes on an emitted base gave a reason that is not the one — that a type of ours refuses to be a base. it does not. the reason is the order module init runs in: the whole interpreted source runs first and the emitted types are installed over the names afterwards, so such a class is already built on the interpreted definition of its base and is left on an orphaned copy. relaxing the guard on the wrong reason takes webbrowser from 1 compiled unit to 21 and makes isinstance(MacOSXOSAScript("default"), BaseBrowser) False. the rule that a module rebinding a name must not have a compiled definition installed over it was blanket for a removal through `globals()`: `del globals()[k]`, `pop`, `popitem` and `clear` set one flag with no name attached, and every definition the module wrote was then treated as the one that went. `ast` is the only module in the 3.13 stdlib that does this, and it does it with five string literals in plain sight, so the key is read where it can be — a string literal, or a loop target walking a display of them, for a `for` statement and the four comprehensions alike. a key that does not resolve keeps the blanket rule, and so does a body that assigns the target or iterates anything but a display. ast.py goes from 9 compiled units and 59 interpreted to 167 and 13, and the census over the 550 stdlib modules from 7435/1103 to 7593/1057. two constructs the source writes as punctuation were lowered as reads of a name: `a[i:j]` as a call to `slice`, and `...` as a read of `Ellipsis`. python emits BUILD_SLICE for one and loads a constant for the other and never touches a namespace, so a module binding either name for itself was obeyed where python ignores it — `ast` binds both, and `self._source[i:]` raised `TypeError: slice() takes no arguments`. `Op::MakeSlice` and `Op::LoadEllipsis` read no module dict. and a call handed back to the interpreted definition kept only its positional arguments, so `ast.literal_eval` parsed in `exec` mode and refused every literal with `malformed node or string`; such a call goes through the unpacked form now. also: a `def` counts among the definitions a computed removal reaches, and the two namespace-removal enums are `Copy`, which is what clippy's needless_pass_by_value asks about. ## stop assuming what a decorator, a calling frame or a nested scope leaves behind four reads a compiled function was not entitled to make. the window a decorator moved to module init opens was tested for a body that reads the decorated name and for an effect the body sees under another name. the third shape had no test and is the one a real module fails on: an attribute that exists only on what the decorator handed back. `pkgutil` writes it as `@simplegeneric` with `@iter_importer_modules.register` below, and with the gate lifted its import stops with `AttributeError: 'function' object has no attribute 'register'`. the gate is coarse and looks expensive, so its price is written down where the next reader will be: sixty definitions over the 42 modules that carry a decorator decline, against three modules that then stop importing, with the two obvious weakenings named alongside the module each one fails on. a removal key read from the sequence a loop walks is disqualified by every binding of the target inside the loop, compared by position — but a `global` write cannot be compared by position, since it is in a frame of its own and the module body only has to call the function that makes it. reading the tuple would then decline the name the tuple says and install a compiled definition over the name the module actually took out of its namespace, which is a silent wrong answer where the unreadable key was an honest decline. so a target some nested scope declares `global` is not read, and the walk that collects those declarations descends into a `def` and a `class`, which is the point. a compiled function pushes no python frame, so a call that walks the stack from its caller starts one frame further out than the source meant and answers about another module, raising nothing. only `sys._getframe` and `warnings.warn` were guarded, and only when the call spelled them that way — `from sys import _getframe as grab` compiled and answered about the caller. the callee decides now, by identity, against the eight functions in `FRAME_WALKERS`. over the 550-module corpus this costs 4 compiled functions: 7367 -> 7363 compiled. the differential tests reach each walk through a caller, because a walk called straight from the driver's module body has no frame between the two legs to differ over and passed with the defect fully present. and a compiled function read a module global and, where the checker typed it as an instance of one of this module's own classes, narrowed it to that class's emitted representation. the interpreted definitions run first and the whole module body binds its globals against them, so such a name holds an instance of the interpreted definition until module init moves it — and where the class takes its layout from a base outside the module there is no move at all. `Annotated` is one of those values, so `annotation_origin is Annotated` raised `TypeError: expected _TypedCacheSpecialForm, got _TypedCacheSpecialForm` and every `TypedDict('T', {...})` call against a compiled standard library failed. the read is left an object now. ## install a family of classes together, and publish what the class body built a `@property` group left the method table for one object written onto the finished type, and where module init built that type by *calling the metaclass* the name was never in the namespace the metaclass read. `abc.ABCMeta` decides what is still abstract from exactly that namespace, so a class overriding an abstract property of its base came out with the property in `__abstractmethods__` and refused to be instantiated — compiled cleanly, declined nothing, diverged silently from its own twin. the namespace now carries the `property` the interpreted body built, and `By_PublishProperty` leaves that object standing rather than replacing it. that is what lets the gate on a group of one come off a base. a property half's `PyMethodDef` also carried no `ml_doc`, so `C.value.__doc__` was `None` where the interpreted class answered with the getter's docstring. measured over the 76 stdlib modules holding a lone getter with a new `propcensus.sh` rung: 461 lone getters, of which 74 ran compiled before and 150 after, none moved back. a class whose fields sit past a base's instance is built from a type spec, and the spec can refuse. where compiled code that still runs would have read one of those instances the answer has to be the whole module, and that was worked out one class at a time — so a class any other class named, and any class named as a base, took the module down with it. module init now works out the largest family of classes the module could leave interpreted together and installs each member behind a test of every class it reaches. the wait is transitive, because one compiled method calls another's emitted body directly with no type object in between, and it follows the base relation both ways. `asyncio.unix_events` goes from 0 of its 4 classes and 0 of its 3 functions to 1 class and all 3 functions, and nothing else in the 130 stdlib modules that write a class on an outside base installs less. an implicit dunder call — `x[k]`, `x + 1` — is resolved on the type rather than the value, and a structural protocol keeps its members on the instance with no class object for such a lookup to find them on. a bound signature recovery writes is an intersection, and only its other elements have class objects, so a method none of them states was not found at all. `_Unframer.readinto` is that shape: `len(buf)` bounds `buf` by `Sized` while the assignment asks for a `__setitem__` only the recovered protocol states, so the assignment was reported invalid, its right-hand side never inferred, and `_Unframer`, `_Unpickler`, `_load` and `_loads` declined with 83 compiled definitions behind them. the structural half answers only what the rest of the bound did not, so a forwarded type that states the method keeps its precision. `X.register(Y)` records `Y` inside `X`'s own `_abc_impl`, and the fallback source runs the whole module body before module init builds anything, so every registration landed on the interpreted definition while `ABCMeta.__new__` handed the replacement a fresh, empty one — `issubclass(dict, MutableMapping)` answered False off the compiled `_collections_abc` and True off the interpreted one. `By_AdoptTwinAttributes` hands the twin's `_abc_impl` over. and a class keyword no longer turns down a base this module emits: only a class placing storage of its own needs the type spec a keyword has nowhere to sit in, and that question is asked where the fields are known. `_collections_abc.ByteString` is the shape, and it took `Sequence`, `Reversible`, `Collection`, `Iterable`, `Container` and two generators down with it; that module goes from 100 compiled / 16 declined to 115 / 8. a module-level instance is moved onto the type that replaced its class by reading each of the layout's fields off the twin. a name the twin carried that the layout has none of refused the whole move, so `obj = A(1); obj.extra = 7` left the instance standing on the interpreted definition and `isinstance(obj, Cls)` answered False. an emitted class keeps a dict beside its layout wherever the source did not declare `__slots__` throughout, so such a name is carried across with the fields and the refusal applies only where the emitted type keeps no dict at all. a move also registers itself before its fields are filled so a cyclic graph resolves to one object, which left a window where an instance dropped after a later field failed was still standing in somebody else's field with the rest of its layout zeroed — `r.peer is a` False, and reading the unwritten field raised `SystemError: error return without exception set`. a failed move now refuses every move begun inside it. `_colorize` is the one value over the 550-module corpus the carried extras rescue. --- crates/by_build/tests/differential.rs | 2203 +++++++++++++++-- crates/by_codegen_c/src/lib.rs | 580 ++++- crates/by_ir/src/ops.rs | 36 + crates/by_ir/src/print.rs | 13 + crates/by_ir/src/verify.rs | 14 + crates/by_irbuild/src/lib.rs | 747 +++++- crates/by_irbuild/src/tests.rs | 477 +++- crates/by_opt/src/copy_propagation.rs | 2 + crates/by_opt/src/infallible.rs | 5 + crates/by_rt/include/by.h | 549 +++- .../mdtest/basedpython_sound_types.md | 32 + crates/ty_python_semantic/src/lib.rs | 4 +- crates/ty_python_semantic/src/place.rs | 27 + crates/ty_python_semantic/src/types.rs | 18 +- .../src/types/protocol_class.rs | 69 +- .../development/compilation/runtime.md | 59 +- scripts/native-sweeps/propcensus.sh | 95 + 17 files changed, 4314 insertions(+), 616 deletions(-) create mode 100755 scripts/native-sweeps/propcensus.sh diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index 2ab15e8cb5..6f32a6f471 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -2748,6 +2748,131 @@ def digits(n: int) -> str: ); } +/// the source the three tests below share +/// +/// `Held` is what puts them on the road they are about to walk. a module with a class +/// carrying a class-level constant has its body run with each `class` statement captured, +/// and it is that run which decides the builtins every function the body defines will +/// resolve through for the rest of the process — so a module with no such class never +/// reaches the code these test and passes them whatever it does. +/// +/// `eval` is what keeps `scaled` and `newcomer` off the compiled leg. a declined function +/// *is* the definition that run produced, which is why they are the half that can differ; +/// `compiled_scaled` is beside them to say what the same module's other half answers +const LIVE_BUILTINS_SOURCE: &str = "\ +class Held: + tag = 'held' + + def read(self) -> str: + return self.tag + +def scaled(n: int) -> int: + eval('0') + return abs(n) + +def compiled_scaled(n: int) -> int: + return abs(n) + +def newcomer() -> object: + eval('0') + return arriving +"; + +/// a builtin rebound after import is seen by a declined function +/// +/// both halves of the module are asked, because the answer that matters is not only that +/// the declined one is wrong but that it disagrees with the compiled one standing next to +/// it. a test that called `scaled` alone would still fail, but it would not show that one +/// module was answering `abs` two ways at once +#[test] +fn a_declined_function_sees_a_builtin_rebound_after_its_module_was_imported() { + agree_python_with_declines( + "livebuiltinrebind", + LIVE_BUILTINS_SOURCE, + &[ + "(setattr(__import__('builtins'), 'abs', lambda n: n * 100), \ + [m.compiled_scaled(-3), m.scaled(-3)])[1]", + ], + ); +} + +/// a builtin deleted after import stops answering a declined function +/// +/// the deletion is the case a stale namespace cannot express at all: it still holds the +/// entry, so the read succeeds where python's own would raise. the exception is compared +/// rather than caught and counted, so a `NameError` naming something else would not pass +#[test] +fn a_declined_function_stops_seeing_a_builtin_deleted_after_its_module_was_imported() { + agree_python_with_declines( + "livebuiltindelete", + LIVE_BUILTINS_SOURCE, + &["(delattr(__import__('builtins'), 'abs'), _capture(m.scaled, -3))[1]"], + ); +} + +/// a name added to builtins after import is found by a declined function +/// +/// the other direction of the same property: a namespace fixed at import has no entry to +/// find, and a `NameError` is what a stale one answers with +#[test] +fn a_declined_function_finds_a_name_added_to_builtins_after_its_module_was_imported() { + agree_python_with_declines( + "livebuiltinadd", + LIVE_BUILTINS_SOURCE, + &["(setattr(__import__('builtins'), 'arriving', 'here'), m.newcomer())[1]"], + ); +} + +/// a module body writing through `__builtins__` reaches the interpreter's own +/// +/// the other side of the same mapping being live: the body does not merely *read* what +/// the process has done to `builtins`, it writes where the process will see it. a run +/// given a namespace of its own would take this write and drop it on the floor at the end +/// of the import, which is a change made and silently undone +#[test] +fn a_module_body_writing_through_builtins_reaches_the_interpreter_s_own() { + agree_python( + "bodywritesbuiltins", + "\ +class Held: + tag = 'held' + + def read(self) -> str: + return self.tag + +__builtins__['planted'] = 'by the body' +", + &["[getattr(__import__('builtins'), 'planted', ''), m.Held.tag]"], + ); +} + +/// capturing a module's class bodies leaves a `class` statement written elsewhere alone +/// +/// the capture stands in the interpreter's own builtins while a body runs, so every +/// `class` statement in the process reaches it and only this module's have anything to do +/// with it — what tells them apart is the namespace the statement was written in. the two +/// classes are named the same on purpose: a capture that recorded the other one would +/// overwrite what this module's own statement wrote, and the constant `Shared` ends up +/// carrying is what says which body was read +#[test] +fn a_capturing_module_body_records_only_the_classes_written_in_it() { + agree_python( + "foreigncapture", + "\ +class Shared: + origin = 'outer' + + def where(self) -> str: + return self.origin + +_elsewhere: dict = {} +exec(\"class Shared:\\n origin = 'inner'\\n\", _elsewhere) +held = _elsewhere['Shared'] +", + &["[m.Shared.origin, m.Shared().where(), m.held.origin]"], + ); +} + /// a builtin rebound while a call site is already holding it is seen at once /// /// the site remembers which namespace answered, so *that* namespace has to be one @@ -3895,6 +4020,59 @@ SNAPSHOT = list(REGISTRY) ); } +/// and so is an attribute that exists only on what the decorator handed back +/// +/// the two cases above are a decorated name the body reads and an effect the body sees +/// under another name. this is the third and it is the one the standard library actually +/// breaks on: `render` is a `Table` from the moment its `def` stands, and `@render.register` +/// below reads an attribute a plain function does not have. move `@dispatcher` to init and +/// that read happens while the name still holds the function, so the import stops rather +/// than answering wrongly — `pkgutil` writes this as `@simplegeneric` with +/// `@iter_importer_modules.register` under it, and with this gate lifted its import fails +/// with `AttributeError: 'function' object has no attribute 'register'`. +/// +/// the assertions are about *identity* rather than an answer, because that is the half a +/// decorator moved to init gets wrong: `render` is the decorator's object on both legs, and +/// `other` is what `register` gave back +#[test] +fn a_decorated_definition_the_body_reaches_through_the_decorators_answer_declines() { + agree_python_with_declines( + "decoratoranswer", + "\ +class Table: + def register(self, case): + self.cases.append(case.__name__) + return case + + def __init__(self): + self.cases = [] + + +def dispatcher(f): + return Table() + + +@dispatcher +def render() -> str: + return \"default\" + + +@render.register +def other() -> str: + return \"other\" + + +AT_IMPORT = list(render.cases) +", + &[ + "type(m.render).__name__", + "m.render.cases", + "m.AT_IMPORT", + "m.other()", + ], + ); +} + /// the source both method-decorator tests below compile /// /// `mark` hands back what it was given, so the binding is right however many times it ran @@ -11087,6 +11265,254 @@ class Box: ); } +/// a published property answers `__doc__` with what the getter's body opens with +/// +/// two surfaces, and one docstring behind both. `p.fget.__doc__` is read straight off the +/// `PyMethodDef` the half is built from, and `p.__doc__` is python's own: `property` takes +/// its documentation off the getter exactly when it is handed none, which is what +/// `By_PublishProperty` does. +/// +/// this is the shape of divergence no comparison of *answers* reaches. the property reads, +/// writes and refuses identically with the entry left NULL — it simply says `None` where +/// the interpreted class says the text, and every other property test here passed with it +#[test] +fn a_published_property_carries_the_getters_docstring() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_propdoc"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Box: + def __init__(self, n: int) -> None: + self._n = n + + @property + def value(self) -> int: + \"what the box holds\" + return self._n + + @value.setter + def value(self, given: int) -> None: + \"put something else in it\" + self._n = given + + @property + def lone(self) -> int: + \"a group of one documents itself the same way\" + return self._n + 1 + + @property + def bare(self) -> int: + return 0 +"; + let built = match build_source( + source, + "by_diff_propdoc", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_propdoc as m\n\ + for name in ('value', 'lone', 'bare'):\n\ + \x20 p = m.Box.__dict__[name]\n\ + \x20 print(type(p.fget).__name__, repr(p.__doc__), repr(p.fget.__doc__))\n\ + print(repr(m.Box.__dict__['value'].fset.__doc__))\n", + ); + assert_eq!( + out, + "method_descriptor 'what the box holds' 'what the box holds'\n\ + method_descriptor 'a group of one documents itself the same way' 'a group of one documents itself the same way'\n\ + method_descriptor None None\n\ + 'put something else in it'" + ); +} + +/// a `@property` over an abstract one is not still abstract on the emitted class +/// +/// a class whose base carries `abc.ABCMeta` is built by *calling* that metaclass, because +/// a type spec gives what it builds `type` as its own and any other metaclass on a base is +/// a conflict. which of the two applies is a runtime answer — it depends on what the base +/// names resolved to — so nothing at build time can tell this class apart from one a spec +/// will build. +/// +/// `ABCMeta.__new__` decides what is still abstract from the namespace it is handed, and a +/// property's halves are in no method table, so the namespace has to carry the `property` +/// the interpreted body built or the name is simply not there. it was not: the emitted +/// class reported `value` abstract and refused to be instantiated at all, while the +/// interpreted twin of the same source built. nothing declined and nothing was reported +/// +/// what the class answers with here is that carried `property` — the interpreted body, +/// which is what a class statement would have left under the name +#[test] +fn a_property_over_an_abstract_base_is_not_left_abstract() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_propabstract"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +import abc + + +class Held(abc.ABC): + @property + @abc.abstractmethod + def value(self) -> int: ... + + @property + @abc.abstractmethod + def lone(self) -> int: ... + + +class Box(Held): + @property + def value(self) -> int: + return 5 + + @value.setter + def value(self, given: int) -> None: + raise RuntimeError(\"no\") + + @property + def lone(self) -> int: + return 6 +"; + let built = match build_source( + source, + "by_diff_propabstract", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + // `Held`'s two stubs decline, because a body of `...` reaches its end without + // returning what it says it returns. `Box` itself is emitted whole, which is the + // question here + assert!( + !built + .declined + .iter() + .any(|declined| declined.name.starts_with("Box")), + "declined: {:?}", + built.declined + ); + let out = run( + &python, + &dir, + "import by_diff_propabstract as m\n\ + print(sorted(m.Box.__abstractmethods__))\n\ + b = m.Box()\n\ + print(b.value, b.lone)\n\ + for name in ('value', 'lone'):\n\ + \x20 p = m.Box.__dict__[name]\n\ + \x20 print(type(p).__name__, type(p.fget).__name__, repr(p.__doc__))\n", + ); + assert_eq!( + out, + "[]\n\ + 5 6\n\ + property function None\n\ + property function None" + ); +} + +/// a group of one on a class with a base of this module's is published over the body +/// +/// such a class is built from a spec standing on the base's finished type, because a class +/// this module built from a spec has `type` for its own metaclass — so nothing was handed a +/// namespace and the published property is the only thing that ever reaches the name. +/// +/// a base used to be enough on its own to leave the group alone, which is what left 101 of +/// the standard library's lone getters running interpreted while their classes compiled. +/// what makes the base safe is not the base: it is that the two constructions now answer +/// the question separately at import +#[test] +fn a_lone_property_getter_over_an_emitted_base_is_published() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_propbase"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Held: + def held(self) -> int: + return 1 + + +class Box(Held): + @property + def value(self) -> int: + \"the value over a base\" + return self.held() * 10 +"; + let built = match build_source( + source, + "by_diff_propbase", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_propbase as m\n\ + p = m.Box.__dict__['value']\n\ + print(type(p).__name__, isinstance(p, property), p.fset, p.fdel)\n\ + print(type(p.fget).__name__, p.fget.__name__, p.fget.__qualname__, repr(p.__doc__))\n\ + b = m.Box()\n\ + print(b.value, p.__get__(b), hasattr(m.Box, 'value$get'))\n\ + for verb, run_it in (('set', lambda: setattr(b, 'value', 1)),\n\ + \x20 ('del', lambda: delattr(b, 'value'))):\n\ + \x20 try:\n\ + \x20 run_it()\n\ + \x20 except AttributeError as e:\n\ + \x20 print(verb, e)\n", + ); + assert_eq!( + out, + "property True None None\n\ + method_descriptor value Box.value 'the value over a base'\n\ + 10 10 False\n\ + set property 'value' of 'Box' object has no setter\n\ + del property 'value' of 'Box' object has no deleter" + ); +} + /// a `@property` with no setter answers and refuses exactly as the interpreted one does /// /// the group of one is published rather than left to the class body's own object, so @@ -12077,26 +12503,112 @@ def declared_global(n: int) -> int: } #[test] -fn a_bad_first_argument_raises_rather_than_crashing() { - // the wrapper releases every argument local on the error path, so one whose - // declaration a `goto` skipped would be released while indeterminate. a wrong - // type in the *first* parameter is the reachable case: it jumps over the rest. +fn a_module_binding_slice_or_ellipsis_does_not_change_what_the_punctuation_means() { + // `a[i:j]` and `...` are punctuation rather than names: python builds the slice + // from `BUILD_SLICE` and loads the singleton as a constant, and neither reads the + // module namespace. lowering them as a call to `slice` and a read of `Ellipsis` + // meant a module binding either name for itself was obeyed where python ignores + // it — `ast` binds both, so `self._source[i:]` inside its unparser built a + // deprecated AST node and raised `TypeError: slice() takes no arguments`. // - // only the exception *type* is compared: the boundary rejects a bad argument - // where the interpreted leg gets as far as the operation that uses it, so the - // two agree that it is a `TypeError` and not on where it was raised - agree( - "badarg", + // the two classes are what the names would otherwise resolve to, and reading them + // back is what says the module really does bind them + agree_python( + "shadowedslice", "\ -def two(a: int, b: str) -> int: - return a + len(b) +class slice: + def __init__(self) -> None: + self.tag = 'not a slice' -def three(a: int, b: str, c: list[int]) -> int: - return a + len(b) + len(c) -", - &[ - "type(_capture(m.two, 'x', 'y')).__name__", - "type(_capture(m.two, 1, 2)).__name__", +class Ellipsis: + def __init__(self) -> None: + self.tag = 'not the singleton' + +def sliced(xs: list[int]) -> str: + return str(xs[1:3]) + str(xs[::2]) + str(xs[2:]) + str(xs[::-1]) + +def assigned(xs: list[int]) -> str: + xs[1:3] = [9, 9, 9] + return str(xs) + +def dots() -> bool: + return ... is Ellipsis + +def bound() -> str: + return slice().tag + Ellipsis().tag +", + &[ + "m.sliced([1, 2, 3, 4])", + "m.assigned([1, 2, 3, 4])", + "m.dots()", + "m.bound()", + "type(m.slice).__name__", + "m.Ellipsis().tag", + ], + ); +} + +#[test] +fn a_call_handed_back_to_the_interpreted_definition_keeps_its_keywords() { + // a parameter whose default the compiler cannot inline — `-1` is a negation rather + // than a literal, so only the interpreted definition holds it — makes every call + // that omits it reach that definition instead of the native entry. that hand-back + // used to pass the positional arguments and nothing else, so a keyword the caller + // wrote was dropped and the callee fell back on its *own* default for it. + // + // `ast.literal_eval` is where this was found: it calls `parse(source, mode='eval')` + // and `parse` has such a default, so the parse ran in `exec` mode and every literal + // came back a `Module` the converter refused with `malformed node or string`. + // + // `named` is the same call shape with a default the compiler *can* inline, which + // reaches the native entry — the two together say the keyword survives both routes + agree_python( + "deferredkeyword", + "\ +def deferring(a: str, b: str = 'B', c: int = -1) -> str: + return a + b + str(c) + +def inlined(a: str, b: str = 'B', c: int = 1) -> str: + return a + b + str(c) + +def by_keyword(x: str) -> str: + return deferring(x, b='b') + +def by_keyword_out_of_order(x: str) -> str: + return deferring(x, c=7, b='b') + +def named(x: str) -> str: + return inlined(x, b='b') +", + &[ + "m.by_keyword('a')", + "m.by_keyword_out_of_order('a')", + "m.named('a')", + ], + ); +} + +#[test] +fn a_bad_first_argument_raises_rather_than_crashing() { + // the wrapper releases every argument local on the error path, so one whose + // declaration a `goto` skipped would be released while indeterminate. a wrong + // type in the *first* parameter is the reachable case: it jumps over the rest. + // + // only the exception *type* is compared: the boundary rejects a bad argument + // where the interpreted leg gets as far as the operation that uses it, so the + // two agree that it is a `TypeError` and not on where it was raised + agree( + "badarg", + "\ +def two(a: int, b: str) -> int: + return a + len(b) + +def three(a: int, b: str, c: list[int]) -> int: + return a + len(b) + len(c) +", + &[ + "type(_capture(m.two, 'x', 'y')).__name__", + "type(_capture(m.two, 1, 2)).__name__", "type(_capture(m.three, 'x', 'y', [1])).__name__", "type(_capture(m.three, 1, 'y', 'z')).__name__", "m.two(1, 'yy')", @@ -13066,6 +13578,127 @@ fn the_classes_beside_a_refused_one_lay_out_and_deallocate() { assert_eq!(out, "True\n(0, 0, 0) (0, 0, 0)"); } +/// a family standing on a heap base from another module, which is the shape the standard +/// library's handler and transport hierarchies have +/// +/// `Base` keeps a field past a `SubprocessError` instance, so it is built from a type +/// spec — and no spec can stand on a heap base, so it refuses. what that used to cost was +/// the whole module: `Rotating` names `Base` in its own header, a NULL is not something a +/// bases tuple can be packed from, and so a base another class stands on was always the +/// module's to refuse. `count` was left interpreted along with it. +/// +/// nothing outside the family reads either class, so the two stand down together instead. +/// they have to move as one in both directions: leaving `Rotating` interpreted while +/// `Base`'s emitted type took the base's name would leave it standing on an orphaned copy, +/// and `isinstance(m.Rotating(...), m.Base)` would answer False where python answers True. +/// +/// `asyncio.unix_events` is the module this is for. its four classes stand on heap bases +/// from other asyncio modules, only each other names them, and one refusal left every one +/// of them — and every function in the module — interpreted +const REFUSED_FAMILY: &str = "\ +from subprocess import SubprocessError + + +class Base(SubprocessError): + def __init__(self, tag): + SubprocessError.__init__(self, tag) + self.tag = tag + + def label(self): + return 'base:' + self.tag + + +class Rotating(Base): + def __init__(self, tag, limit): + Base.__init__(self, tag) + self.limit = limit + + def label(self): + return 'rotating:' + self.tag + ':' + str(self.limit) + + +def count(values): + total = 0 + for value in values: + total += value + return total +"; + +#[test] +fn a_family_no_spec_can_build_agrees_beside_a_module_function() { + agree_python( + "refusedfamily", + REFUSED_FAMILY, + &[ + "(m.Base('b').label(), m.Rotating('r', 2).label(), m.count([1, 2, 3]))", + "[(e.tag, e.args, str(e)) for e in (m.Base('b'), m.Rotating('r', 2))]", + "[c.__name__ for c in m.Rotating.__mro__]", + // the pair's identity, which is what parting them company would break + "(isinstance(m.Rotating('r', 2), m.Base),\n\ + \x20 issubclass(m.Rotating, m.Base), m.Rotating.__base__ is m.Base,\n\ + \x20 isinstance(m.Base('b'), __import__('subprocess').SubprocessError))", + // a field the base declared, read and written through the subclass + "[(e.tag, (setattr(e, 'tag', 'w'), e.tag)[1], e.label())\n\ + \x20 for e in [m.Rotating('r', 2)]]", + // raised and caught, which is what an exception family is for + "[(type(e).__name__, e.tag, str(e)) for e in\n\ + \x20 (_raised_and_caught(m.Base, 'boom'),\n\ + \x20 _raised_and_caught(m.Rotating, 'bang', 3))]", + ], + ); +} + +/// which build answered, which no comparison of the two legs can say +/// +/// the family is interpreted either way — `__firstlineno__` is in both class dicts, since +/// a spec has no code object to write one from — so the only thing that moves is whether +/// the module's own function was installed. before the family could stand down together, +/// `by_exec` gave up before `PyModule_AddFunctions` ran and `count` was an ordinary python +/// function +#[test] +fn a_module_function_stands_where_the_whole_family_stood_down() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_refusedfamily_t"); + let _ = std::fs::remove_dir_all(&dir); + let built = match build_source( + REFUSED_FAMILY, + "by_diff_refusedfamily_t", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_refusedfamily_t as m\n\ + _leg = lambda f: 'native' if f.__code__.co_filename == '' else type(f).__name__\n\ + print(_leg(m.count), m.count([1, 2, 3]))\n\ + print('__firstlineno__' in vars(m.Base), '__firstlineno__' in vars(m.Rotating))\n\ + # the pair kept the definitions the module body built, so they still agree on\n\ + # what a `Rotating` is an instance of\n\ + print(m.Rotating.__base__ is m.Base, isinstance(m.Rotating('r', 2), m.Base))\n", + ); + assert_eq!( + out, + "native 6\n\ + True True\n\ + True True" + ); +} + /// a chain where every rung keeps fields of its own past the one below, which is the /// stdlib's commonest exception family — `configparser` writes ten of them /// @@ -13811,6 +14444,109 @@ def widened(value: object) -> str: ); } +/// the source the two tests below compile +/// +/// `Marked` carries a class keyword *and* a base this module emits, which is what +/// `_collections_abc.ByteString` is: `class ByteString(Sequence, metaclass=...)` over a +/// `Sequence` the same module writes. the keyword is what names the construction such a +/// class has — calling the metaclass on that very base — so it is no reason to turn the +/// base down. +/// +/// each class is then registered against at module level, the way a body tells an abstract +/// base which built-in types satisfy it. that registration lands on the interpreted +/// definition, because the fallback source runs the whole body before module init builds +/// anything, so the type replacing it has to be given the registry the twin collected +const KEYED_OVER_AN_EMITTED_BASE: &str = "\ +from abc import ABCMeta + + +class Root(metaclass=ABCMeta): + def kind(self) -> str: + return \"root\" + + +class Marked(Root, metaclass=ABCMeta): + def kind(self) -> str: + return \"marked\" + + +Root.register(list) +Marked.register(tuple) +"; + +#[test] +fn a_class_keyed_over_a_base_this_module_emits_agrees() { + agree_python( + "keyedbase", + KEYED_OVER_AN_EMITTED_BASE, + &[ + // the base is the emitted one, not a second class left standing under the + // name — which is the whole of what the construction had to get right + "m.Marked.__bases__ == (m.Root,)", + "m.Marked.__bases__[0] is m.Root", + "[c.__name__ for c in m.Marked.__mro__]", + // the keyword reached the metaclass, so the class is `ABCMeta`'s and not + // `type`'s — a spec-built one would report `type` here + "(type(m.Root).__name__, type(m.Marked).__name__)", + // the two directions of each question, so a leg answering `True` to + // everything is not mistaken for agreement + "(issubclass(m.Marked, m.Root), issubclass(m.Root, m.Marked))", + "(isinstance(m.Marked(), m.Root), isinstance(m.Root(), m.Marked))", + // what the module body registered, which only the twin was ever told + "(issubclass(list, m.Root), issubclass(list, m.Marked))", + "(issubclass(tuple, m.Marked), isinstance((1,), m.Marked))", + // registering against a subclass reaches the base, which takes `__subclasses__` + // over the emitted pair rather than over the twins + "issubclass(tuple, m.Root)", + "(isinstance([], m.Root), isinstance([], m.Marked))", + "(m.Root().kind(), m.Marked().kind())", + "(m.Root.__module__, m.Marked.__module__)", + "(m.Root.__name__, m.Marked.__qualname__)", + ], + ); +} + +#[test] +fn a_class_keyed_over_a_base_this_module_emits_is_the_compiled_type() { + // the agreement above is answered exactly the same way by a pair that fell back to + // their interpreted definitions, so it cannot say which build answered. a + // `method_descriptor` can: a compiled type holds one where an interpreted class holds + // a plain function + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_keyedbase_t"); + let _ = std::fs::remove_dir_all(&dir); + let built = match build_source( + KEYED_OVER_AN_EMITTED_BASE, + "by_diff_keyedbase_t", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_keyedbase_t as m\n\ + print(type(m.Root.__dict__['kind']).__name__,\n\ + \x20 type(m.Marked.__dict__['kind']).__name__)\n\ + print(m.Marked.__bases__[0] is m.Root)\n\ + print(issubclass(list, m.Root), issubclass(tuple, m.Marked))\n", + ); + assert_eq!(out, "method_descriptor method_descriptor\nTrue\nTrue True"); +} + /// a decorated method reaches the namespace the metaclass reads, not the type it built /// /// `ABCMeta` decides `__abstractmethods__` by walking the namespace it is handed, so a @@ -14843,38 +15579,90 @@ class Caught: } #[test] -fn a_slots_declaration_reaches_the_metaclass_rather_than_the_finished_type() { - // `__slots__` is the constant that proves the namespace is where these have to go. - // `type.__new__` reads it *out of the namespace* to decide whether the instances get - // a dict at all, so one copied onto the finished type afterwards is not a `__slots__` - // — the class already has the dict, and the entry sits there saying otherwise. +fn a_class_written_in_a_class_body_is_carried_off_the_interpreted_definition() { + // a `class` in a class body binds a name there like any other statement, and the + // interpreted definition built the class already — bases read, decorator applied, + // inner body run, all where python runs them. so the object it left behind is copied + // across the way every class-level constant is, and the outer class is compiled + // around it. // - // 29 stdlib classes are this shape. `Open` is the boundary: the same base and the - // same construction with no `__slots__`, and python gives *its* instances a dict, so - // this is not a rule about emitted classes but about what the body wrote + // `error` / `abort` is `imaplib.IMAP4`'s shape, where the second inner class stands + // on the first. `__Private` is python's own name mangling: the binding lands in the + // namespace as `_Holder__Private`, so the copy has to look for it under that name. + // and a `class` under a conditional is the same binding a `def` under one makes + agree_python( + "nestedclass", + "\ +on = True + + +class Holder: + class error(Exception): + pass + + class abort(error): + pass + + class __Private: + n = 4 + + if on: + class Conditional: + n = 5 + + def raised(self) -> str: + try: + raise Holder.abort('gone') + except Holder.error as failure: + return type(failure).__name__ + + def total(self) -> int: + return Holder.__Private.n + Holder.Conditional.n +", + &[ + "m.Holder().raised()", + "m.Holder().total()", + "(m.Holder.error.__qualname__, m.Holder.abort.__qualname__)", + "issubclass(m.Holder.abort, m.Holder.error)", + "m.Holder._Holder__Private.__qualname__", + "hasattr(m.Holder, '__Private')", + "type(m.Holder.error('boom')).__name__", + // an interpreted class can still be derived from, which is the whole reason + // the inner one keeps its own definition rather than being emitted + "type('Derived', (m.Holder.abort,), {})('x').args", + ], + ); +} + +#[test] +fn the_outer_class_is_compiled_and_the_class_written_in_it_is_not() { + // the two legs of the test above answer alike whichever definition stands, so this + // asks which one did: `method_descriptor` is the emitted method table, `function` is + // the interpreted leg. the outer class is the whole point — `imaplib.IMAP4` has 79 + // methods behind one nested `class` statement let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_metaslots"); + let dir = diff_root().join("by_diff_nested_class_legs"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -from abc import ABC - - -class Slotted(ABC): - __slots__ = () +def tag(cls: type) -> type: + cls.tagged = True + return cls - def label(self) -> str: - return \"slotted\" +class Holder: + @tag + class Inner: + def v(self) -> int: + return 7 -class Open(ABC): - def label(self) -> str: - return \"open\" + def make(self) -> int: + return Holder.Inner().v() "; let built = match build_source( source, - "by_diff_metaslots", + "by_diff_nested_class_legs", &toolchain, &dir, &Options { @@ -14889,56 +15677,72 @@ class Open(ABC): return; } }; - assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + assert!( + !built + .declined + .iter() + .any(|declined| declined.name == "Holder"), + "the outer class declined: {:?}", + built.declined + ); let out = run( &python, &dir, - "import by_diff_metaslots as m\n\ - print(m.Slotted.__slots__, hasattr(m.Slotted(), '__dict__'), hasattr(m.Open(), '__dict__'))\n\ - print(m.Slotted().label(), m.Open().label())\n\ - print(type(m.Slotted.label).__name__, type(m.Open.label).__name__)\n", + "import by_diff_nested_class_legs as m\n\ + print(m.Holder().make(), m.Holder.Inner.tagged)\n\ + print(type(m.Holder.make).__name__, type(m.Holder.Inner.v).__name__)\n", ); assert_eq!( out, - "() False True\n\ - slotted open\n\ - method_descriptor method_descriptor" + "7 True\n\ + method_descriptor function" ); } #[test] -fn a_class_constant_naming_another_class_reaches_the_metaclass_namespace_remapped() { - // the value a constant carries comes off the twin, so `pair = Other` in a class body - // hands over the *interpreted* `Other` — a class nothing else in the module can - // reach, and one `isinstance` denies against `m.Other`. the substitution that fixes - // that is the copy's, and the namespace has to make the same one or the two - // constructions disagree about what a constant is. +fn a_base_a_class_written_in_a_class_body_stands_on_gives_up_its_emission() { + // such a class is never emitted: it is copied off the interpreted definition whole. so + // it stands on whatever its base name held while that body ran, which is the + // interpreted definition's class — and if the module emitted a type under that name, + // the copy carries a second, orphaned copy of it and `isinstance` answers `False` + // where python answers `True`, from ordinary code, with nothing reported. // - // `Below` is what forces the metaclass here: `ABCMeta` closes the spec, so the - // constant goes in before the call rather than onto the type after it + // the base gives up its emission instead, exactly as it does for a module-level class + // this module does not emit, and both types are then the interpreted ones. the outer + // class still compiles, which is the whole point of lowering the nested `class`. + // + // `twice()` answers `6` either way, which is why this asserts on the *types*: a + // behavioural check on the method alone passes with the divergence fully present let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_metaconstant_remap"); + let dir = diff_root().join("by_diff_nested_class_emitted_base"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -from abc import ABCMeta +class Emitted: + def __init__(self, v: int) -> None: + self.v = v + def read(self) -> int: + return self.v -class Other: - def kind(self) -> str: - return \"other\" +class Based: + class Inner(Emitted): + def twice(self) -> int: + return self.read() * 2 -class Below(metaclass=ABCMeta): - pair = Other + def held(self) -> object: + return Based.Inner - def label(self) -> str: - return \"below\" + +class Computed: + class Inner(*[Emitted]): + pass "; let built = match build_source( source, - "by_diff_metaconstant_remap", + "by_diff_nested_class_emitted_base", &toolchain, &dir, &Options { @@ -14953,15 +15757,238 @@ class Below(metaclass=ABCMeta): return; } }; - assert!(built.declined.is_empty(), "declined: {:?}", built.declined); - let out = run( - &python, - &dir, - "import by_diff_metaconstant_remap as m\n\ - print(m.Below.pair is m.Other, isinstance(m.Below.pair(), m.Other))\n\ - print(m.Below.pair().kind(), m.Below().label())\n\ - print(type(m.Below.label).__name__, type(m.Other.kind).__name__)\n", - ); + let mut declined: Vec<(&str, &str)> = built + .declined + .iter() + .map(|declined| (declined.name.as_str(), declined.reason.as_str())) + .collect(); + declined.sort_unstable(); + assert_eq!( + declined, + vec![ + // a base the header does not *name* is one no collected list can hold, so the + // class holding it is turned down instead + ( + "Computed", + "a base of `Inner` is worked out rather than named, so whether it is a class this module emits cannot be told", + ), + ( + "Emitted", + "`Based.Inner` is written in a class body, so it stands on the interpreted definition rather than this type", + ), + ] + ); + let out = run( + &python, + &dir, + "import by_diff_nested_class_emitted_base as m\n\ + print(m.Based.Inner.__bases__[0] is m.Emitted, m.Computed.Inner.__bases__[0] is m.Emitted)\n\ + print(isinstance(m.Based.Inner(3), m.Emitted), m.Based().held() is m.Based.Inner)\n\ + print(m.Based.Inner(3).twice(), type(m.Based.held).__name__)\n", + ); + // `Based` is still compiled — `held` answers from the method table — while the base it + // holds a class over, and that class, are both the interpreted definitions + assert_eq!( + out, + "True True\n\ + True True\n\ + 6 method_descriptor" + ); +} + +#[test] +fn the_shapes_a_class_written_in_a_class_body_is_not_lowered_for_decline() { + // the two the copy cannot answer for. a dunder is settled from the body text — a + // type slot, an instance layout, what the class publishes — while the copy only + // knows what the name holds once the interpreter has run the body. and a `def` + // beside a `class` of the same name is two definitions of one attribute: the `def` + // would go into the method table while the copy carried whatever python kept + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_nested_class_declines"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Dunder: + n = 0 + + class __repr__: + pass + + +class Twice: + def load(self, n: int) -> int: + return n + + class load: + pass +"; + let built = match build_source( + source, + "by_diff_nested_class_declines", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + let mut declined: Vec<(&str, &str)> = built + .declined + .iter() + .map(|declined| (declined.name.as_str(), declined.reason.as_str())) + .filter(|(name, _)| matches!(*name, "Twice" | "Dunder")) + .collect(); + declined.sort_unstable(); + assert_eq!( + declined, + vec![ + ( + "Dunder", + "`__repr__` is written as a class in the class body, and a dunder is settled before one runs", + ), + ( + "Twice", + "`load` is both defined by this class body and written as a class in it", + ), + ] + ); + let out = run( + &python, + &dir, + "import by_diff_nested_class_declines as m\n\ + print(m.Dunder.n, type(m.Dunder.__repr__).__name__, type(m.Twice.load).__name__)\n", + ); + assert_eq!(out, "0 type type"); +} + +#[test] +fn a_slots_declaration_reaches_the_metaclass_rather_than_the_finished_type() { + // `__slots__` is the constant that proves the namespace is where these have to go. + // `type.__new__` reads it *out of the namespace* to decide whether the instances get + // a dict at all, so one copied onto the finished type afterwards is not a `__slots__` + // — the class already has the dict, and the entry sits there saying otherwise. + // + // 29 stdlib classes are this shape. `Open` is the boundary: the same base and the + // same construction with no `__slots__`, and python gives *its* instances a dict, so + // this is not a rule about emitted classes but about what the body wrote + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_metaslots"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +from abc import ABC + + +class Slotted(ABC): + __slots__ = () + + def label(self) -> str: + return \"slotted\" + + +class Open(ABC): + def label(self) -> str: + return \"open\" +"; + let built = match build_source( + source, + "by_diff_metaslots", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_metaslots as m\n\ + print(m.Slotted.__slots__, hasattr(m.Slotted(), '__dict__'), hasattr(m.Open(), '__dict__'))\n\ + print(m.Slotted().label(), m.Open().label())\n\ + print(type(m.Slotted.label).__name__, type(m.Open.label).__name__)\n", + ); + assert_eq!( + out, + "() False True\n\ + slotted open\n\ + method_descriptor method_descriptor" + ); +} + +#[test] +fn a_class_constant_naming_another_class_reaches_the_metaclass_namespace_remapped() { + // the value a constant carries comes off the twin, so `pair = Other` in a class body + // hands over the *interpreted* `Other` — a class nothing else in the module can + // reach, and one `isinstance` denies against `m.Other`. the substitution that fixes + // that is the copy's, and the namespace has to make the same one or the two + // constructions disagree about what a constant is. + // + // `Below` is what forces the metaclass here: `ABCMeta` closes the spec, so the + // constant goes in before the call rather than onto the type after it + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_metaconstant_remap"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +from abc import ABCMeta + + +class Other: + def kind(self) -> str: + return \"other\" + + +class Below(metaclass=ABCMeta): + pair = Other + + def label(self) -> str: + return \"below\" +"; + let built = match build_source( + source, + "by_diff_metaconstant_remap", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_metaconstant_remap as m\n\ + print(m.Below.pair is m.Other, isinstance(m.Below.pair(), m.Other))\n\ + print(m.Below.pair().kind(), m.Below().label())\n\ + print(type(m.Below.label).__name__, type(m.Other.kind).__name__)\n", + ); assert_eq!( out, "True True\n\ @@ -14972,16 +15999,15 @@ class Below(metaclass=ABCMeta): #[test] fn a_class_the_module_pops_out_of_its_own_globals_stays_off_the_compiled_surface() { - // `ast` builds `Num` and then pops the name straight out of its own globals. that is - // a `del` whose target this cannot read — the name comes off a comprehension there — - // so every definition the module writes is treated as one the pop could have taken. - // installing a compiled `Gone` over a name the body removed would put a class on the - // surface python does not have there, and the interpreted definition the construction - // would otherwise fall back to is not there to be found either. + // `ast` builds `Num` and then pops the name straight out of its own globals, through + // a dict comprehension over a tuple of string literals. installing a compiled `Gone` + // over a name the body removed would put a class on the surface python does not have + // there, and the interpreted definition the construction would otherwise fall back to + // is not there to be found either. // - // the class-level-constant gate used to carry this, and this is what stayed behind - // when it went. `Kept` is no longer a boundary — the rule reaches the whole module, - // which is what its second decline says + // the tuple is what says which names went, so `Kept` is not one of them and compiles. + // the last line is the one that says so: a `method_descriptor` is the compiled leg + // answering, a `function` the interpreted one let Some((python, toolchain)) = environment() else { return; }; @@ -15029,16 +16055,10 @@ HIDDEN = {name: globals().pop(name) for name in (\"Gone\",)} .collect(); assert_eq!( declined, - vec![ - ( - "Gone", - "`Gone` is rebound at module level, so installing this over it would replace what the rebind produced" - ), - ( - "Kept", - "`Kept` is rebound at module level, so installing this over it would replace what the rebind produced" - ) - ] + vec![( + "Gone", + "`Gone` is rebound at module level, so installing this over it would replace what the rebind produced" + )] ); let out = run( &python, @@ -15052,6 +16072,76 @@ HIDDEN = {name: globals().pop(name) for name in (\"Gone\",)} out, "False True\n\ 1 gone kept\n\ + function method_descriptor" + ); +} + +#[test] +fn a_class_the_module_pops_out_under_a_computed_name_takes_the_whole_module_with_it() { + // the other half of the rule above. a key worked out at runtime names nothing that + // can be read where the module is compiled, so every definition the body wrote is + // treated as the one that went — including `Kept`, which the pop never touches. + // both legs answer alike either way, and `function` twice is what says both fell + // back to the interpreted definitions + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_computedpop"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Gone: + def label(self) -> str: + return \"gone\" + + +class Kept: + def label(self) -> str: + return \"kept\" + + +def pick() -> str: + return \"Gone\" + + +HIDDEN = globals().pop(pick()) +"; + let built = match build_source( + source, + "by_diff_computedpop", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + let declined: Vec<&str> = built + .declined + .iter() + .map(|declined| declined.name.as_str()) + .collect(); + // `pick` too: the rule reaches every definition the module wrote, and a `def` is + // one of them + assert_eq!(declined, vec!["Gone", "Kept", "pick"]); + let out = run( + &python, + &dir, + "import by_diff_computedpop as m\n\ + print('Gone' in m.__dict__, 'Kept' in m.__dict__)\n\ + print(m.HIDDEN().label(), m.Kept().label())\n\ + print(type(m.HIDDEN.label).__name__, type(m.Kept.label).__name__)\n", + ); + assert_eq!( + out, + "False True\n\ + gone kept\n\ function function" ); } @@ -15393,14 +16483,16 @@ Holder.wrapper = Holder(leaf) /// loud failure it already gave. #[test] fn an_instance_the_layout_cannot_hold_is_left_where_the_body_built_it() { - // three refusals, one for each thing the layout has no room for. + // two refusals, one for each thing the layout has no answer for. + // + // `bare` was built through `__new__` and never ran `__init__`, so the field the layout + // treats as always defined was never written and there is nothing to move onto it. + // `raised` is an instance of a class standing on a base python allocates, and whatever + // `Exception` keeps for it lives in a part of the object nothing here can read back. // - // `spare` carries an attribute nothing declared, so the emitted instance would answer - // the layout's fields and quietly lose `extra`. `bare` was built through `__new__` and - // never ran `__init__`, so the field the layout treats as always defined was never - // written and there is nothing to move onto it. `raised` is an instance of a class - // standing on a base python allocates, and whatever `Exception` keeps for it lives in - // a part of the object nothing here can read back. + // an attribute nothing declared is *not* among these — the emitted class keeps a dict + // beside its layout, and the name goes there; see + // `an_instance_carrying_a_name_the_layout_never_had_moves_with_it`. // // both classes still compile — the refusal is about one value, not about the class — // and `wrapper_descriptor`/`method_descriptor` is what says so @@ -15420,49 +16512,290 @@ class Tagged(Exception): return \"tagged\" -spare = Loose() -spare.extra = 2 -Loose.spare = spare +bare = Loose.__new__(Loose) +Loose.bare = bare + +raised = Tagged(\"boom\") +Tagged.raised = raised +"; + let built = match build_source( + source, + "by_diff_twinunmoved", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_twinunmoved as m\n\ + print(hasattr(m.Loose, 'bare'), hasattr(m.Tagged, 'raised'))\n\ + print(type(m.bare) is m.Loose, type(m.raised) is m.Tagged)\n\ + print(type(m.Loose.__init__).__name__, type(m.Tagged.tag).__name__)\n", + ); + assert_eq!( + out, + "False False\n\ + False False\n\ + wrapper_descriptor method_descriptor" + ); +} + +/// an instance the move left behind is still a value the program holds, and compiled code +/// reading the name it is under has to answer for it +/// +/// the test above says such a value stays what the module body built. this says what +/// compiled code may then assume about it, which is nothing: `Form` extends a class python +/// allocates, so there is no field table to move `marker` through and the name goes on +/// holding an instance of the interpreted definition for the life of the module. narrowing +/// the global read to the emitted representation refused it — and refused it under a name +/// that prints the same on both sides, because the interpreted definition and the emitted +/// type share one +/// +/// `typing` is where this was found, and it broke the whole construct: `Annotated` is such +/// a value, `_get_typeddict_qualifiers` compares `annotation_origin is Annotated`, and so +/// every `TypedDict('T', {...})` against a compiled standard library raised +/// `expected _TypedCacheSpecialForm, got _TypedCacheSpecialForm` +#[test] +fn a_global_holding_an_instance_the_move_left_behind_is_read_as_an_object() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_globaltwin"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Form(Exception): + def label(self) -> str: + return \"form\" + + +marker = Form(\"m\") + + +def is_marker(value: object) -> bool: + return value is marker + + +def marker_label() -> str: + return marker.label() +"; + let built = match build_source( + source, + "by_diff_globaltwin", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + // `method_descriptor` is what says the compiled leg answered: both functions run + // natively, and both reach the global the emitted type never took over + let out = run( + &python, + &dir, + "import by_diff_globaltwin as m\n\ + print(type(m.marker) is m.Form)\n\ + print(m.is_marker(m.marker), m.is_marker(m.Form('other')))\n\ + print(m.marker_label())\n\ + print(type(m.Form.label).__name__)\n", + ); + assert_eq!( + out, + "False\n\ + True False\n\ + form\n\ + method_descriptor" + ); +} + +/// the source both halves of the extra-attribute move use +/// +/// `bump` is here for the reason [`A_CLASS_WITH_A_FIELD`] has it: without a method that +/// answers differently on the two legs, a test about a move passes just as well with the +/// class never compiled at all +const AN_INSTANCE_GIVEN_A_NAME_ITS_CLASS_NEVER_MENTIONED: &str = "\ +class Holder: + def __init__(self, tag: str) -> None: + self.tag = tag + + def shout(self) -> str: + return self.tag.upper() + + +class Other: + def label(self) -> str: + return \"other\" + + +held = Holder(\"one\") +held.extra = 7 +held.owner = Other + +alias = held +Holder.standing = held +"; + +/// an instance carrying a name its class never mentioned still moves onto the emitted type +/// +/// the move writes each of the layout's fields onto a fresh instance, and a name the +/// layout has none of used to refuse the whole move — so `held` went on standing on the +/// interpreted definition and `isinstance(held, Holder)` was False where python says True. +/// that is the silent wrong answer the move exists to prevent, and it was reachable from +/// two lines of ordinary python. +/// +/// an emitted class keeps a dict beside its layout wherever the source did not declare +/// `__slots__` throughout, and that dict is where `o.brand_new = 7` on a freshly built +/// emitted instance already goes. so the name has somewhere to be written after all, and +/// the refusal now applies only to a class whose instances have no dict at all. +#[test] +fn an_instance_carrying_a_name_the_layout_never_had_moves_with_it() { + // `extra` is a plain value and `owner` is one of this module's own classes, which has + // to come across as the type that replaced it — an extra reaches a twin exactly as a + // field does. `alias` and `Holder.standing` are the other two holders of the same + // object, and they have to answer it too. + // + // `vars` is asked for as a list of names rather than as a mapping, because the values + // include a class and the two legs spell a class's repr differently + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_twinextra"); + let _ = std::fs::remove_dir_all(&dir); + let built = match build_source( + AN_INSTANCE_GIVEN_A_NAME_ITS_CLASS_NEVER_MENTIONED, + "by_diff_twinextra", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + // the compiled leg answered with the emitted type, which is what makes the agreement + // below mean anything: a class that fell back would hold the very object the body + // wrote and agree for the one reason that makes the comparison worthless + let out = run( + &python, + &dir, + "import by_diff_twinextra as m\n\ + print(type(m.Holder.shout).__name__, type(m.held) is m.Holder)\n", + ); + assert_eq!(out, "method_descriptor True"); + agree_python( + "twinextra2", + AN_INSTANCE_GIVEN_A_NAME_ITS_CLASS_NEVER_MENTIONED, + &[ + "isinstance(m.held, m.Holder)", + "type(m.held) is m.Holder", + "(m.held.tag, m.held.extra, m.held.shout())", + "m.held.owner is m.Other", + "list(vars(m.held))", + "vars(m.held)['extra']", + "(m.alias is m.held, m.Holder.standing is m.held)", + "isinstance(m.Holder.standing, m.Holder)", + ], + ); +} + +/// the standard library's own spelling of it: a class of constants, blanked one name at a +/// time on an instance the module body built +/// +/// `_colorize` writes this, and it is what the refusal cost in practice — every attribute +/// on `NoColors` is a name the class never assigned in a method, so the whole instance was +/// turned down and `isinstance(NoColors, ANSIColors)` answered False. +/// +/// the class has no fields at all, which is the shape the extras are the *whole* of: there +/// is nothing in the layout to write, and the move is entirely the dict beside it. +#[test] +fn an_instance_blanked_by_a_setattr_loop_still_answers_its_own_class() { + agree_python( + "twinblanked", + "\ +class Palette: + RED = \"r\" + GREEN = \"g\" + + +blank = Palette() + +for name in dir(blank): + if not name.startswith(\"__\"): + setattr(blank, name, \"\") +", + &[ + "(isinstance(m.blank, m.Palette), type(m.blank) is m.Palette)", + "(m.blank.RED, m.blank.GREEN)", + "(m.Palette().RED, m.Palette().GREEN)", + "sorted(vars(m.blank))", + ], + ); +} + +/// a class body's own constant is the earliest a move can be asked for, and it is in time +/// +/// the constants are copied while their class is being built, which is before the pass +/// that walks the module namespace — so `class Bag: first = Seed("body")` reaches the move +/// earlier than a `Bag.second = later` written after the statement does. what the move +/// needs by then is the emitted type of `Seed`, and the types are filled one class at a +/// time as each is built. +/// +/// it is in time because a class body can only name a class already defined above it: the +/// body runs where the `class` statement stands, so a constant naming an instance of a +/// class further down would have raised `NameError` at import in the first place. this +/// pins the tightest spelling of that — the class immediately above, read in the very +/// first statement of the next class's body. +#[test] +fn a_class_body_constant_holding_an_instance_moves_with_the_class_above_it() { + agree_python( + "twinbodyorder", + "\ +class Seed: + def __init__(self, tag: str) -> None: + self.tag = tag + + +class Bag: + first = Seed(\"body\") -bare = Loose.__new__(Loose) -Loose.bare = bare -raised = Tagged(\"boom\") -Tagged.raised = raised -"; - let built = match build_source( - source, - "by_diff_twinunmoved", - &toolchain, - &dir, - &Options { - language: by_irbuild::Language::Python, - ..Options::default() - }, - ) { - Ok(built) => built, - Err(error) => { - assert!(missing_toolchain(&error), "failed to build: {error:#}"); - eprintln!("skipping: no working C toolchain ({error})"); - return; - } - }; - assert!(built.declined.is_empty(), "declined: {:?}", built.declined); - let out = run( - &python, - &dir, - "import by_diff_twinunmoved as m\n\ - print(hasattr(m.Loose, 'spare'), hasattr(m.Loose, 'bare'),\n\ - \x20 hasattr(m.Tagged, 'raised'))\n\ - print(type(m.spare) is m.Loose, type(m.bare) is m.Loose,\n\ - \x20 type(m.raised) is m.Tagged)\n\ - print(type(m.Loose.__init__).__name__, type(m.Tagged.tag).__name__)\n", - ); - assert_eq!( - out, - "False False False\n\ - False False False\n\ - wrapper_descriptor method_descriptor" +later = Seed(\"gift\") +Bag.second = later +", + &[ + "(isinstance(m.Bag.first, m.Seed), type(m.Bag.first) is m.Seed)", + "m.Bag.first.tag", + "(isinstance(m.Bag.second, m.Seed), type(m.Bag.second) is m.Seed)", + "(m.Bag.second is m.later, m.Bag.second.tag)", + "vars(m.Bag.first)", + ], ); } @@ -15528,52 +16861,147 @@ Blank.nothing = nothing ); } -/// a module-level *function* has an interpreted twin too, and it is deliberately left -/// where it stands +/// a move that fails partway takes the moves begun inside it down as well /// -/// the same staleness a class has: the module body runs against the interpreted -/// definitions, so everything it captured holds the `def`'s own function object, while -/// `PyModule_AddFunctions` puts the compiled `PyCFunction` under the name at the end of -/// init. `ALIAS is fn` is then False where python says True. +/// a move registers itself before its fields are filled, which is what lets a cyclic graph +/// resolve to one object rather than to two copies of it. the cost is a window: while the +/// fields are being filled the instance is reachable, and a graph member that leads back +/// to it puts *that* half-written object into a field of its own. an instance dropped +/// after that has not gone anywhere. +/// +/// this is what that used to produce, and it is the worst answer on the ladder: `r.peer` +/// answered an emitted `Node` with only its first field written, `r.peer is a` was False +/// where python says True, and reading the unwritten field raised +/// `SystemError: error return without exception set`. +#[test] +fn a_move_that_fails_partway_does_not_leave_a_half_written_instance_standing() { + // `spare` cannot move: it was built through `__new__`, so the field the layout treats + // as always defined was never written. that makes `a` fail at its *second* field — + // `load`, whose setter refuses the interpreted `Load` the twin holds — and `a`'s first + // field is what reaches `r`, which moves completely and takes `a`'s half-written + // instance into `peer` on the way. + // + // so `a` and `r` are refused together, and the two legs go on agreeing about identity + // and about what every attribute reads back. what they still disagree about is which + // class the two objects answer: that is the module-level instance defect itself, and + // it is asserted here rather than left out, because a test that only checked the + // compiled leg ran is how the half-written instance survived in the first place + let Some((python, toolchain)) = environment() else { + return; + }; + let compiled = diff_root().join("by_diff_twinhalf_c"); + let interpreted = diff_root().join("by_diff_twinhalf_i"); + let _ = std::fs::remove_dir_all(&compiled); + let _ = std::fs::remove_dir_all(&interpreted); + let source = "\ +class Load: + def __init__(self) -> None: + self.n = 0 + + +class Node: + def __init__(self, load: Load) -> None: + self.peer: object = None + self.load = load + + +class Ring: + def __init__(self) -> None: + self.peer: object = None + + +spare = Load.__new__(Load) + +a = Node(spare) +r = Ring() +a.peer = r +r.peer = a +"; + let built = match build_source( + source, + "by_diff_twinhalf", + &toolchain, + &compiled, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + std::fs::create_dir_all(&interpreted).expect("the directory is created"); + std::fs::write(interpreted.join("by_diff_twinhalf.py"), source) + .expect("the interpreted module is written"); + + // what both legs have to answer the same way: the graph is one object each way round, + // and every field reads back the value the body put there + let shared = "import by_diff_twinhalf as m\n\ + print(m.r.peer is m.a, m.a.peer is m.r)\n\ + print(type(m.r.peer.load).__name__, m.a.load is m.spare)\n\ + print(type(m.r).__name__, type(m.a).__name__)\n"; + let agreed = "True True\n\ + Load True\n\ + Ring Node"; + assert_eq!(run(&python, &compiled, shared), agreed); + assert_eq!(run(&python, &interpreted, shared), agreed); + + // and what they still do not: neither object was moved, so both go on standing on the + // interpreted definition their class no longer publishes + let classes = "import by_diff_twinhalf as m\n\ + print(type(m.a) is m.Node, type(m.r) is m.Ring)\n\ + print(type(m.Node.__init__).__name__)\n"; + assert_eq!( + run(&python, &compiled, classes), + "False False\nwrapper_descriptor" + ); + assert_eq!(run(&python, &interpreted, classes), "True True\nfunction"); +} + +/// a module-level *function* has an interpreted twin too, and it is moved onto the +/// forwarder that replaced it — so what a class body captured still binds /// -/// the class fix does not transfer, and this pins that it has not been made to. a class -/// twin is *incompatible* with the type that replaced it — `isinstance` denies it and a -/// compiled method refuses its instances — so a reference still holding one is already -/// broken, and moving it repairs damage. a function twin is **interchangeable** with the -/// compiled function for every use except identity: it computes the same answer, and -/// nothing rejects it. so moving one repairs nothing that was broken and breaks two -/// things that were not: +/// the same staleness a class has: the module body runs against the interpreted +/// definitions, so everything it captured holds the `def`'s own function object while the +/// name it was read from goes on to answer something else. `ALIAS is fn` was False where +/// python says True. /// -/// * a `function` in a class dict binds `self` and a `PyCFunction` does not, so the -/// moved reference stops being a method. `optparse` writes `class Option: __repr__ = -/// _repr` and `multiprocessing.reduction` writes `class AbstractReducer: dump = dump` -/// * `inspect.signature` works on a `function` and raises `ValueError` on a -/// `PyCFunction`, so a captured callback stops being introspectable +/// this used to be left alone, and the reason was that the module published a +/// `PyCFunction` under its function names. moving a captured reference onto one broke two +/// things that were not broken: a `function` in a class dict binds `self` and a +/// `PyCFunction` does not, so the reference stopped being a method — `optparse` writes +/// `class Option: __repr__ = _repr` and `multiprocessing.reduction` writes `class +/// AbstractReducer: dump = dump`; and `inspect.signature` works on a `function` and +/// raised `ValueError` on a `PyCFunction`. both turned a right answer into a raise. /// -/// both turn a *right* answer into a raise, and a wrong answer is better than a crash. -/// over the stdlib corpus the captured references are overwhelmingly dispatch tables — -/// `copy._deepcopy_dispatch`, `shutil._ARCHIVE_FORMATS`, `xml.etree.ElementTree._serialize` -/// — whose entries are only ever *called*, so the divergence is unobservable there while -/// the repair would be plainly observable. +/// the forwarder dissolved both objections. what a module publishes under a function name +/// is now a real `function` — see `by_irbuild::shims` — so it binds like one and +/// `inspect` reads its signature through `__wrapped__`. the substitution is therefore +/// made, and this test is what says the binding survived it: the class answers the same +/// value either way, and `function` in the slot is the only thing that says so. /// -/// the two questions a remap would also have had to answer turn out to be answered -/// already, and neither needs runtime machinery: a function whose module-level name the -/// body rebinds is not `exported`, so an accelerator import (`asyncio.events` keeping -/// `_py_get_event_loop`, `operator`'s trailing `from _operator import *`) never produces -/// a twin at all; and a *decorated* module-level definition the module reads already -/// declines, because its decorator cannot run where the `def` stands and again over the -/// compiled one +/// what is still left alone is a *decorated* definition. the twin's source has the +/// decorators taken out of it, so what the body bound an alias to is not what the +/// interpreted module would have bound it to either, and pairing it with the undecorated +/// forwarder swaps one wrong answer for another. a function whose module-level name the +/// body rebinds is not `exported` at all — an accelerator import (`asyncio.events` +/// keeping `_py_get_event_loop`, `operator`'s trailing `from _operator import *`) — so it +/// never produces a twin to move #[test] fn a_class_attribute_naming_a_module_function_keeps_the_definition_that_binds() { // `multiprocessing.reduction` writes `class AbstractReducer: dump = dump`, and this is // that: a class body binding a name to a module-level function that goes on to - // compile. the value the emitted type carries is the *twin*, and it has to be — a - // `PyCFunction` in a class dict is not a descriptor, so `Reducer().dump()` would call - // `_dump` with no `self` at all. + // compile. what the emitted type carries is the forwarder, and `function` in the slot + // is what says that is still a descriptor — a `PyCFunction` there would call `_dump` + // with no `self` at all. // - // `function` against the forwarder is the whole assertion. the class answers the - // same *value* either way, so nothing but what sits in the slot says which - // definition is standing there + // the class answers the same *value* either way, so nothing but what sits in the slot + // says which of the two objects is standing there let Some((python, toolchain)) = environment() else { return; }; @@ -15615,26 +17043,32 @@ class Reducer: _leg = lambda f: 'native' if f.__code__.co_filename == '' else type(f).__name__\n\ print(m.Reducer().dump(), m.Reducer().kind())\n\ print(_leg(m._dump), type(m.Reducer.__dict__['dump']).__name__)\n\ - print(type(m.Reducer.kind).__name__)\n", + print(type(m.Reducer.kind).__name__)\n\ + print(m.Reducer.__dict__['dump'] is m._dump)\n", ); // `method_descriptor` says the emitted type answered rather than a class that fell // back to its interpreted definition — which would carry the slot for the other reason + // + // the last line is what says *which* function is in the slot. it is the only + // observable that tells the two apart at all, which is why the divergence it now + // closes survived: `dump` answered correctly whichever one was there assert_eq!( out, "dumped reducer\n\ native function\n\ - method_descriptor" + method_descriptor\n\ + True" ); } /// the same, for the slot a *declined* class keeps and for a dunder /// /// `optparse` writes `class Option: __repr__ = _repr`, and over the corpus that is a -/// captured twin — the compiled `optparse` keeps `Option` interpreted and its `__repr__` -/// holds the definition the module's own `_repr` no longer names. the alias remap walks -/// exactly that dict, so it is the second route a function substitution would take into a -/// descriptor position, and `repr()` is where it would show: python looks a dunder up on -/// the type and calls what it finds, and what it finds has to bind +/// captured definition — the compiled `optparse` keeps `Option` interpreted and its +/// `__repr__` held the object the module's own `_repr` no longer named. the alias remap +/// walks exactly that dict, so it is the second route the substitution takes into a +/// descriptor position, and `repr()` is where a substitution that stopped binding would +/// show: python looks a dunder up on the type and calls what it finds #[test] fn a_declined_class_keeps_the_dunder_slot_a_module_function_filled() { let Some((python, toolchain)) = environment() else { @@ -15691,26 +17125,28 @@ Option.__ge__ = lambda self, other: True _leg = lambda f: 'native' if f.__code__.co_filename == '' else type(f).__name__\n\ print(repr(m.Option()), m.Option().kind())\n\ print(_leg(m._repr), type(m.Option.__dict__['__repr__']).__name__)\n\ - print(type(m.Option.kind).__name__)\n", + print(type(m.Option.kind).__name__)\n\ + print(m.Option.__dict__['__repr__'] is m._repr)\n", ); - // `function` on the last line is the class itself confirming it stayed interpreted, + // `function` on the third line is the class itself confirming it stayed interpreted, // which is the only state in which this slot exists to be got wrong assert_eq!( out, "