diff --git a/.gitignore b/.gitignore index f75e68c745..cc450645e1 100644 --- a/.gitignore +++ b/.gitignore @@ -227,3 +227,6 @@ cython_debug/ !crates/ruff_python_resolver/resources/test/airflow/venv/lib !crates/ruff_python_resolver/resources/test/airflow/venv/lib/python3.11/site-packages/_watchdog_fsevents.cpython-311-darwin.so !crates/ruff_python_resolver/resources/test/airflow/venv/lib/python3.11/site-packages/orjson/orjson.cpython-311-darwin.so + +# a patch backup, never source +*.orig diff --git a/crates/by_build/src/lib.rs b/crates/by_build/src/lib.rs index dcc8fb89b8..923b5da622 100644 --- a/crates/by_build/src/lib.rs +++ b/crates/by_build/src/lib.rs @@ -300,6 +300,18 @@ fn finish( // and the same program compiled, so that importing the artefact does not have to // parse it all over again. it is asked for after every rewrite above, because what // gets compiled has to be exactly what would otherwise be run + // a compiled module publishes a real `function` under each of its own function + // names rather than the native object, because a `PyCFunction` is not a + // descriptor and so never receives a receiver when it is installed on a class. + // the forwarders are python, and this is where python gets compiled — so they + // are written into the twin before it is handed to the interpreter + let twin = match by_irbuild::shims::shims(&module, &twin) { + Some(shims) => { + module.shims = Some(shims.install); + format!("{twin}{}", shims.source) + } + None => twin, + }; module.fallback_code = toolchain.and_then(|toolchain| toolchain.marshal(&twin)); module.fallback_source = Some(twin); Ok(module) @@ -509,6 +521,7 @@ mod tests { lines: None, fallback_source: None, fallback_code: None, + shims: None, }; let dir = std::env::temp_dir().join("by_build_refuses_test"); let _ = fs::remove_dir_all(&dir); diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index ab759798c2..4f7c3490b3 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -212,6 +212,48 @@ def _blamed_module(module, fn, name=_no_name): finally: module.__dict__['__name__'] = saved +# a chain of interpreted frames for a warning to be blamed on, `depth` of them between +# the caller and the call +# +# a warning above the default stack level blames a frame further out than the function +# that wrote it, and the frames out there have to be *these*: one in the module under +# test would be a frame the interpreted leg pushes and the compiled leg does not, which +# is a question about the missing frame rather than about the walk +def _under(depth, fn, *args): + if depth > 0: + return _under(depth - 1, fn, *args) + return fn(*args) + +# a frame `warn` walks past rather than blames +# +# it steps over every frame whose file name holds both 'importlib' and '_bootstrap', so +# a warning raised under the import machinery is blamed on whoever asked for the import. +# the file name is the whole of that rule, so a function compiled under one stands in +# for the loader here +_machinery = {} +exec(compile('def hop(fn, *args):\\n return fn(*args)\\n', + 'importlib/_bootstrap.py', 'exec'), _machinery) +_walked_past = _machinery['hop'] + +# how often a repeated warning is shown, and what the blamed module recorded +# +# the registry that suppresses a repeat belongs to the frame the warning was blamed on, +# so above the default level it is *this* module's rather than the one under test's. a +# lowering that wrote the registry it knows instead of the one it walked to would keep +# printing +def _registry_of_the_blamed(fn, times=3): + globals().pop('__warningregistry__', None) + with warnings.catch_warnings(record=True) as seen: + warnings.resetwarnings() + warnings.simplefilter('default') + for _ in range(times): + fn() + shown = len(seen) + recorded = sorted(key[0] for key in globals().get('__warningregistry__', {}) + if isinstance(key, tuple)) + globals().pop('__warningregistry__', None) + return (shown, recorded) + # the registry carries a version, and changing the filters invalidates it: a warning # already shown is shown again rather than stayed silent about def _registry_after_a_filter_change(module): @@ -2449,7 +2491,7 @@ def raw_ender() -> bytes: #[test] fn the_compiled_build_is_the_one_that_answers_for_a_nul_literal() { // `agree` cannot say which build answered — a declined function answers the same. - // `builtin_function_or_method` is what says the compiled leg is under these calls + // the forwarder's own code object is what says the compiled leg is under these calls let Some((python, toolchain)) = environment() else { return; }; @@ -2474,9 +2516,10 @@ fn the_compiled_build_is_the_one_that_answers_for_a_nul_literal() { &python, &dir, "import by_diff_nulstr_which as m\n\ - print(type(m.whole).__name__, m.length(), m.equals_prefix(), ascii(m.whole()))\n", + _leg = lambda f: 'native' if f.__code__.co_filename == '' else type(f).__name__\n\ + print(_leg(m.whole), m.length(), m.equals_prefix(), ascii(m.whole()))\n", ); - assert_eq!(out, "builtin_function_or_method 3 False 'a\\x00b'"); + assert_eq!(out, "native 3 False 'a\\x00b'"); } #[test] @@ -3178,6 +3221,93 @@ def shout_all(items: list) -> object: ); } +/// `split`, `startswith`, `join` and `upper` are called through their own C-API +/// entry points rather than through the method, so every argument shape that entry +/// point does *not* serve has to be handed back to the method +/// +/// each of them is asked with the shape it takes directly and with every shape it +/// refuses, in the same process and through the same call site: a separator that is +/// `None` or a `str` subclass or not a string at all, a `maxsplit`, a `startswith` +/// given a range or a tuple, a receiver that is a `str` subclass or not a string, +/// and — for `upper`, which is the one that repeats the interpreter's work rather +/// than calling it — a string above ascii, where the case mapping can change the +/// length of the answer +#[test] +fn a_string_method_taken_directly_answers_what_the_method_answers() { + agree_python( + "strdirect", + "\ +def split_all(line) -> object: + return line.split() + +def split_on(line, sep) -> object: + return line.split(sep) + +def split_capped(line, sep, limit) -> object: + return line.split(sep, limit) + +def leads(s, prefix) -> object: + return s.startswith(prefix) + +def leads_from(s, prefix, start) -> object: + return s.startswith(prefix, start) + +def leads_within(s, prefix, start, end) -> object: + return s.startswith(prefix, start, end) + +def glued(sep, parts) -> object: + return sep.join(parts) + +def shout(s) -> object: + return s.upper() +", + &[ + // the separator shapes: absent, a string, `None`, a subclass, and the + // empty string python refuses + "[m.split_on(line, ' ') for line in ('', 'a', 'a b c', ' a b ', 'a b ')]", + "[m.split_all(line) for line in ('', ' ', 'a b c', ' a\\tb\\nc ')]", + "[m.split_on('a b c', None), m.split_all('a b c')]", + "m.split_on('a-b', type('S', (str,), {})('-'))", + "[(type(e).__name__, str(e)) for e in [_capture(m.split_on, 'ab', '')]]", + "[(type(e).__name__, str(e)) for e in [_capture(m.split_on, 'ab', 5)]]", + // a maxsplit is a second argument, which the direct call does not serve + "[m.split_capped('a b c d', ' ', n) for n in (-1, 0, 1, 2, 9)]", + // a receiver that is not an exact `str` + "m.split_on(type('S', (str,), {'split': lambda self, sep: ['Z']})('a b'), ' ')", + "m.split_on(b'a b', b' ')", + "m.split_all(b'a b')", + // `startswith`: a string prefix, a subclass, a tuple, a range, a refusal + "[m.leads(s, 'w') for s in ('', 'w', 'word', 'xw')]", + "[m.leads('word', p) for p in ('', 'word', 'words', ('x', 'w'), ('x',), ())]", + "m.leads('word', type('S', (str,), {})('w'))", + "[m.leads_from('xword', 'w', n) for n in (0, 1, 2, -4, 99)]", + "[m.leads_within('xword', 'wo', 1, n) for n in (1, 2, 3, 99)]", + "[(type(e).__name__, str(e)) for e in [_capture(m.leads, 'word', 5)]]", + "m.leads(type('S', (str,), {'startswith': lambda self, p: 'Z'})('ab'), 'a')", + "m.leads(b'ab', b'a')", + // `join` over the sequences and iterables python accepts, and the two + // errors it raises + "[m.glued(s, p) for s in ('', '-', '::') for p in ([], ['a'], ['a', 'b'], ('c', 'd'))]", + "[m.glued('-', iter(['a', 'b'])), m.glued('-', 'abc')]", + "[(type(e).__name__, str(e)) for e in [_capture(m.glued, '-', [1, 2]), _capture(m.glued, '-', 5)]]", + "m.glued(type('S', (str,), {'join': lambda self, p: 'Z'})('-'), ['a'])", + "m.glued(b'-', [b'a', b'b'])", + // `upper` over ascii, where the direct path answers, and above it, where + // it does not — `ß` uppercases to two characters and `ff` to three + "[m.shout(s) for s in ('', 'a', 'abc', 'ABC', 'a1-b_c', '\\x7f', '\\x00')]", + "[m.shout(s) for s in ('é', '🎉', 'ß', 'ff', 'aß', 'ıi')]", + "[type(m.shout('ab')).__name__, m.shout('') == '', len(m.shout('abc'))]", + "m.shout(type('S', (str,), {'upper': lambda self: 'Z'})('ab'))", + "m.shout(type('P', (str,), {})('ab'))", + "m.shout(b'ab')", + // the same site asked for an exact string, then a subclass, then an exact + // string again — the direct path has to be taken, given up and retaken + "[m.shout(s) for s in ('ab', type('S', (str,), {'upper': lambda self: 'Z'})('cd'), 'ef')]", + "[m.split_on(o, ' ') for o in ('a b', type('S', (str,), {})('c d'), 'e f')]", + ], + ); +} + /// the other way a remembered answer stops being right: the method is rebound on /// the type after the site was armed /// @@ -4195,14 +4325,15 @@ class Held: &python, &dir, "import by_diff_pathdecolive as m\n\ - print(type(m.cached).__name__, type(m.cached.__wrapped__).__name__)\n\ + _leg = lambda f: 'native' if f.__code__.co_filename == '' else type(f).__name__\n\ + print(type(m.cached).__name__, _leg(m.cached.__wrapped__))\n\ print(type(m.Marks.area).__name__, type(m.Marks.sized).__name__,\n\ \x20 type(m.Held.read).__name__)\n\ print(m.cached(4), m.Marks.area.__isabstractmethod__, m.Held.tag)\n", ); assert_eq!( out, - "_lru_cache_wrapper builtin_function_or_method\n\ + "_lru_cache_wrapper native\n\ function method_descriptor method_descriptor\n\ 8 True seen" ); @@ -4361,17 +4492,18 @@ class Rooted: assert_eq!(out, "7 property method_descriptor 3"); } -/// a class whose type slots publish more than its body wrote keeps its decorator's -/// decline +/// a decorator that fills in the comparisons a class left out is handed the gaps it +/// expects /// -/// python reaches `<=` through `tp_richcompare`, one slot behind all six comparisons — -/// so an emitted type that writes `__lt__` publishes `__le__` as well, answering -/// `NotImplemented`. `functools.total_ordering` reads exactly that: it saw `__le__` -/// already there, filled in nothing, and `a <= b` raised where the interpreted class -/// answered `True`. that was a live wrong answer for the plain-name spelling before the -/// path spelling could reach it at all -#[test] -fn a_class_decorator_over_a_partly_filled_slot_declines() { +/// python reaches `<=` through `tp_richcompare`, one slot behind all six comparisons, and +/// publishes a wrapper for every name a filled slot backs — so an emitted type that +/// writes `__lt__` used to publish `__le__` as well. `functools.total_ordering` reads +/// exactly those names: it saw `__le__` already there, filled in nothing, and `a <= b` +/// raised where the interpreted class answered `True`. the class declined its whole +/// compilation to stay out of that. now that the type publishes only what the body wrote, +/// the decorator finds the three gaps, fills them, and the class compiles +#[test] +fn a_class_decorator_over_a_partly_filled_slot_fills_it_in() { let Some((python, toolchain)) = environment() else { return; }; @@ -4409,22 +4541,145 @@ class Ranked: return; } }; - assert!( - built - .declined - .iter() - .any(|declined| declined.reason.contains("publishes `__le__`")), - "declined: {:?}", - built.declined - ); + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + // `wrapper_descriptor` rather than `function` is the whole point: the class compiled + // and `tp_init` is a real slot, where it used to hand its whole definition back to + // the interpreter let out = run( &python, &dir, "import by_diff_partialslot as m\n\ - print(m.Ranked(1) <= m.Ranked(2), m.Ranked(3) > m.Ranked(2))\n\ + print(m.Ranked(1) <= m.Ranked(2), m.Ranked(3) > m.Ranked(2), m.Ranked(1) != m.Ranked(1))\n\ print(type(m.Ranked.__init__).__name__)\n", ); - assert_eq!(out, "True True\nfunction"); + assert_eq!(out, "True True False\nwrapper_descriptor"); +} + +/// the same decorator applied from *another* module refuses out loud +/// +/// nothing in this module says the class will be decorated, so its type is the sealed one +/// every undecorated class gets, and `total_ordering`'s `setattr` cannot land on it. what +/// matters is *when* that is said. while the type published all six comparisons the +/// decorator found nothing missing, set nothing, raised nothing — and the first `<=` +/// raised instead, a long way from the decoration that caused it. publishing only the +/// body's own names puts the refusal back where the decision is made +#[test] +fn a_class_decorator_applied_from_another_module_refuses_at_decoration() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_outsideslot"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Ordered: + def __init__(self, n: int) -> None: + self.n = n + + def __eq__(self, other: object) -> bool: + return isinstance(other, Ordered) and self.n == other.n + + def __lt__(self, other: object) -> bool: + return self.n < other.n +"; + let built = match build_source( + source, + "by_diff_outsideslot", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import functools, by_diff_outsideslot as m\n\ + try:\n\ + \x20 functools.total_ordering(m.Ordered)\n\ + \x20 print('applied')\n\ + except TypeError as error:\n\ + \x20 print(type(error).__name__, 'immutable' in str(error))\n", + ); + assert_eq!(out, "TypeError True"); +} + +/// a class that declines for its decorator takes no other class down with it +/// +/// a written decorator has to be applied at module init rather than where the `class` +/// statement stands, so a class the module body goes on running below declines. that was +/// settled at the end of the class's lowering — by which point it had a layout, and every +/// other class had been lowered against it. `Holder` then declined too, for a layout that +/// was no longer there, and anything holding a `Holder` after it. +/// +/// nothing showed while a decorated class with a half-filled slot group declined earlier, +/// where the layout is decided, and got there first. with that decline gone, `tracemalloc` +/// fell from 50 compiled functions to 26 — 24 of them to this cascade, for a class that +/// was always going to decline +#[test] +fn a_class_declining_for_its_decorator_leaves_no_layout_behind() { + let Some((_, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_decorlayout"); + let _ = std::fs::remove_dir_all(&dir); + // `Holder` names a base, so evaluating its header reaches the module — which is what + // leaves `Held` visible, undecorated, in the window a moved decorator opens + let source = "\ +from collections.abc import Sequence + + +def mark(cls): + return cls + + +@mark +class Held: + def __init__(self, n): + self._n = n + + +class Holder(Sequence): + def __init__(self, ns): + self._ns = ns + + def __len__(self): + return len(self._ns) + + def __getitem__(self, index): + return Held(self._ns[index]) +"; + let built = match build_source( + source, + "by_diff_decorlayout", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + let declined: Vec<_> = built + .declined + .iter() + .map(|declined| declined.name.as_str()) + .collect(); + assert_eq!(declined, ["Held"], "declined: {:?}", built.declined); } #[test] @@ -5349,9 +5604,9 @@ def replaces_itself() -> int: fn a_compiled_frame_is_what_reaches_the_module_namespace() { // the differential tests above compare two legs, and a leg that fell back to its // interpreted definition answers exactly as the interpreted leg does — so they - // cannot say *which* build wrote the global. this one can: a module-level function - // python calls through `PyModule_AddFunctions` is a `builtin_function_or_method`, - // and one that fell back is a `function` + // cannot say *which* build wrote the global. this one can: what a compiled module + // publishes is a forwarder onto the native, and its code object says so where a + // definition that fell back names the module's own file let Some((python, toolchain)) = environment() else { return; }; @@ -5395,12 +5650,13 @@ def init() -> None: &python, &dir, "import by_diff_globalidentity as m\n\ - print(type(m.init).__name__, m.C().x, m.inited)\n", + _leg = lambda f: 'native' if f.__code__.co_filename == '' else type(f).__name__\n\ + print(_leg(m.init), m.C().x, m.inited)\n", ); // and `m.inited` read from out here is the module's own binding, which a register // write never touched. before there was an op for it, `C()` inside `init` saw the // old `False` and called `init` again until the stack ran out - assert_eq!(out, "builtin_function_or_method 1 True"); + assert_eq!(out, "native 1 True"); } #[test] @@ -5470,10 +5726,11 @@ def declines_and_reads() -> str: &python, &dir, "import by_diff_globaltwin as m\n\ - print(type(m.writes).__name__, type(m.declines_and_reads).__name__,\n\ + _leg = lambda f: 'native' if f.__code__.co_filename == '' else type(f).__name__\n\ + print(_leg(m.writes), _leg(m.declines_and_reads),\n\ \x20 m.declines_and_reads(), m.writes(42), m.declines_and_reads(), m.flag)\n", ); - assert_eq!(out, "builtin_function_or_method function 0 42 42 42"); + assert_eq!(out, "native function 0 42 42 42"); } #[test] @@ -5720,19 +5977,26 @@ data class Point: return; } // a declared field is the *layout*: a descriptor on the type, read at an offset, - // never an entry in an instance dict. and `__dict__` itself is refused however the - // instance is built — a mapping naming only what the layout has no room for would - // be an empty answer where the interpreted class gives a full one, which is quiet - // and wrong where the refusal is at least loud + // never an entry in an instance dict. `__dict__` still names it, because python's + // `__dict__` is one mapping over the whole of an object's state — a mapping naming + // only what the layout has no room for would be an empty answer where the + // interpreted class gives a full one, which is quiet and wrong let out = run( &python, &dir, "import by_diff_layout as m\n\ p = m.Point(1, 2)\n\ print(type(vars(m.Point)['x']).__name__)\n\ - print(hasattr(p, '__dict__'))\n", + print(hasattr(p, '__dict__'), vars(p))\n\ + p.extra = 3\n\ + print(vars(p), p.__dict__['x'])\n", + ); + assert_eq!( + out, + "getset_descriptor\n\ + True {'x': 1, 'y': 2}\n\ + {'x': 1, 'y': 2, 'extra': 3} 1" ); - assert_eq!(out, "getset_descriptor\nFalse"); } /// the source both of the instance-dict tests build @@ -6303,6 +6567,213 @@ def took(obj, name): ); } +/// a class keeping its state in a layout, an attribute put on an instance from outside +/// it, and a name the class body binds beside a field its `__init__` writes on only one +/// path — which between them are every kind of entry an instance's `__dict__` can hold +const A_CLASS_WHOSE_DICT_IS_READ: &str = "\ +class Record: + tag = 'none' + + def __init__(self, msg): + self.msg = msg + if msg: + self.tag = 'set' + + def rename(self, msg): + self.msg = msg + + +def render(r): + return '%(msg)s/%(tag)s' % r.__dict__ + + +def state(r): + return sorted(r.__dict__.items(), key=str) +"; + +#[test] +fn an_emitted_instance_agrees_with_its_twin_about_its_dict() { + // an emitted instance keeps its attributes in two places — the class's own in the + // layout and anything put on it afterwards in the dict beside them — so the dict + // alone names the *extra* attributes and none of the real ones. answering `__dict__` + // with it would be an empty mapping where the interpreted class gives a full one, and + // the refusal that stood here instead broke a compiled `logging`: `Formatter.format` + // reads `record.__dict__`, and the read declines to a function that is then handed an + // object with no `__dict__` at all. a decline protects the function, not the class + agree_python_with_declines( + "instdict", + A_CLASS_WHOSE_DICT_IS_READ, + &[ + "m.render(m.Record('hello'))", + "m.state(m.Record('hello'))", + // the class body's value is not the instance's, so `__dict__` does not name it + "m.state(m.Record(''))", + "m.Record('a').__dict__['msg']", + "'msg' in m.Record('a').__dict__", + "'nothing' in m.Record('a').__dict__", + "len(vars(m.Record('a')))", + "sorted(vars(m.Record('a')).keys())", + "sorted(vars(m.Record('a')).values(), key=str)", + "vars(m.Record('a')).get('msg')", + "vars(m.Record('a')).get('nothing', 'fallback')", + "vars(m.Record('a')) == {'msg': 'a', 'tag': 'set'}", + "dict(vars(m.Record('a')))", + "{**vars(m.Record('a'))}", + "repr(vars(m.Record('a')))", + "'{msg}'.format_map(vars(m.Record('a')))", + "[k for k in vars(m.Record('a'))]", + // a name put on the instance from outside the class joins the same mapping + "(lambda r: [setattr(r, 'extra', 3), m.state(r), r.__dict__['extra']])(m.Record('a'))", + // and a write *through* the mapping reaches the layout, which is the whole + // point of it being a view rather than a copy + "(lambda r: [r.__dict__.__setitem__('msg', 'bye'), r.msg, m.state(r)])(m.Record('a'))", + "(lambda r: [r.__dict__.update({'msg': 'z', 'fresh': 1}), r.msg, r.fresh])(m.Record('a'))", + "(lambda r: [r.__dict__.pop('tag'), m.state(r), r.tag])(m.Record('a'))", + "(lambda r: [r.__dict__.setdefault('msg', 'no'), r.__dict__.setdefault('new', 5), r.new])(m.Record('a'))", + "(lambda r: [r.__dict__.__delitem__('tag'), m.state(r)])(m.Record('a'))", + // and replacing the whole mapping replaces the whole of the object's state + "(lambda r: [setattr(r, '__dict__', {'msg': 'fresh', 'other': 2}), m.state(r), r.msg, r.other])(m.Record('a'))", + "vars(m.Record('a')).copy()", + "type(vars(m.Record('a')).copy()).__name__", + // and it has to *be* a dict, not merely read like one. `isinstance(x, dict)` + // gates a great deal of library code, and every reader the C api offers reads + // the base's own storage and ignores an override — a mapping answering out of + // a side table serialises as `{}`, silently + "isinstance(vars(m.Record('a')), dict)", + "__import__('json').dumps(vars(m.Record('a')))", + "vars(m.Record('a')).keys()", + // a write through the mapping has to reach the storage as well as the + // object, or every reader that goes straight to the storage answers with + // what stood there before + "(lambda d: [d.__setitem__('msg', 'bye'), __import__('json').dumps(d)])(vars(m.Record('a')))", + "(lambda r: [r.__dict__.popitem(), m.state(r)])(m.Record('a'))", + "(lambda r: [r.__dict__.__ior__({'msg': 'or'}), r.msg])(m.Record('a'))", + // the type's own name is the one thing left that differs, so the refusal is + // asked for by its kind rather than by its wording + "[type(e).__name__ for e in [_capture(hash, vars(m.Record('a')))]]", + // and a mapping taken and *held* has to see what the object does next. + // answering with what stood there when it was handed out is a wrong answer + // nothing marks — no exception, no missing key, just the old value + "(lambda r: (lambda d: [setattr(r, 'msg', 'three'), d['msg'], d.get('msg'), sorted(d.items(), key=str)])(r.__dict__))(m.Record('two'))", + "(lambda r: (lambda d: [r.rename('three'), d['msg'], sorted(d.items(), key=str)])(r.__dict__))(m.Record('two'))", + "(lambda r: (lambda d: [setattr(r, 'extra', 3), sorted(d.items(), key=str)])(r.__dict__))(m.Record('two'))", + // including through every reader that goes straight to the storage + "(lambda r: (lambda d: [setattr(r, 'msg', 'three'), __import__('json').dumps(d)])(r.__dict__))(m.Record('two'))", + // asked twice it is the same mapping, which is what makes the one somebody + // holds the one the object goes on writing to + "(lambda r: r.__dict__ is r.__dict__)(m.Record('a'))", + "(lambda r: [setattr(r, 'extra', 3), r.__dict__ is vars(r)])(m.Record('a'))", + // a mapping outliving the object it stood for is an ordinary dict holding the + // state that stood at the end, and writing to it reaches nothing + "sorted((lambda r: r.__dict__)(m.Record('two')).items(), key=str)", + "(lambda d: [d.__setitem__('msg', 'after'), sorted(d.items(), key=str)])((lambda r: r.__dict__)(m.Record('two')))", + "(lambda d: [d.update({'late': 1}), sorted(d.items(), key=str)])((lambda r: r.__dict__)(m.Record('two')))", + // and a field the object gives up leaves the mapping with it + "(lambda r: (lambda d: [delattr(r, 'tag'), sorted(d.items(), key=str), 'tag' in d])(r.__dict__))(m.Record('a'))", + ], + ); +} + +#[test] +fn the_class_that_answered_the_dict_is_the_compiled_one() { + // the legs agree whichever class answered, so this is where the compiled one is + // pinned: with the codegen path off, `Record` is the interpreted definition and every + // assertion above passes for the wrong reason + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_instdictlive"); + let _ = std::fs::remove_dir_all(&dir); + if build_source( + A_CLASS_WHOSE_DICT_IS_READ, + "by_diff_instdictlive", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + "import by_diff_instdictlive as m\n\ + r = m.Record('a')\n\ + print(type(m.Record.__init__).__name__, hasattr(r, '__dict__'))\n\ + print(type(r).__dict__['msg'].__class__.__name__)\n\ + print(m.render(r))\n", + ); + // `msg` reaching python through a `getset_descriptor` is what says the attribute is + // the *layout's* and not an entry in a dict — an interpreted `Record` has no such + // descriptor at all + assert_eq!( + out, + "wrapper_descriptor True\n\ + getset_descriptor\n\ + a/set" + ); +} + +#[test] +fn an_emitted_instance_agrees_with_its_twin_about_its_state() { + // `object.__getstate__` reads the dict word straight out of the instance, and on an + // emitted one that word holds the *extra* attributes and none of the class's own — so + // whatever asks an object for its state was handed half of it, and handed it quietly. + // that is the same hole as the `__dict__` one, through a door a decline never reaches + agree_python_with_declines( + "inststate", + A_CLASS_WHOSE_DICT_IS_READ, + &[ + "m.Record('hello').__getstate__()", + "m.Record('').__getstate__()", + "(lambda r: [setattr(r, 'extra', 3), r.__getstate__()])(m.Record('a'))", + ], + ); +} + +#[test] +fn a_formatter_reading_a_record_dict_agrees() { + // the shape that motivated the view, reduced to the two classes `logging` uses: a + // record whose state is its layout, and a formatter in the same module that reads + // `record.__dict__` off a parameter it knows nothing about. the read declines, so the + // formatter runs interpreted — against the *emitted* record, which is exactly the + // case a decline cannot cover on its own + agree_python_with_declines( + "recorddict", + "\ +class LogRecord: + def __init__(self, name, level, message): + self.name = name + self.levelname = level + self.message = message + + +class Style: + def __init__(self, fmt): + self.fmt = fmt + + def format(self, record): + return self.fmt % record.__dict__ + + +def emit(fmt, name, level, message): + return Style(fmt).format(LogRecord(name, level, message)) +", + &[ + "m.emit('%(levelname)s:%(name)s:%(message)s', 'root', 'WARNING', 'hello')", + "m.emit('%(message)s', 'x', 'INFO', 'quiet')", + // the extra a caller attaches to a record, which `logging` puts there through + // the mapping itself + "(lambda r: [r.__dict__.__setitem__('zz', 1), '%(name)s %(zz)s' % r.__dict__])(m.LogRecord('n', 'W', 'm'))", + ], + ); +} + #[test] fn a_native_class_instance_does_not_leak() { let Some((python, toolchain)) = environment() else { @@ -6427,9 +6898,11 @@ fn a_compiled_function_is_a_c_function_object() { let out = run( &python, &dir, - "import by_diff_cfunc as m\nprint(type(m.f).__name__)\n", + "import by_diff_cfunc as m\n\ + _leg = lambda f: 'native' if f.__code__.co_filename == '' else type(f).__name__\n\ + print(type(m.f).__name__, _leg(m.f))\n", ); - assert_eq!(out, "builtin_function_or_method"); + assert_eq!(out, "function native"); } #[test] @@ -7369,14 +7842,12 @@ fn the_shadowed_calls_are_answered_by_compiled_bodies() { &python, &dir, "import by_diff_aloneshadowkind as m\n\ + _leg = lambda f: 'native' if f.__code__.co_filename == '' else type(f).__name__\n\ print(type(m.Alone.double).__name__)\n\ print(type(m.Slotted.double).__name__)\n\ - print(type(m.alone).__name__)\n", - ); - assert_eq!( - out, - "method_descriptor\nmethod_descriptor\nbuiltin_function_or_method" + print(_leg(m.alone))\n", ); + assert_eq!(out, "method_descriptor\nmethod_descriptor\nnative"); } /// a builtin whose entry point wants the defining class, which no call site can supply @@ -7757,6 +8228,40 @@ class Loud(Quiet): ); } +#[test] +fn a_class_a_helper_takes_a_weak_reference_of_is_declined() { + // `logging.Handler.__init__`'s shape: it writes no `weakref.ref` at all, it calls + // `_addHandlerRef(self)` and the reference is taken a whole function away. nothing + // about `__init__`'s own body says it raises, so the refusal has to travel back along + // the call — and if it does not, constructing the class raises `TypeError` rather + // than answering + agree_python_with_declines( + "weakhelper", + "\ +import weakref + +registry = [] + + +def registers(thing): + registry.append(weakref.ref(thing)) + + +def forwards(thing): + registers(thing) + + +class Handler: + def __init__(self): + forwards(self) + + def alive(self): + return registry[-1]() is self +", + &["m.Handler().alive()", "len(m.registry)"], + ); +} + #[test] fn a_basedpython_default_that_is_not_an_immediate_is_re_evaluated_at_each_call() { // basedpython has no mutable-default gotcha: `mutable_defaults` rewrites such a @@ -9129,15 +9634,13 @@ async def finishing(v: object) -> object: &dir, "import asyncio\n\ import by_diff_sendslot_pin as m\n\ - print(type(m.counting).__name__)\n\ + _leg = lambda f: 'native' if f.__code__.co_filename == '' else type(f).__name__\n\ + print(_leg(m.counting))\n\ print(type(m.counting(0)).__name__)\n\ print(asyncio.run(m.finishing((1, 2))))\n", ); // a declined function would be a plain `function` and its state a `generator` - assert_eq!( - out, "builtin_function_or_method\ncounting$gen\n(1, 2)", - "{out}" - ); + assert_eq!(out, "native\ncounting$gen\n(1, 2)", "{out}"); } /// a `StopIteration` the body *raised* leaves the frame as a `RuntimeError`, and one @@ -10372,56 +10875,128 @@ def raised(fn: object) -> str: ); } -/// what the type publishes under a property's name is a `property` -/// -/// this is the half no behavioural comparison can reach. an attribute published through -/// `tp_getset` reads and writes exactly as a property does, so every test that only -/// *uses* the attribute passes either way — while `C.value.fget` raises, and so do -/// `.fset`, `.getter(...)`, `.setter(...)` and `isinstance(C.value, property)`. so this -/// asks the type what it holds rather than asking an instance what it answers. +/// a property reached from *compiled* code answers as the descriptor would /// -/// the half inside it is what says which leg ran: a `method_descriptor` is the compiled -/// one, and an interpreted fallback would hold a `function` there -#[test] -fn a_property_is_published_as_a_property_object() { - let Some((python, toolchain)) = environment() else { - return; - }; - let dir = diff_root().join("by_diff_propobject"); - let _ = std::fs::remove_dir_all(&dir); - let source = "\ +/// every other property comparison here reads the attribute from python, which goes +/// through the `property` object whatever the compiler did. a compiled frame with a typed +/// receiver calls the half outright instead, and these are the answers that route has to +/// keep giving: the setter's body runs rather than a store landing beside it, the getter's +/// body runs rather than the field it reads being taken directly, and a property with no +/// setter still refuses the write in python's own wording +#[test] +fn a_property_reached_from_compiled_code_agrees() { + agree_python( + "propdirect", + "\ class Box: def __init__(self, n: int) -> None: self._n = n @property def value(self) -> int: - return self._n + return self._n * 10 @value.setter def value(self, given: int) -> None: - self._n = given -"; - let built = match build_source( - source, - "by_diff_propobject", - &toolchain, - &dir, - &Options { - language: by_irbuild::Language::Python, - ..Options::default() - }, - ) { - Ok(built) => built, - Err(error) => { - assert!(missing_toolchain(&error), "failed to build: {error:#}"); - eprintln!("skipping: no working C toolchain ({error})"); - return; - } - }; - assert!(built.declined.is_empty(), "declined: {:?}", built.declined); - let out = run( - &python, + self._n = given + 1 + + +class Reading: + def __init__(self, n: int) -> None: + self._n = n + + @property + def value(self) -> int: + return self._n + + @value.deleter + def value(self) -> None: + self._n = 0 + + +def through(box: Box, given: int) -> int: + box.value = given + return box.value + + +def underneath(box: Box) -> int: + return box._n + + +def store(reading: Reading) -> None: + reading.value = 1 + + +def refused(reading: Reading) -> str: + try: + store(reading) + except AttributeError as error: + return str(error) + return 'nothing raised' +", + &[ + // the setter added one and the getter multiplied by ten, so neither half was + // skipped on the way through + "m.through(m.Box(0), 4)", + "m.underneath(m.Box(0))", + "(lambda b: (m.through(b, 7), m.underneath(b)))(m.Box(0))", + "m.refused(m.Reading(3))", + // and the refusal left the field the getter reads alone + "(lambda r: (m.refused(r), r.value))(m.Reading(3))", + ], + ); +} + +/// what the type publishes under a property's name is a `property` +/// +/// this is the half no behavioural comparison can reach. an attribute published through +/// `tp_getset` reads and writes exactly as a property does, so every test that only +/// *uses* the attribute passes either way — while `C.value.fget` raises, and so do +/// `.fset`, `.getter(...)`, `.setter(...)` and `isinstance(C.value, property)`. so this +/// asks the type what it holds rather than asking an instance what it answers. +/// +/// the half inside it is what says which leg ran: a `method_descriptor` is the compiled +/// one, and an interpreted fallback would hold a `function` there +#[test] +fn a_property_is_published_as_a_property_object() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_propobject"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Box: + def __init__(self, n: int) -> None: + self._n = n + + @property + def value(self) -> int: + return self._n + + @value.setter + def value(self, given: int) -> None: + self._n = given +"; + let built = match build_source( + source, + "by_diff_propobject", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, &dir, "import by_diff_propobject as m\n\ p = m.Box.__dict__['value']\n\ @@ -10445,6 +11020,135 @@ class Box: ); } +/// a `@property` with nothing written under it is published over the compiled body +/// +/// a group of one is the same construct with two of its three halves absent, and python +/// folds it into a `property` exactly as it folds a pair. what makes it worth publishing +/// is the half inside: `@property` is a decorator the class body already applied, so left +/// to the ordinary method path the type is given the object *that* body left — a +/// `property` holding the interpreted function, written over the method table's entry, so +/// the compiled body is emitted and then never reached. +/// +/// `method_descriptor` for `fget` is what says the published one won. it is also what says +/// the twin's own `property` did not land beside it: the adoption that carries a twin's +/// attributes over skips a name the type already holds, and this is that name +#[test] +fn a_lone_property_getter_is_published_over_the_compiled_body() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_proplone"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Box: + def __init__(self, n: int) -> None: + self._n = n + + @property + def value(self) -> int: + return self._n * 10 +"; + let built = match build_source( + source, + "by_diff_proplone", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_proplone as m\n\ + p = m.Box.__dict__['value']\n\ + print(type(p).__name__, isinstance(p, property), p.fset, p.fdel)\n\ + print(type(p.fget).__name__, p.fget.__name__, p.fget.__qualname__)\n\ + # the getter is reached through the property and under no name of its own\n\ + print(m.Box(4).value, hasattr(m.Box, 'value$get'))\n", + ); + assert_eq!( + out, + "property True None None\n\ + method_descriptor value Box.value\n\ + 40 False" + ); +} + +/// a `@property` with no setter answers and refuses exactly as the interpreted one does +/// +/// the group of one is published rather than left to the class body's own object, so +/// every answer it gives is now the compiled type's: the read runs the getter's body, and +/// the write and the `del` are refused by the `property` itself — in python's own wording, +/// which names the property, so it is also what says `__set_name__` reached it. +/// +/// `refused` asks the same question from a *compiled* frame with a typed receiver, which +/// calls the half outright instead of going round the descriptor. that route has its own +/// way of getting a missing half wrong: a write with nowhere to go could land beside the +/// property instead of raising. +/// +/// none of it says *which* leg answered, and it cannot: an interpreted fallback answers a +/// property exactly as the published one does, so every line here passes with the group of +/// one left unlowered. the test above is what says the compiled body is the one running +#[test] +fn a_lone_property_getter_agrees() { + agree_python( + "proplonediff", + "\ +class Box: + def __init__(self, n: int) -> None: + self._n = n + + @property + def value(self) -> int: + return self._n * 10 + + +def read(box: Box) -> int: + return box.value + + +def store(box: Box) -> None: + box.value = 1 + + +def refused(box: Box) -> str: + try: + store(box) + except AttributeError as error: + return str(error) + return 'nothing raised' + + +def raised(fn: object) -> str: + try: + fn() + except AttributeError as error: + return str(error) + return 'nothing raised' +", + &[ + "m.Box(3).value", + "m.read(m.Box(3))", + "m.raised(lambda: setattr(m.Box(1), 'value', 2))", + "m.raised(lambda: delattr(m.Box(1), 'value'))", + "m.refused(m.Box(3))", + // and the refusal left the field the getter reads alone rather than + // dropping a value beside the property + "(lambda b: (m.refused(b), b.value, b._n))(m.Box(3))", + ], + ); +} + /// a class whose type is a static struct is published to the same way /// /// nearly every emitted class is built from a type spec, and a property could have been @@ -10679,7 +11383,7 @@ class Wide: return 5 @value.setter - def value(self, given: int, extra: int = 1) -> None: + def value(self, given: int, extra: int) -> None: pass "; let built = match build_source( @@ -12287,16 +12991,17 @@ fn only_the_class_no_spec_can_build_is_left_interpreted() { &python, &dir, "import by_diff_perclassheld_t as m\n\ + _leg = lambda f: 'native' if f.__code__.co_filename == '' else type(f).__name__\n\ print(type(m.Held.__dict__['read']).__name__,\n\ \x20 type(m.Kept.__dict__['note']).__name__,\n\ \x20 type(m.Deeper.__dict__['down']).__name__,\n\ - \x20 type(m.add).__name__)\n\ + \x20 _leg(m.add))\n\ # a spec has no code object to write one from, so its absence is the emitted type\n\ print('__firstlineno__' in vars(m.Held), '__firstlineno__' in vars(m.Kept))\n", ); assert_eq!( out, - "function method_descriptor method_descriptor builtin_function_or_method\n\ + "function method_descriptor method_descriptor native\n\ True False" ); } @@ -14863,9 +15568,9 @@ fn a_class_attribute_naming_a_module_function_keeps_the_definition_that_binds() // `PyCFunction` in a class dict is not a descriptor, so `Reducer().dump()` would call // `_dump` with no `self` at all. // - // `function` against `builtin_function_or_method` is the whole assertion. the class - // answers the same *value* either way, so nothing but the type of what sits in the - // slot says which definition is standing there + // `function` against the forwarder is the whole assertion. the class answers the + // same *value* either way, so nothing but what sits in the slot says which + // definition is standing there let Some((python, toolchain)) = environment() else { return; }; @@ -14904,8 +15609,9 @@ class Reducer: &python, &dir, "import by_diff_fntwinbind as m\n\ + _leg = lambda f: 'native' if f.__code__.co_filename == '' else type(f).__name__\n\ print(m.Reducer().dump(), m.Reducer().kind())\n\ - print(type(m._dump).__name__, type(m.Reducer.__dict__['dump']).__name__)\n\ + print(_leg(m._dump), type(m.Reducer.__dict__['dump']).__name__)\n\ print(type(m.Reducer.kind).__name__)\n", ); // `method_descriptor` says the emitted type answered rather than a class that fell @@ -14913,7 +15619,7 @@ class Reducer: assert_eq!( out, "dumped reducer\n\ - builtin_function_or_method function\n\ + native function\n\ method_descriptor" ); } @@ -14979,8 +15685,9 @@ Option.__ge__ = lambda self, other: True &python, &dir, "import by_diff_fntwindunder as m\n\ + _leg = lambda f: 'native' if f.__code__.co_filename == '' else type(f).__name__\n\ print(repr(m.Option()), m.Option().kind())\n\ - print(type(m._repr).__name__, type(m.Option.__dict__['__repr__']).__name__)\n\ + print(_leg(m._repr), type(m.Option.__dict__['__repr__']).__name__)\n\ print(type(m.Option.kind).__name__)\n", ); // `function` on the last line is the class itself confirming it stayed interpreted, @@ -14988,7 +15695,7 @@ Option.__ge__ = lambda self, other: True assert_eq!( out, "