Extension module work

.gitignore

@@ -227,3 +227,6 @@ cython_debug/
 !crates/ruff_python_resolver/resources/test/airflow/venv/lib
 !crates/ruff_python_resolver/resources/test/airflow/venv/lib/python3.11/site-packages/_watchdog_fsevents.cpython-311-darwin.so
 !crates/ruff_python_resolver/resources/test/airflow/venv/lib/python3.11/site-packages/orjson/orjson.cpython-311-darwin.so
+
+# a patch backup, never source
+*.orig

crates/by_build/src/lib.rs

@@ -300,6 +300,18 @@ fn finish(
     // and the same program compiled, so that importing the artefact does not have to
     // parse it all over again. it is asked for after every rewrite above, because what
     // gets compiled has to be exactly what would otherwise be run
+    // a compiled module publishes a real `function` under each of its own function
+    // names rather than the native object, because a `PyCFunction` is not a
+    // descriptor and so never receives a receiver when it is installed on a class.
+    // the forwarders are python, and this is where python gets compiled — so they
+    // are written into the twin before it is handed to the interpreter
+    let twin = match by_irbuild::shims::shims(&module, &twin) {
+        Some(shims) => {
+            module.shims = Some(shims.install);
+            format!("{twin}{}", shims.source)
+        }
+        None => twin,
+    };
     module.fallback_code = toolchain.and_then(|toolchain| toolchain.marshal(&twin));
     module.fallback_source = Some(twin);
     Ok(module)
@@ -509,6 +521,7 @@ mod tests {
             lines: None,
             fallback_source: None,
             fallback_code: None,
+            shims: None,
         };
         let dir = std::env::temp_dir().join("by_build_refuses_test");
         let _ = fs::remove_dir_all(&dir);

crates/by_build/tests/end_to_end.rs

@@ -148,6 +148,7 @@ fn arith_module() -> ModuleIr {
         lines: None,
         fallback_source: None,
         fallback_code: None,
+        shims: None,
     }
 }
 
@@ -213,6 +214,7 @@ fn fib_module() -> ModuleIr {
         lines: None,
         fallback_source: None,
         fallback_code: None,
+        shims: None,
     }
 }
 
@@ -350,6 +352,7 @@ fn division_floors_like_python_and_raises_on_zero() {
         lines: None,
         fallback_source: None,
         fallback_code: None,
+        shims: None,
     };
     let Some(dir) = built(&module, &toolchain, "divzero") else {
         return;
@@ -407,6 +410,7 @@ fn floats_are_unboxed_and_exclude_int() {
         lines: None,
         fallback_source: None,
         fallback_code: None,
+        shims: None,
     };
     let Some(dir) = built(&module, &toolchain, "float") else {
         return;
@@ -479,6 +483,7 @@ fn calls_between_compiled_functions_stay_native() {
         lines: None,
         fallback_source: None,
         fallback_code: None,
+        shims: None,
     };
     let Some(dir) = built(&module, &toolchain, "call") else {
         return;
@@ -514,6 +519,45 @@ fn repeated_calls_do_not_leak_the_boxed_representation() {
     assert_eq!(out, "stable");
 }
 
+#[test]
+fn a_raising_body_releases_the_registers_it_never_wrote() {
+    let Some((python, toolchain)) = environment() else {
+        return;
+    };
+    // every refcounted register starts holding the int error sentinel, and the block
+    // that runs on the way out releases all of them — so a body that raises early
+    // hands the sentinel to `By_DecRefTagged` several times over. it is skipped
+    // because the sentinel is the tag bit alone and `By_LongOf` turns it into NULL;
+    // release it as if it were an object and the process dies on the first raise
+    let Some(dir) = built_from_source(
+        "\
+def chain(a: int, b: int) -> int:
+    first = a * b
+    second = first // b
+    third = second + first
+    return third
+",
+        "by_e2e_sentinel",
+        &toolchain,
+        "sentinel",
+    ) else {
+        return;
+    };
+
+    let out = script(
+        &python,
+        &dir,
+        "import by_e2e_sentinel\n\
+         print(by_e2e_sentinel.chain(6, 3))\n\
+         try:\n    by_e2e_sentinel.chain(6, 0)\n\
+         except ZeroDivisionError:\n    print('caught')\n\
+         else:\n    print('no error')\n",
+    );
+    // the first line proves the body ran compiled at all; the second proves the
+    // release block survived three unwritten registers
+    assert_eq!(out, "24\ncaught");
+}
+
 #[test]
 fn a_returned_value_outlives_the_frame_that_made_it() {
     let Some((python, toolchain)) = environment() else {
@@ -1459,6 +1503,75 @@ else:
     assert_eq!(asserted(&["-O"]), "False");
 }
 
+#[test]
+fn running_the_interpreter_with_oo_takes_the_docstrings_out_of_the_compiled_definitions() {
+    // a docstring is baked into the method table at build time, but the level the
+    // artefact is *run* at is only known at import. under `-OO` python compiles a
+    // function with no docstring at all, so the interpreted twin of this very source
+    // answers `None` — and a compiled definition still holding the baked-in string would
+    // disagree with its own twin, in the same process
+    let Some((python, toolchain)) = environment() else {
+        return;
+    };
+    let dir = e2e_root().join("by_e2e_docoptrun");
+    let _ = std::fs::remove_dir_all(&dir);
+    let source = "\
+def plain():
+    \"the plain one\"
+    return 1
+
+
+class Described:
+    def described(self):
+        \"on a method\"
+        return 2
+";
+    let Ok(built) = build_source(
+        source,
+        "by_e2e_docoptrun",
+        &toolchain,
+        &dir,
+        &Options {
+            language: by_irbuild::Language::Python,
+            ..Options::default()
+        },
+    ) else {
+        eprintln!("skipping: no working C toolchain");
+        return;
+    };
+    assert!(built.declined.is_empty(), "declined: {:?}", built.declined);
+    let docs = |flags: &[&str]| {
+        let mut command = Command::new(&python);
+        command.args(flags).args([
+            "-c",
+            &format!(
+                "import sys\nsys.path.insert(0, {:?})\n\
+                 import by_e2e_docoptrun as m\n\
+                 _leg = lambda f: 'native' if f.__code__.co_filename == '<by native forwarder>' else type(f).__name__\n\
+                 print(_leg(m.plain),\n\
+                 \x20     type(m.Described.__dict__['described']).__name__,\n\
+                 \x20     repr(m.plain.__doc__), repr(m.Described.described.__doc__))\n",
+                dir.display().to_string()
+            ),
+        ]);
+        let out = command.output().expect("the interpreter runs");
+        assert!(
+            out.status.success(),
+            "{}",
+            String::from_utf8_lossy(&out.stderr)
+        );
+        String::from_utf8_lossy(&out.stdout).trim().to_string()
+    };
+    // the kinds are in every line so that a leg which fell back to its interpreted
+    // definition cannot answer for one of these: it would agree about `None` under `-OO`
+    // for a reason that says nothing about the table
+    let kept = "native method_descriptor 'the plain one' 'on a method'";
+    assert_eq!(docs(&[]), kept);
+    // `-O` takes out `assert`, and only `-OO` takes out the docstrings as well
+    assert_eq!(docs(&["-O"]), kept);
+    assert_eq!(docs(&["-OO"]), "native method_descriptor None None");
+}
+
 #[test]
 fn compiling_the_twin_twice_gives_the_same_bytes() {
     // the emitted C has to be a function of the source alone, or a rebuild recompiles a