Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
553 changes: 233 additions & 320 deletions Cargo.lock

Large diffs are not rendered by default.

20 changes: 10 additions & 10 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,20 +16,20 @@ clap = { version = "4.5.20", features = ["derive"] }
tar = "0.4.42"
tempfile = "3.13.0"
zstd = "0.13.2"
wasm-encoder = { version = "0.245.0", features = ["wasmparser"] }
wit-dylib = "0.245.0"
wit-parser = "0.245.0"
wit-component = "0.245.0"
wasmparser = "0.245.0"
wasm-encoder = { version = "0.258.0", features = ["wasmparser"] }
wit-dylib = "0.258.0"
wit-parser = "0.258.0"
wit-component = "0.258.0"
wasmparser = "0.258.0"
indexmap = "2.6.0"
bincode = "1.3.3"
heck = "0.5.0"
pyo3 = { version = "0.28.3", features = [
"abi3-py39",
"extension-module",
], optional = true }
wasmtime = "46.0.1"
wasmtime-wasi = { version = "46.0.1", features = [ "p3" ] }
wasmtime = "48.0.0"
wasmtime-wasi = { version = "48.0.0", features = [ "p3" ] }
once_cell = "1.20.2"
component-init-transform = "0.2.0"
async-trait = "0.1.83"
Expand All @@ -47,7 +47,7 @@ im-rc = "15.1.0"
serde = { version = "1.0.213", features = ["derive"] }
toml = "0.8.19"
semver = "1.0.23"
wit-bindgen-core = "0.53.1"
wit-bindgen-core = "0.61.0"

[dev-dependencies]
assert_cmd = "2.0.16"
Expand Down Expand Up @@ -81,8 +81,8 @@ tar = "0.4.41"
zstd = "0.13.2"
componentize-py-test-generator = { path = "test-generator", version = "0.25.0" }
flate2 = "1.1.1"
wasmparser = "0.245.0"
wasm-encoder = "0.245.0"
wasmparser = "0.258.0"
wasm-encoder = "0.258.0"

[workspace]
package.version = "0.25.0"
Expand Down
4 changes: 3 additions & 1 deletion build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -808,9 +808,11 @@ fn strip(input: &[u8]) -> anyhow::Result<Vec<u8>> {
}

if let Some((id, range)) = payload.as_section() {
let start = usize::try_from(range.start).unwrap();
let end = usize::try_from(range.end).unwrap();
RawSection {
id,
data: &input[range],
data: &input[start..end],
}
.append_to(&mut output);
}
Expand Down
5 changes: 2 additions & 3 deletions runtime/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,8 @@ anyhow = "1.0.91"
once_cell = "1.20.2"
pyo3 = { version="0.28.3", features = ["abi3-py312", "num-bigint"] }
num-bigint = "0.4.6"
wit-bindgen = { version = "0.52.0", default-features = false, features = ["macros", "realloc"] }
# TODO: switch to a release when available:
wit-dylib-ffi = { git = "https://github.com/dicej/wasm-tools", rev = "b072b0ca", features = ["async-raw"] }
wit-bindgen = { version = "0.61.0", default-features = false, features = ["macros", "realloc"] }
wit-dylib-ffi = { git = "https://github.com/bytecodealliance/wasm-tools", tag = "v1.258.0", features = ["async-raw"] }

[features]
async = []
70 changes: 61 additions & 9 deletions runtime/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,9 @@ use {
exceptions::PyAssertionError,
intern,
types::{
PyAnyMethods, PyBool, PyBytes, PyBytesMethods, PyDict, PyList, PyListMethods,
PyMapping, PyMappingMethods, PyModule, PyModuleMethods, PyString, PyTuple,
PyAnyMethods, PyBool, PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyList,
PyListMethods, PyMapping, PyMappingMethods, PyModule, PyModuleMethods, PyString,
PyTuple,
},
},
std::{
Expand All @@ -36,7 +37,7 @@ use {
sync::Once,
},
wit_dylib_ffi::{
self as wit, Call, ExportFunction, Interpreter, List, Type, Wit, WitOption, WitResult,
self as wit, Call, ExportFunction, Interpreter, List, Map, Type, Wit, WitOption, WitResult,
},
};

Expand Down Expand Up @@ -816,10 +817,7 @@ mod async_ {

ty.write()(handle, buffer.cast())
};
let resources = call
.resources
.take()
.and_then(|v| if v.is_empty() { None } else { Some(v) });
let resources = call.resources.take().filter(|v| !v.is_empty());

Ok(if code == RETURN_CODE_BLOCKED {
ERR_CONSTRUCTOR
Expand Down Expand Up @@ -1638,6 +1636,7 @@ impl Call for MyCall<'_> {
.bind(py)
.get_item(value.get_type())
.unwrap()
.unwrap()
.extract::<usize>()
.unwrap();

Expand Down Expand Up @@ -1705,7 +1704,7 @@ impl Call for MyCall<'_> {
})
}

fn pop_iter_next(&mut self, _ty: List) {
fn pop_list_iter_next(&mut self, _ty: List) {
Python::attach(|py| {
let index = *self.iter_stack.last().unwrap();
let element = self
Expand All @@ -1721,7 +1720,40 @@ impl Call for MyCall<'_> {
})
}

fn pop_iter(&mut self, _ty: List) {
fn pop_list_iter(&mut self, _ty: List) {
self.iter_stack.pop().unwrap();
Python::attach(|py| {
self.stack.pop().unwrap().drop_ref(py);
})
}

fn pop_map(&mut self, _ty: Map) -> usize {
Python::attach(|py| {
self.iter_stack.push(0);
let value = self.stack.pop().unwrap();
let value = value.cast_bound::<PyDict>(py).unwrap();
let length = value.len();
self.stack.push(value.items().into_any().unbind());
length
})
}

fn pop_map_iter_next(&mut self, _ty: Map) {
Python::attach(|py| {
let index = *self.iter_stack.last().unwrap();
let items = self.stack.last().unwrap();
let items = items.cast_bound::<PyList>(py).unwrap();
let element = items.get_item(index).unwrap();
let element = element.cast::<PyTuple>().unwrap();
*self.iter_stack.last_mut().unwrap() = index + 1;
self.stack
.push(element.get_item(1).unwrap().into_any().unbind());
self.stack
.push(element.get_item(0).unwrap().into_any().unbind());
})
}

fn pop_map_iter(&mut self, _ty: Map) {
self.iter_stack.pop().unwrap();
Python::attach(|py| {
self.stack.pop().unwrap().drop_ref(py);
Expand Down Expand Up @@ -2076,6 +2108,26 @@ impl Call for MyCall<'_> {
.unwrap()
});
}

fn push_map(&mut self, _ty: Map, _capacity: usize) {
self.stack.push(Python::attach(|py| {
PyDict::new(py).to_owned().into_any().unbind()
}));
}

fn map_append(&mut self, _ty: Map) {
Python::attach(|py| {
let value = self.stack.pop().unwrap();
let key = self.stack.pop().unwrap();
self.stack
.last()
.unwrap()
.cast_bound::<PyDict>(py)
.unwrap()
.set_item(key, value)
.unwrap()
});
}
}

fn imported_resource_from_canon(py: Python<'_>, ty: wit::Resource, handle: u32) -> Py<PyAny> {
Expand Down
17 changes: 6 additions & 11 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ use {
component::{Component, Instance, Linker, ResourceTable, ResourceType},
},
wasmtime_wasi::{
DirPerms, FilePerms, WasiCtx, WasiCtxBuilder, WasiCtxView, WasiView,
FsPerms, WasiCtx, WasiCtxBuilder, WasiCtxView, WasiView,
p2::pipe::{MemoryInputPipe, MemoryOutputPipe},
},
wit_component::metadata,
Expand Down Expand Up @@ -559,6 +559,7 @@ impl ComponentGenerator<'_> {
&resolve,
world,
Some(&mut DylibOpts {
stack_pointer: wit_dylib::StackPointer::Global,
interpreter: Some("libcomponentize_py_runtime.so".into()),
async_: Default::default(),
}),
Expand Down Expand Up @@ -680,19 +681,13 @@ impl ComponentGenerator<'_> {
.preopened_dir(
embedded_python_standard_lib.path(),
"python",
DirPerms::all(),
FilePerms::all(),
FsPerms::ReadWrite,
)?
.preopened_dir(
embedded_helper_utils.path(),
"bundled",
DirPerms::all(),
FilePerms::all(),
)?;
.preopened_dir(embedded_helper_utils.path(), "bundled", FsPerms::ReadWrite)?;

// Generate guest mounts for each host directory in `python_path`.
for (index, path) in python_path.iter().enumerate() {
wasi.preopened_dir(path, index.to_string(), DirPerms::all(), FilePerms::all())?;
wasi.preopened_dir(path, index.to_string(), FsPerms::ReadWrite)?;
}

// For each Python module with a `componentize-py.toml` file that
Expand Down Expand Up @@ -785,7 +780,7 @@ impl ComponentGenerator<'_> {
world_dir.path().display()
);
}
wasi.preopened_dir(world_dir.path(), mount, DirPerms::all(), FilePerms::all())?;
wasi.preopened_dir(world_dir.path(), mount, FsPerms::ReadWrite)?;
}
}

Expand Down
9 changes: 4 additions & 5 deletions src/link.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,20 +5,19 @@ use anyhow::Result;
use crate::Library;

pub fn link_libraries(libraries: &[Library]) -> Result<Vec<u8>> {
let mut linker = wit_component::Linker::default()
.validate(true)
.use_built_in_libdl(true);
let mut linker = wit_component::Linker::default();
linker.use_built_in_libdl(true).encoder().validate(true);

for Library {
name,
module,
dl_openable,
} in libraries
{
linker = linker.library(name, module, *dl_openable)?;
linker.library(name, module, *dl_openable)?;
}

linker = linker.adapter(
linker.encoder().adapter(
"wasi_snapshot_preview1",
&zstd::decode_all(Cursor::new(include_bytes!(concat!(
env!("OUT_DIR"),
Expand Down
11 changes: 6 additions & 5 deletions src/stubwasi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,8 @@ type LinkedStubModules = Option<(Vec<u8>, Box<dyn Fn(u32) -> u32>)>;

pub fn link_stub_modules(libraries: Vec<Library>) -> Result<LinkedStubModules, Error> {
let mut wasi_imports = HashMap::new();
let mut linker = wit_component::Linker::default()
.validate(true)
.use_built_in_libdl(true);
let mut linker = wit_component::Linker::default();
linker.use_built_in_libdl(true).encoder().validate(true);

for Library {
name,
Expand All @@ -24,11 +23,13 @@ pub fn link_stub_modules(libraries: Vec<Library>) -> Result<LinkedStubModules, E
} in &libraries
{
add_wasi_imports(module, &mut wasi_imports)?;
linker = linker.library(name, module, *dl_openable)?;
linker.library(name, module, *dl_openable)?;
}

for (module, imports) in &wasi_imports {
linker = linker.adapter(module, &make_stub_adapter(module, imports))?;
linker
.encoder()
.adapter(module, &make_stub_adapter(module, imports))?;
}

let component = linker.encode()?;
Expand Down
2 changes: 1 addition & 1 deletion src/summary.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1422,7 +1422,7 @@ impl<'a> Summary<'a> {
let mut future_payloads = HashSet::new();
let mut types = Types::default();
types.analyze(self.resolve);
types.collect_equal_types(self.resolve, &|_| true);
types.collect_equal_types(self.resolve, world, &|_| true);

for (index, id) in self.types.iter().copied().enumerate() {
if !self
Expand Down
4 changes: 2 additions & 2 deletions src/test/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ use {
StreamProducer, StreamReader, StreamResult, VecBuffer,
},
},
wasmtime_wasi::{DirPerms, FilePerms, WasiCtxBuilder, WasiView},
wasmtime_wasi::{FsPerms, WasiCtxBuilder, WasiView},
};

wasmtime::component::bindgen!({
Expand Down Expand Up @@ -972,7 +972,7 @@ fn filesystem() -> Result<()> {
let wasi = WasiCtxBuilder::new()
.inherit_stdout()
.inherit_stderr()
.preopened_dir(dir.path(), "/", DirPerms::all(), FilePerms::all())?
.preopened_dir(dir.path(), "/", FsPerms::ReadWrite)?
.build();

TESTER.test_with_wasi::<Host>(wasi, |world, store, runtime| {
Expand Down
Loading