diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f149c542..1942bd35 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,14 +56,23 @@ jobs: - name: Build run: mvn compile + # Runs both test tiers. The 'interop' tag is excluded from a bare `mvn test` so a + # contributor without a working uv/zarr-python setup can still run the suite locally; + # CI has Python, so it clears the exclusion and runs everything. + # + # Interop belongs on the pull-request path, not on a schedule: it catches bugs caused by + # the change under review -- a dtype or codec where zarr-java's reader and writer agree + # with each other but not with anyone else -- and that is feedback you want on the PR. + # It costs about 20s now that zarr-python runs as one worker process instead of one + # interpreter launch per test case. - name: Test env: MAVEN_OPTS: "-Xmx6g" run: | if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then - mvn --no-transfer-progress test -DargLine="-Xmx6g" -DrunS3Tests=true + mvn --no-transfer-progress test -DargLine="-Xmx6g" -DrunS3Tests=true -DexcludedTestGroups= else - mvn --no-transfer-progress test -DargLine="-Xmx6g" + mvn --no-transfer-progress test -DargLine="-Xmx6g" -DexcludedTestGroups= fi - name: Assemble JAR run: mvn package -DskipTests diff --git a/.github/workflows/data-type-drift.yml b/.github/workflows/data-type-drift.yml new file mode 100644 index 00000000..ccaa3199 --- /dev/null +++ b/.github/workflows/data-type-drift.yml @@ -0,0 +1,105 @@ +# Keeps src/test/resources/spec-data-types-v3.json honest. +# +# That file is the external conformance list DataTypeConformanceTest checks zarr-java against. +# It is generated from zarr-python's data type registry and committed, so the Java test stays +# offline and fast. Committing it buys reproducibility -- a given commit always tests against a +# known list -- at the cost of the list going stale. These two jobs pay that cost back. +# +# The split follows who caused the breakage: +# +# stale-check runs per pull request. Fails when the committed list disagrees with the +# zarr-python version this repository pins. That is caused by the change under +# review (a dependency bump without a regeneration, or a hand-edited file), so +# it belongs on the pull-request path. +# +# upstream-drift runs nightly. Fails when the *latest* zarr-python knows a data type our +# pinned version did not. No pull request caused that, so gating pull requests +# on it would fail innocent changes for reasons their authors cannot act on. +# As a scheduled job it becomes a task someone picks up: bump the pin, +# regenerate, and either implement the new data type or add it to +# KNOWN_UNSUPPORTED with a note. + +name: Data Type Conformance List + +on: + workflow_dispatch: + pull_request: + branches: [ "main" ] + push: + branches: [ "main" ] + schedule: + # 04:23 UTC daily. Off the hour so it does not queue behind everyone else's midnight cron. + - cron: '23 4 * * *' + +jobs: + stale-check: + name: Committed list matches the pinned zarr-python + if: github.event_name != 'schedule' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.11' + + - name: Install uv + uses: astral-sh/setup-uv@v6 + + # Pinned explicitly: uv.lock resolves zarr differently per Python version, so letting uv + # pick an interpreter would make this check depend on what happens to be installed. 3.11 is + # what the committed list was generated with and what ci.yml runs the interop tier on. + - name: Regenerate the conformance list + run: uv run --python 3.11 src/test/python-scripts/generate_spec_data_types.py + + - name: Fail if it differs from the committed copy + run: | + if ! git diff --exit-code src/test/resources/spec-data-types-v3.json; then + echo + echo "::error::spec-data-types-v3.json is out of date with the pinned zarr-python." + echo "Regenerate and commit it:" + echo " uv run src/test/python-scripts/generate_spec_data_types.py" + echo "If the diff adds a data type, either implement it in dev.zarr.zarrjava.v3.DataType" + echo "or add it to DataTypeConformanceTest.KNOWN_UNSUPPORTED with a note on what it needs." + exit 1 + fi + + upstream-drift: + name: Latest zarr-python has no data types we have not seen + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.11' + + - name: Install uv + uses: astral-sh/setup-uv@v6 + + # `uv run` alone would reuse uv.lock and regenerate a byte-identical file forever, which + # would make this job pass vacuously. Re-resolving zarr is the entire point: it is how a + # new upstream data type reaches us. + - name: Resolve the latest zarr-python + run: | + uv lock --upgrade-package zarr + uv run --python 3.11 python -c "import zarr; print('resolved zarr-python', zarr.__version__)" + + - name: Regenerate the conformance list against it + run: uv run --python 3.11 src/test/python-scripts/generate_spec_data_types.py + + - name: Report drift + run: | + if ! git diff --exit-code src/test/resources/spec-data-types-v3.json; then + echo + echo "::warning::A newer zarr-python describes data types differently than our pinned version." + echo "This is not a broken build -- it is a to-do. To act on it:" + echo " 1. bump the zarr pin in pyproject.toml and commit the updated uv.lock" + echo " 2. uv run src/test/python-scripts/generate_spec_data_types.py" + echo " 3. implement any new data type, or list it in KNOWN_UNSUPPORTED with a note" + exit 1 + fi + echo "No drift: the latest zarr-python matches the committed conformance list." diff --git a/README.md b/README.md index faf0cf2f..3d04ba45 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,36 @@ data ### Run Tests Locally -To be able to run the tests locally, make sure to have `python3.11` and `uv` installed. +The test suite is split into two tiers. + +**Fast tier (default).** Offline, no Python required, runs in seconds: + +``` +mvn test +``` + +It covers the metadata-only data type conformance checks, per-data-type round-trips, and the +committed golden fixtures. + +**Interop tier.** Cross-checks zarr-java against zarr-python, and needs `python3.11` and `uv` +installed. It is tagged `interop` and excluded from a bare `mvn test`, so that a contributor +without a working Python setup still gets a useful run. Clear the exclusion to include it: + +``` +mvn test -DexcludedTestGroups= # everything, both tiers +mvn test -DexcludedTestGroups= -Dtest=ZarrPythonTests # just the interop tier +``` + +**CI runs both tiers on every pull request** — the tag exists for local convenience, not to keep +these tests out of the build. They belong on the pull-request path because they catch bugs caused +by the change under review: a test that writes with zarr-java and reads back with zarr-java passes +even when the reader and writer share the same misunderstanding of the spec. The resulting store is +then wrong for every other tool, and only a second implementation can catch that. + +The interop tier is cheap enough for that. zarr-python runs as one long-lived worker process for +the whole suite (see `ZarrPythonWorker` and `src/test/python-scripts/zarr_python_worker.py`) rather +than one `uv run` per test case; interpreter startup used to dominate, and removing it took the +full interop run from ~129s to ~19s. Furthermore, you will need the `l4_sample` test data: @@ -69,6 +98,33 @@ Furthermore, you will need the `l4_sample` test data: && unzip l4_sample.zip ` +### Data Type Coverage + +`DataTypeConformanceTest` checks zarr-java's data types against an external list +(`src/test/resources/spec-data-types-v3.json`, generated from zarr-python's registry). Data types we +do not implement yet are enumerated in that test's `KNOWN_UNSUPPORTED` set. + +The list is deliberately *not* derived from our own `DataType` enum. A provider built from the enum +can only assert that what we implemented is implemented, so a data type we never added stays +invisible and no test can fail for it — which is how `float16` and `string` went unnoticed. + +Implementing a data type therefore means deleting its entry from `KNOWN_UNSUPPORTED`; the assertion +is bidirectional, so leaving a stale entry there fails the build too. + +To refresh the list after a zarr-python upgrade: + +``` +uv run src/test/python-scripts/generate_spec_data_types.py +``` + +Two CI jobs keep it from silently rotting (`.github/workflows/data-type-drift.yml`), split by who +caused the problem. On every pull request, the list is regenerated against the pinned zarr-python +and the build fails if it differs — that catches a dependency bump without a regeneration. Nightly, +it is regenerated against the *latest* zarr-python instead, which is how a newly specified data +type reaches us. The nightly check is deliberately not a pull-request gate: no pull request causes +upstream to release a version, and failing unrelated changes for it only teaches people to ignore +a red build. + ### Code Style & Formatting This project uses IntelliJ IDEA default Java formatting diff --git a/pom.xml b/pom.xml index e3d04d97..ddcca93a 100644 --- a/pom.xml +++ b/pom.xml @@ -44,6 +44,11 @@ 8 UTF-8 + + interop 2.20.0 2.34.6 5.9.1 @@ -156,6 +161,22 @@ 3.2.5 false + + ${excludedTestGroups} 1.44 diff --git a/src/test/java/dev/zarr/zarrjava/DataTypeConformanceTest.java b/src/test/java/dev/zarr/zarrjava/DataTypeConformanceTest.java new file mode 100644 index 00000000..c65ef6d5 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/DataTypeConformanceTest.java @@ -0,0 +1,242 @@ +package dev.zarr.zarrjava; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import dev.zarr.zarrjava.v3.ArrayMetadata; +import dev.zarr.zarrjava.v3.DataType; +import dev.zarr.zarrjava.v3.Node; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.stream.Stream; + +/** + * Metadata-only conformance tests for Zarr v3 data types. + * + *

These tests exist because the rest of the suite cannot detect a data type we never + * implemented. Providers such as {@link ZarrTest#dataTypeProviderV3()} enumerate our own + * {@link DataType} enum, so they can only ever assert that what we implemented is implemented -- a + * missing data type is invisible by construction, and no test can fail for an enum constant that + * does not exist. That blind spot is how {@code float16} and {@code string} went unnoticed. + * + *

The fix is to drive coverage from an external list: {@code + * /spec-data-types-v3.json}, generated from zarr-python's data type registry by {@code + * src/test/python-scripts/generate_spec_data_types.py} and committed so these tests stay offline + * and fast. Every entry must either parse or be named in {@link #KNOWN_UNSUPPORTED}. + * + *

The central assertion is deliberately bidirectional. An entry that fails to parse and is not + * listed is a coverage gap; an entry that parses but is still listed means the list went stale. + * Implementing a data type therefore shows up as a one-line deletion from {@link + * #KNOWN_UNSUPPORTED}, and forgetting to delete it is itself a test failure. + * + *

No I/O, no compression, no Python: these run in milliseconds and belong in every pull-request + * build. + */ +public class DataTypeConformanceTest { + + private static final String RESOURCE = "/spec-data-types-v3.json"; + + /** + * Data types that a reference implementation supports and zarr-java does not. + * + *

This list is the honest inventory of our gaps. It is expected to shrink, never grow: each + * entry removed is a data type gained. Keeping the gaps enumerated here rather than merely + * absent is the whole point -- an unimplemented data type is now a visible, deliberate decision + * instead of a silent hole. + * + *

Grouped by the work each needs: + * + *

    + *
  • Fixed itemsize -- {@code float16}, {@code complex64}, {@code complex128}, + * {@code fixed_length_utf32}, {@code raw_bytes}, {@code null_terminated_bytes}. Additive: + * these need an enum entry, an in-memory representation, and fill-value rules. The + * existing {@code bytes} codec already serializes them; no new codec is required. + * {@code ucar.ma2} has no half-float and no complex type, so the open question is how to + * represent them in memory, not how to encode them. + *
  • Variable itemsize -- {@code string}, {@code variable_length_bytes}. These + * require a dedicated array-to-bytes codec ({@code vlen-utf8} / {@code vlen-bytes}) + * because there is no itemsize to compute offsets from, which means breaking the + * fixed-size assumption in {@code core.DataType#getByteCount()}. + *
  • Compound / parameterized -- {@code structured}, {@code numpy.datetime64}, + * {@code numpy.timedelta64}. Note that zarr-python itself warns that {@code structured} + * and {@code variable_length_bytes} have no stable Zarr v3 specification yet. + *
+ */ + static final Set KNOWN_UNSUPPORTED = Collections.unmodifiableSet( + new LinkedHashSet<>(Arrays.asList( + "float16", + "complex64", + "complex128", + "fixed_length_utf32", + "string", + "raw_bytes", + "null_terminated_bytes", + "variable_length_bytes", + "structured", + "numpy.datetime64", + "numpy.timedelta64" + ))); + + private static JsonNode loadSpecDocument() { + try (InputStream in = DataTypeConformanceTest.class.getResourceAsStream(RESOURCE)) { + Assertions.assertNotNull(in, "Missing test resource " + RESOURCE + + ". Regenerate it with 'uv run src/test/python-scripts/generate_spec_data_types.py'."); + return new ObjectMapper().readTree(in); + } catch (IOException e) { + throw new RuntimeException("Could not read " + RESOURCE, e); + } + } + + static Stream specDataTypes() { + JsonNode dataTypes = loadSpecDocument().get("data_types"); + Stream.Builder builder = Stream.builder(); + for (JsonNode entry : dataTypes) { + builder.add(Arguments.of( + entry.get("name").asText(), + entry.get("data_type"), + entry.get("fill_value") + )); + } + return builder.build(); + } + + /** + * Builds the smallest {@code zarr.json} that exercises a data type, so a parse failure can only + * be about the data type or its fill value. + */ + private static String minimalArrayMetadata(JsonNode dataType, JsonNode fillValue) { + ObjectMapper objectMapper = new ObjectMapper(); + ObjectNode root = objectMapper.createObjectNode(); + root.put("zarr_format", 3); + root.put("node_type", "array"); + ArrayNode shape = root.putArray("shape"); + shape.add(4); + root.set("data_type", dataType); + ObjectNode chunkGrid = root.putObject("chunk_grid"); + chunkGrid.put("name", "regular"); + chunkGrid.putObject("configuration").putArray("chunk_shape").add(2); + root.putObject("chunk_key_encoding").put("name", "default"); + root.set("fill_value", fillValue); + root.putArray("codecs").addObject().put("name", "bytes"); + return root.toString(); + } + + /** + * Every data type a reference implementation writes must either be understood by zarr-java or be + * an acknowledged gap. + * + *

Deserializes the {@code data_type} member alone rather than a whole metadata document, so + * the result reflects data type support specifically and cannot be perturbed by unrelated + * validation elsewhere in {@link ArrayMetadata} (codec/data type compatibility checks, for + * instance). + */ + @ParameterizedTest(name = "{0}") + @MethodSource("specDataTypes") + public void specDataTypeIsSupportedOrKnownUnsupported(String name, JsonNode dataTypeJson, + JsonNode fillValueJson) { + boolean parsed; + String failure = null; + try { + DataType parsedDataType = + Node.makeObjectMapper().treeToValue(dataTypeJson, DataType.class); + parsed = parsedDataType != null; + } catch (Exception e) { + parsed = false; + failure = e.getClass().getSimpleName() + ": " + e.getMessage(); + } + + boolean listedAsUnsupported = KNOWN_UNSUPPORTED.contains(name); + + if (parsed && listedAsUnsupported) { + Assertions.fail("Data type '" + name + "' now parses, but is still listed in " + + "KNOWN_UNSUPPORTED. Remove it from that list -- implementing a data type is " + + "meant to show up as a deletion there."); + } + if (!parsed && !listedAsUnsupported) { + Assertions.fail("Data type '" + name + "' is supported by the reference implementation " + + "but zarr-java cannot parse it: " + failure + + "\nEither implement it in dev.zarr.zarrjava.v3.DataType or add it to " + + "KNOWN_UNSUPPORTED with a note on what it needs."); + } + } + + /** + * For the data types we do support, the fill value a reference implementation writes must parse + * as part of a complete metadata document. + * + *

Skipped for acknowledged gaps -- those already fail the test above, and reporting the same + * gap twice would only obscure it. + */ + @ParameterizedTest(name = "{0}") + @MethodSource("specDataTypes") + public void specFillValueParsesForSupportedDataType(String name, JsonNode dataTypeJson, + JsonNode fillValueJson) throws Exception { + Assumptions.assumeFalse(KNOWN_UNSUPPORTED.contains(name), + "data type not supported yet: " + name); + + String json = minimalArrayMetadata(dataTypeJson, fillValueJson); + ArrayMetadata metadata = Node.makeObjectMapper().readValue(json, ArrayMetadata.class); + + Assertions.assertNotNull(metadata.dataType, + "Parsed metadata for '" + name + "' has no data type"); + Assertions.assertNotNull(metadata.parsedFillValue, + "Fill value " + fillValueJson + " for '" + name + "' parsed to null"); + } + + /** + * Guards the conformance list itself: a resource that failed to load, or that lost entries in a + * regeneration, would otherwise make the tests above pass vacuously. + */ + @Test + public void specDataTypeListIsPresentAndPlausible() { + JsonNode document = loadSpecDocument(); + JsonNode dataTypes = document.get("data_types"); + Assertions.assertNotNull(dataTypes, "Conformance list has no 'data_types' member"); + Assertions.assertTrue(dataTypes.size() >= 22, + "Expected at least the 22 data types zarr-python 3.1.6 supports, found " + + dataTypes.size() + ". Was " + RESOURCE + " regenerated correctly?"); + + Set names = new HashSet<>(); + for (JsonNode entry : dataTypes) { + names.add(entry.get("name").asText()); + } + Assertions.assertTrue(names.containsAll(KNOWN_UNSUPPORTED), + "KNOWN_UNSUPPORTED names missing from " + RESOURCE + ": " + + KNOWN_UNSUPPORTED.stream().filter(n -> !names.contains(n)).toArray()); + } + + /** + * Documents the gap as a single number, so a reviewer sees at a glance how much of a reference + * implementation's data type surface we cover. Update the expected count when the list shrinks. + */ + @Test + public void supportedDataTypeCountIsAsExpected() { + JsonNode dataTypes = loadSpecDocument().get("data_types"); + int total = dataTypes.size(); + int unsupported = 0; + for (JsonNode entry : dataTypes) { + if (KNOWN_UNSUPPORTED.contains(entry.get("name").asText())) { + unsupported++; + } + } + Assertions.assertEquals(KNOWN_UNSUPPORTED.size(), unsupported, + "Every KNOWN_UNSUPPORTED entry should appear in the conformance list"); + Assertions.assertEquals(DataType.values().length, total - unsupported, + "Number of data types the conformance list says we support should match the " + + "DataType enum. If these diverge, either the enum gained a data type that " + + "is not in the conformance list, or KNOWN_UNSUPPORTED is stale."); + } +} diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index 5c22b5fc..b3126102 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -12,6 +12,7 @@ import dev.zarr.zarrjava.v3.codec.CodecBuilder; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -22,16 +23,31 @@ import java.io.IOException; import java.io.InputStreamReader; import java.nio.ByteBuffer; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Arrays; import java.util.stream.Stream; +/** + * Cross-implementation tests: everything here checks zarr-java against zarr-python. + * + *

These are the only tests that need Python, and they are tagged {@code interop} so that a bare + * {@code mvn test} skips them -- a contributor without a working uv/zarr-python setup still gets a + * useful run from the fast offline tiers ({@link DataTypeConformanceTest}, the per-data-type + * round-trips in {@link ZarrV3Test} and {@link ZarrV2Test}, and the committed golden fixtures). + * The tag is a local convenience only: CI clears the exclusion, so these run on every pull request. + * See the "Run Tests Locally" section of the README. + * + *

An external reference implementation is not optional for a format library. A test that writes + * with zarr-java and reads it back with zarr-java passes even when reader and writer share the same + * misunderstanding of the spec, and the resulting store is wrong for every other tool. Only a second + * implementation can catch that. + * + *

zarr-python runs as a single long-lived process ({@link ZarrPythonWorker}) rather than one + * launch per test case; see that class for why. + */ +@Tag("interop") public class ZarrPythonTests extends ZarrTest { - final static Path PYTHON_TEST_PATH = Paths.get("src/test/python-scripts/"); - public static int runCommand(String... command) throws IOException, InterruptedException { ProcessBuilder pb = new ProcessBuilder(); pb.command().addAll(Arrays.asList(command)); @@ -114,17 +130,23 @@ static Stream compressorAndDataTypeProviderV2() { return Stream.concat(datatypeTests, bloscTests); } - public void run_python_script(String scriptName, String... args) throws IOException, InterruptedException { - int exitCode = runCommand(Stream.concat(Stream.of("uv", "run", PYTHON_TEST_PATH.resolve(scriptName) - .toString()), Arrays.stream(args)).toArray(String[]::new)); - assert exitCode == 0; + /** + * Runs one zarr-python operation in the shared worker process. + * + *

Replaces the previous one-{@code uv run}-per-call approach. The operation names map to the + * functions in {@code src/test/python-scripts/zarr_fixtures.py}, which the standalone CLI + * scripts also call -- so driving an operation from here and running the matching script by hand + * execute the same code. + */ + public void runPython(String op, String... args) throws IOException, InterruptedException { + ZarrPythonWorker.get().call(op, args); } @ParameterizedTest @MethodSource("compressorAndDataTypeProviderV3") public void testReadV3(String codec, String codecParam, DataType dataType) throws IOException, ZarrException, InterruptedException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testReadV3", codec, codecParam, dataType.name()); - run_python_script("zarr_python_write.py", codec, codecParam, dataType.name().toLowerCase(), storeHandle.toPath().toString()); + runPython("write_v3", codec, codecParam, dataType.name().toLowerCase(), storeHandle.toPath().toString()); Array array = Array.open(storeHandle); ucar.ma2.Array result = array.read(); @@ -199,14 +221,14 @@ public void testWriteV3(String codec, String codecParam, DataType dataType) thro assertIsTestdata(result, dataType); //read in zarr_python - run_python_script("zarr_python_read.py", codec, codecParam, dataType.name().toLowerCase(), storeHandle.toPath().toString()); + runPython("read_v3", codec, codecParam, dataType.name().toLowerCase(), storeHandle.toPath().toString()); } @ParameterizedTest @MethodSource("compressorAndDataTypeProviderV2") public void testReadV2(String compressor, String compressorParam, dev.zarr.zarrjava.v2.DataType dt) throws IOException, ZarrException, InterruptedException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testReadV2", compressor, compressorParam, dt.name()); - run_python_script("zarr_python_write_v2.py", compressor, compressorParam, dt.getValue(), storeHandle.toPath().toString()); + runPython("write_v2", compressor, compressorParam, dt.getValue(), storeHandle.toPath().toString()); dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.open(storeHandle); ucar.ma2.Array result = array.read(); @@ -265,7 +287,7 @@ public void testWriteV2(String compressor, String compressorParam, dev.zarr.zarr assertIsTestdata(result, dt); //read in zarr_python - run_python_script("zarr_python_read_v2.py", compressor, compressorParam, dt.getValue(), storeHandle.toPath().toString()); + runPython("read_v2", compressor, compressorParam, dt.getValue(), storeHandle.toPath().toString()); } @CsvSource({"0,true", "0,false", "5, true", "10, false"}) @@ -292,14 +314,7 @@ public void testZstdLibrary(int level, boolean checksumFlag) throws IOException, } //decompress in python - int exitCode = ZarrPythonTests.runCommand( - "uv", - "run", - PYTHON_TEST_PATH.resolve("zstd_decompress.py").toString(), - compressedDataPath, - Integer.toString(number) - ); - assert exitCode == 0; + runPython("zstd_decompress", compressedDataPath, Integer.toString(number)); } @Test @@ -316,7 +331,7 @@ public void testGroupReadWriteV2() throws Exception { array.write(testdata(dataType)); - run_python_script("zarr_python_group.py", storeHandle.toPath().toString(), storeHandle2.toPath().toString(), "" + 2); + runPython("group", storeHandle.toPath().toString(), storeHandle2.toPath().toString(), "" + 2); Group group2 = Group.open(storeHandle2); Assertions.assertEquals("value", group2.metadata().attributes().get("attr")); @@ -343,7 +358,7 @@ public void testGroupReadWriteV3() throws Exception { array.write(testdata(dataType)); - run_python_script("zarr_python_group.py", storeHandle.toPath().toString(), storeHandle2.toPath().toString(), "" + 3); + runPython("group", storeHandle.toPath().toString(), storeHandle2.toPath().toString(), "" + 3); dev.zarr.zarrjava.v3.Group group2 = dev.zarr.zarrjava.v3.Group.open(storeHandle2); Assertions.assertEquals("value", group2.metadata().attributes().get("attr")); diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonWorker.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonWorker.java new file mode 100644 index 00000000..e6dd5488 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonWorker.java @@ -0,0 +1,183 @@ +package dev.zarr.zarrjava; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +/** + * A single long-lived zarr-python process, shared by every interop test in the JVM. + * + *

The interop suite used to run one {@code uv run