diff --git a/USERGUIDE.md b/USERGUIDE.md
index 65bf9944..db28edc8 100644
--- a/USERGUIDE.md
+++ b/USERGUIDE.md
@@ -288,6 +288,61 @@ Array array = group.createArray(
.build()
);
```
+### Consolidated Metadata (v3)
+A group can keep a copy of the metadata of all of its descendants inside its own `zarr.json`, so that
+the whole hierarchy can be opened with a single read instead of one read per node. This matters most
+over HTTP and S3, where every node otherwise costs a request.
+
+```java
+// Write the cache. This walks the hierarchy once and stores the metadata of every
+// descendant in the metadata of this group.
+Group root = Group.consolidateMetadata(storeHandle);
+Group root = Group.consolidateMetadata("/data/my.zarr"); // or a path
+
+// Consolidate a subtree instead of the whole hierarchy
+Group.consolidateMetadata(storeHandle.resolve("sub"));
+
+// The same, on a group that is already open
+Group root = Group.open(storeHandle).consolidateMetadata();
+
+// Reads and listings are answered from the cache, without touching the store.
+Group sub = (Group) root.get("sub");
+Array array = (Array) sub.get("nested");
+Node[] everything = root.listAsArray();
+
+// Remove the cache again
+root.dropConsolidatedMetadata();
+```
+
+On opening, the cache is used if the group has one. Pass `UseConsolidated` to say otherwise:
+
+```java
+Group auto = Group.open(storeHandle); // use it if present
+Group required = Group.open(storeHandle, UseConsolidated.REQUIRE); // fail if absent
+Group required = Group.openConsolidated(storeHandle); // the same, shorter
+Group fresh = Group.open(storeHandle, UseConsolidated.IGNORE); // read every node instead
+```
+
+`UseConsolidated.IGNORE` drops the cache from the metadata held in memory, so writing that metadata
+again - with `setAttributes()` for example - removes the cache from the store as well. The choice
+applies to the group being opened only: a subgroup reached through `get()` uses a cache of its own if
+it has one.
+
+The cache is written in the same format as `zarr.consolidate_metadata()` in zarr-python, and behaves
+the same way on reading, so both libraries can read each other's output.
+
+**The cache is authoritative.** While a group is using its cache, `get()` and `list()` answer from it
+alone and never fall back to the store. A key the cache does not hold is reported as absent.
+
+**The cache is a snapshot.** Nothing invalidates it when a node is added, removed or changed
+afterwards, so it has to be written again after modifying the hierarchy. Until then a node added
+afterwards is invisible, and a node changed afterwards is served as it was. Open the group with
+`UseConsolidated.IGNORE` if in doubt.
+
+Consolidated metadata is a Zarr v3 feature here; the v2 `.zmetadata` file is not supported. Note that
+consolidated metadata is not part of the Zarr v3 specification, so other implementations may not
+support it; `consolidateMetadata()` logs a warning saying so, as zarr-python does.
+
### Hierarchical Example
```java
Group root = Group.create(
diff --git a/src/main/java/dev/zarr/zarrjava/v3/ConsolidatedMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/ConsolidatedMetadata.java
new file mode 100644
index 00000000..a21f7ac6
--- /dev/null
+++ b/src/main/java/dev/zarr/zarrjava/v3/ConsolidatedMetadata.java
@@ -0,0 +1,168 @@
+package dev.zarr.zarrjava.v3;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.JsonNode;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * An optional cache of the metadata of all descendants of a group, stored inside that group's own
+ * {@code zarr.json} under the {@code consolidated_metadata} key. It allows a reader to open a whole
+ * hierarchy with a single request instead of one request per node.
+ *
+ * The keys of {@link #metadata} are flat, {@code "/"}-joined paths relative to the group holding the
+ * cache, for example {@code "ocean"} and {@code "ocean/salinity"}.
+ *
+ * A cache of a kind other than {@link #KIND_INLINE} makes the group unopenable, which is what
+ * zarr-python does as well.
+ *
+ * The cached node metadata is deliberately kept as raw {@link JsonNode} rather than as parsed
+ * {@link ArrayMetadata} / {@link GroupMetadata}. The cache is declared with
+ * {@code must_understand: false}, so a reader that cannot interpret an entry has to ignore it rather
+ * than fail. Parsing entries eagerly would mean that a single node written by another implementation
+ * with a field this library does not model would make the whole group unopenable. Keeping the raw
+ * JSON also lets {@link Group#consolidateMetadata()} copy each node's metadata verbatim, so the cache
+ * never silently loses information that is present in the node's own {@code zarr.json}.
+ *
+ * The cache is a snapshot taken at the time of consolidation. Nothing invalidates it when a
+ * descendant changes, so {@link Group#consolidateMetadata()} has to be re-run after modifying the
+ * hierarchy.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public final class ConsolidatedMetadata {
+
+ /**
+ * The only cache kind defined so far: the metadata is stored inline in the group's metadata
+ * document. A cache of any other kind is rejected by this library.
+ */
+ public static final String KIND_INLINE = "inline";
+
+ @Nonnull
+ @JsonProperty("kind")
+ public final String kind;
+
+ @JsonProperty("must_understand")
+ public final boolean mustUnderstand;
+
+ /**
+ * The cached metadata documents, keyed by their {@code "/"}-joined path relative to the group
+ * holding this cache.
+ */
+ @Nonnull
+ @JsonProperty("metadata")
+ public final Map metadata;
+
+ @JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
+ public ConsolidatedMetadata(
+ @Nullable @JsonProperty("kind") String kind,
+ @Nullable @JsonProperty("must_understand") Boolean mustUnderstand,
+ @Nullable @JsonProperty("metadata") Map metadata
+ ) {
+ this.kind = kind == null ? KIND_INLINE : kind;
+ this.mustUnderstand = mustUnderstand != null && mustUnderstand;
+ this.metadata = metadata == null
+ ? Collections.emptyMap()
+ : Collections.unmodifiableMap(new LinkedHashMap<>(metadata));
+ }
+
+ public ConsolidatedMetadata(@Nonnull Map metadata) {
+ this(KIND_INLINE, false, metadata);
+ }
+
+ /**
+ * An empty inline cache, used for a consolidated subgroup whose own entries have been hoisted
+ * into the cache of an ancestor.
+ */
+ public static ConsolidatedMetadata empty() {
+ return new ConsolidatedMetadata(Collections.emptyMap());
+ }
+
+ /**
+ * Whether this cache is stored inline and can therefore be used by this library. A group whose
+ * cache is not inline cannot be opened, see {@link GroupMetadata}.
+ */
+ @JsonIgnore
+ public boolean isInline() {
+ return KIND_INLINE.equals(kind);
+ }
+
+ @JsonIgnore
+ public boolean isEmpty() {
+ return metadata.isEmpty();
+ }
+
+ /**
+ * Returns the cached metadata document for a node, or null if this cache does not hold it.
+ *
+ * @param key the path of the node relative to the group holding this cache
+ */
+ @Nullable
+ public JsonNode get(String[] key) {
+ if (!isInline()) {
+ return null;
+ }
+ return metadata.get(String.join("/", key));
+ }
+
+ /**
+ * Returns the keys of this cache with every group listed before its own descendants, and
+ * siblings in the order they are held in this cache. This is the order in which zarr-python
+ * yields the members of a consolidated group, because it walks the nested representation it
+ * builds when reading the cache.
+ */
+ public List depthFirstKeys() {
+ Map> childrenByParent = new LinkedHashMap<>();
+ for (String key : metadata.keySet()) {
+ int lastSlash = key.lastIndexOf('/');
+ String parent = lastSlash < 0 ? "" : key.substring(0, lastSlash);
+ childrenByParent.computeIfAbsent(parent, unused -> new ArrayList<>()).add(key);
+ }
+ Set ordered = new LinkedHashSet<>();
+ appendDepthFirst("", childrenByParent, ordered);
+ // A key whose parent is missing from the cache is not reachable from the group, so it is not
+ // visited above. Append it, so that a malformed cache hides nothing.
+ ordered.addAll(metadata.keySet());
+ return new ArrayList<>(ordered);
+ }
+
+ private static void appendDepthFirst(String parent, Map> childrenByParent,
+ Set out) {
+ List children = childrenByParent.get(parent);
+ if (children == null) {
+ return;
+ }
+ for (String child : children) {
+ out.add(child);
+ appendDepthFirst(child, childrenByParent, out);
+ }
+ }
+
+ /**
+ * Returns the entries below {@code prefix} with the prefix stripped from their keys, so that the
+ * result can serve as the cache of the subgroup at {@code prefix}.
+ */
+ public ConsolidatedMetadata sub(String[] prefix) {
+ if (!isInline()) {
+ return empty();
+ }
+ String keyPrefix = String.join("/", prefix) + "/";
+ Map sub = new LinkedHashMap<>();
+ for (Map.Entry entry : metadata.entrySet()) {
+ if (entry.getKey().startsWith(keyPrefix)) {
+ sub.put(entry.getKey().substring(keyPrefix.length()), entry.getValue());
+ }
+ }
+ return new ConsolidatedMetadata(sub);
+ }
+}
diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java
index 8b1a81bb..465d2ca6 100644
--- a/src/main/java/dev/zarr/zarrjava/v3/Group.java
+++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java
@@ -1,10 +1,14 @@
package dev.zarr.zarrjava.v3;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import dev.zarr.zarrjava.ZarrException;
import dev.zarr.zarrjava.core.Attributes;
import dev.zarr.zarrjava.store.FilesystemStore;
import dev.zarr.zarrjava.store.MemoryStore;
+import dev.zarr.zarrjava.store.Store;
import dev.zarr.zarrjava.store.StoreHandle;
import dev.zarr.zarrjava.utils.Utils;
@@ -15,8 +19,18 @@
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.text.Normalizer;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
import java.util.function.Function;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
import java.util.stream.Stream;
import static dev.zarr.zarrjava.v3.Node.makeObjectMapper;
@@ -25,8 +39,48 @@
public class Group extends dev.zarr.zarrjava.core.Group implements Node {
+ private static final Logger LOGGER = Logger.getLogger(Group.class.getName());
+
+ /**
+ * The order in which entries are written into the consolidated metadata: shallow paths first,
+ * then case-insensitively by name. This only affects the byte layout of the written metadata
+ * document, but it makes consolidating the same hierarchy twice produce an identical file.
+ */
+ private static final Comparator CONSOLIDATED_KEY_ORDER = Comparator
+ .comparingInt((String key) -> (int) key.chars().filter(c -> c == '/').count())
+ .thenComparing(key -> Normalizer.normalize(key, Normalizer.Form.NFKC).toLowerCase(Locale.ROOT))
+ .thenComparing(Comparator.naturalOrder());
+
public GroupMetadata metadata;
+ /**
+ * How {@link #open} treats the consolidated metadata of the group it opens. Mirrors the
+ * {@code use_consolidated} argument of {@code zarr.open_group()} in zarr-python.
+ *
+ * The choice applies to the group being opened only. A subgroup reached through {@link #get} is
+ * opened with {@link #AUTO}, so a subgroup that carries a cache of its own still uses it.
+ */
+ public enum UseConsolidated {
+ /**
+ * Use the consolidated metadata if the group has any, and read the nodes themselves
+ * otherwise. The default, matching {@code use_consolidated=None}.
+ */
+ AUTO,
+ /**
+ * Require consolidated metadata: fail if the group has none. Matches
+ * {@code use_consolidated=True}.
+ */
+ REQUIRE,
+ /**
+ * Ignore the consolidated metadata of the group, and drop it from the metadata held in
+ * memory. Matches {@code use_consolidated=False}.
+ *
+ * Because the cache is dropped from the metadata of the group, writing that metadata again -
+ * with {@link Group#setAttributes} for example - removes the cache from the store as well.
+ */
+ IGNORE
+ }
+
protected Group(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata) throws IOException {
super(storeHandle);
this.metadata = groupMetadata;
@@ -38,10 +92,69 @@ protected Group(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMe
* @param storeHandle the storage location of the Zarr group
* @throws IOException if the metadata cannot be read
*/
- public static Group open(@Nonnull StoreHandle storeHandle) throws IOException {
+ public static Group open(@Nonnull StoreHandle storeHandle) throws IOException, ZarrException {
+ return open(storeHandle, UseConsolidated.AUTO);
+ }
+
+ /**
+ * Opens an existing Zarr group at a specified storage location.
+ *
+ * @param storeHandle the storage location of the Zarr group
+ * @param useConsolidated how the consolidated metadata of the group is to be treated
+ * @throws IOException if the metadata cannot be read
+ * @throws ZarrException if {@link UseConsolidated#REQUIRE} was passed and the group has no
+ * consolidated metadata
+ */
+ public static Group open(@Nonnull StoreHandle storeHandle,
+ @Nonnull UseConsolidated useConsolidated) throws IOException, ZarrException {
StoreHandle metadataHandle = storeHandle.resolve(ZARR_JSON);
ByteBuffer metadataBytes = metadataHandle.readNonNull();
- return new Group(storeHandle, makeObjectMapper().readValue(Utils.toArray(metadataBytes), GroupMetadata.class));
+ GroupMetadata groupMetadata =
+ makeObjectMapper().readValue(Utils.toArray(metadataBytes), GroupMetadata.class);
+ if (useConsolidated == UseConsolidated.REQUIRE && groupMetadata.consolidatedMetadata == null) {
+ throw new ZarrException("Consolidated metadata requested with UseConsolidated.REQUIRE,"
+ + " but not found in '" + storeHandle + "'.");
+ }
+ if (useConsolidated == UseConsolidated.IGNORE && groupMetadata.consolidatedMetadata != null) {
+ groupMetadata = groupMetadata.withConsolidatedMetadata(null);
+ }
+ return new Group(storeHandle, groupMetadata);
+ }
+
+ /**
+ * Opens an existing Zarr group at a specified storage location, requiring it to have consolidated
+ * metadata. Mirrors {@code zarr.open_consolidated()} in zarr-python.
+ *
+ * @param storeHandle the storage location of the Zarr group
+ * @throws IOException if the metadata cannot be read
+ * @throws ZarrException if the group has no consolidated metadata
+ */
+ public static Group openConsolidated(@Nonnull StoreHandle storeHandle) throws IOException, ZarrException {
+ return open(storeHandle, UseConsolidated.REQUIRE);
+ }
+
+ /**
+ * Opens an existing Zarr group at a specified storage location, requiring it to have consolidated
+ * metadata.
+ *
+ * @param path the storage location of the Zarr group
+ * @throws IOException if the metadata cannot be read
+ * @throws ZarrException if the group has no consolidated metadata
+ */
+ public static Group openConsolidated(Path path) throws IOException, ZarrException {
+ return openConsolidated(new StoreHandle(new FilesystemStore(path)));
+ }
+
+ /**
+ * Opens an existing Zarr group at a specified storage location, requiring it to have consolidated
+ * metadata.
+ *
+ * @param path the storage location of the Zarr group
+ * @throws IOException if the metadata cannot be read
+ * @throws ZarrException if the group has no consolidated metadata
+ */
+ public static Group openConsolidated(String path) throws IOException, ZarrException {
+ return openConsolidated(Paths.get(path));
}
@@ -51,7 +164,7 @@ public static Group open(@Nonnull StoreHandle storeHandle) throws IOException {
* @param path the storage location of the Zarr group
* @throws IOException if the metadata cannot be read
*/
- public static Group open(Path path) throws IOException {
+ public static Group open(Path path) throws IOException, ZarrException {
return open(new StoreHandle(new FilesystemStore(path)));
}
@@ -61,7 +174,7 @@ public static Group open(Path path) throws IOException {
* @param path the storage location of the Zarr group
* @throws IOException if the metadata cannot be read
*/
- public static Group open(String path) throws IOException {
+ public static Group open(String path) throws IOException, ZarrException {
return open(Paths.get(path));
}
@@ -184,32 +297,257 @@ public static Group create(String path) throws IOException, ZarrException {
*/
@Nullable
public Node get(String[] key) throws ZarrException, IOException {
- StoreHandle keyHandle = storeHandle.resolve(key);
+ ConsolidatedMetadata consolidated = metadata.consolidatedMetadata;
+ if (consolidated == null) {
+ return openFromStore(key);
+ }
+ JsonNode cached = consolidated.get(key);
+ if (cached == null) {
+ // The cache is authoritative: a node it does not hold is not part of the hierarchy. The
+ // store is deliberately not consulted, which is what zarr-python does as well. The cache
+ // is a snapshot, so a node added after consolidating stays invisible until
+ // consolidateMetadata() is called again, or the group is opened with
+ // UseConsolidated.IGNORE.
+ return null;
+ }
+ Node node = nodeFromConsolidatedMetadata(key, cached, consolidated);
+ if (node != null) {
+ return node;
+ }
+ // The cached document could not be interpreted, fall back to the node itself.
+ return openFromStore(key);
+ }
+
+ /**
+ * Opens the node at {@code key} by reading its metadata from the store, ignoring any consolidated
+ * metadata of this group. A subgroup that carries a cache of its own uses it, just as it would if
+ * it had been opened directly.
+ */
+ @Nullable
+ private Node openFromStore(String[] key) throws ZarrException, IOException {
try {
- return Node.open(keyHandle);
+ return Node.open(storeHandle.resolve(key));
} catch (NoSuchFileException e) {
return null;
}
}
+ /**
+ * Lists all descendants of this group. If this group has consolidated metadata, the whole listing
+ * is answered from it and the store is not touched at all, neither to list nor to read.
+ */
@Override
public Stream list() {
+ ConsolidatedMetadata consolidated = metadata.consolidatedMetadata;
+ if (consolidated != null) {
+ return consolidated.depthFirstKeys().stream()
+ .map(key -> nodeAt(key.split("/")))
+ .filter(Objects::nonNull);
+ }
Stream metadataKeys = storeHandle.list()
.filter(key -> key[key.length - 1].equals(ZARR_JSON))
.filter(key -> key.length > 1); // exclude root from list
- return metadataKeys.map(key -> {
- try {
- return get(Arrays.copyOf(key, key.length - 1));
- } catch (IOException e) {
- throw new RuntimeException(
- "Failed to read node metadata for key '" + String.join("/", key) + "': " + e.getMessage(), e);
- } catch (ZarrException e) {
- throw new RuntimeException(
- "Failed to parse node metadata for key '" + String.join("/", key) + "': " + e.getMessage(), e);
+ return metadataKeys.map(key -> nodeAt(Arrays.copyOf(key, key.length - 1)))
+ .filter(Objects::nonNull);
+ }
+
+ /**
+ * Calls {@link #get} for a listing, turning the checked exceptions into unchecked ones.
+ */
+ @Nullable
+ private dev.zarr.zarrjava.core.Node nodeAt(String[] key) {
+ try {
+ return get(key);
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "Failed to read node metadata for key '" + String.join("/", key) + "': " + e.getMessage(), e);
+ } catch (ZarrException e) {
+ throw new RuntimeException(
+ "Failed to parse node metadata for key '" + String.join("/", key) + "': " + e.getMessage(), e);
+ }
+ }
+
+
+ /**
+ * Builds a node from a cached metadata document, or returns null if the document cannot be
+ * interpreted. The consolidated metadata is declared with {@code must_understand: false}, so an
+ * entry this library does not understand is skipped in favour of reading the node itself rather
+ * than failing.
+ */
+ @Nullable
+ private Node nodeFromConsolidatedMetadata(String[] key, JsonNode cached,
+ ConsolidatedMetadata consolidated) {
+ StoreHandle keyHandle = storeHandle.resolve(key);
+ JsonNode nodeTypeNode = cached.get("node_type");
+ String nodeType = nodeTypeNode == null ? null : nodeTypeNode.asText();
+ try {
+ ObjectMapper objectMapper = makeObjectMapper();
+ if (ArrayMetadata.NODE_TYPE.equals(nodeType)) {
+ return new Array(keyHandle, objectMapper.treeToValue(cached, ArrayMetadata.class));
}
- });
+ if (GroupMetadata.NODE_TYPE.equals(nodeType)) {
+ GroupMetadata groupMetadata = objectMapper.treeToValue(cached, GroupMetadata.class);
+ // The entries of a consolidated subgroup are hoisted into the cache of this group, so
+ // hand the subgroup its own slice of them instead of the emptied cache it carries.
+ return new Group(keyHandle,
+ groupMetadata.withConsolidatedMetadata(consolidated.sub(key)));
+ }
+ LOGGER.warning("Ignoring the consolidated metadata of '" + String.join("/", key)
+ + "' below " + storeHandle + ", it has an unsupported node type '" + nodeType + "'.");
+ return null;
+ } catch (Exception e) {
+ LOGGER.warning("Ignoring the consolidated metadata of '" + String.join("/", key)
+ + "' below " + storeHandle + ", it could not be parsed: " + e.getMessage());
+ return null;
+ }
+ }
+
+ /**
+ * Writes the metadata of all descendants of this group into the metadata of this group, so that
+ * the whole hierarchy can afterwards be opened with a single read.
+ *
+ * The metadata of each descendant is copied verbatim, keyed by its {@code "/"}-joined path
+ * relative to this group. The copy of a subgroup is given an empty cache of its own, marking it as
+ * covered by the cache written here; a subgroup that was consolidated itself therefore does not
+ * have its entries stored twice.
+ *
+ * The result is a snapshot. Nothing invalidates it when a node is added, removed or modified
+ * afterwards, so this has to be called again after changing the hierarchy. Until then the cache is
+ * answered as it stands: a node added afterwards is not found, and a node modified afterwards is
+ * served as it was. Open the group with {@link UseConsolidated#IGNORE} to bypass the cache.
+ *
+ * @return this group, with the consolidated metadata written
+ * @throws IOException if the metadata cannot be read or written
+ * @throws UnsupportedOperationException if the underlying store does not support listing
+ */
+ public Group consolidateMetadata() throws IOException {
+ if (!(storeHandle.store instanceof Store.ListableStore)) {
+ throw new UnsupportedOperationException("The Zarr store in use ("
+ + storeHandle.store.getClass().getSimpleName() + ") doesn't support consolidated"
+ + " metadata, because it cannot be listed.");
+ }
+ LOGGER.warning("Consolidated metadata is currently not part of the Zarr format 3"
+ + " specification. It may not be supported by other zarr implementations and may"
+ + " change in the future.");
+ Map entries = new LinkedHashMap<>();
+ collectDescendantMetadata(new String[0], entries);
+
+ List keys = new ArrayList<>(entries.keySet());
+ keys.sort(CONSOLIDATED_KEY_ORDER);
+ Map sorted = new LinkedHashMap<>();
+ for (String key : keys) {
+ sorted.put(key, entries.get(key));
+ }
+ return writeMetadata(
+ metadata.withConsolidatedMetadata(new ConsolidatedMetadata(sorted)));
+ }
+
+ /**
+ * Opens the group at a storage location, consolidates its metadata and writes the result, in one
+ * call. Mirrors {@code zarr.consolidate_metadata()} in zarr-python.
+ *
+ * Any consolidated metadata the group already has is ignored while walking, so the result is
+ * always built from the nodes themselves. Pass a handle resolved deeper into the store,
+ * {@code handle.resolve("sub")}, to consolidate a subtree instead of the whole hierarchy.
+ *
+ * @param storeHandle the storage location of the Zarr group
+ * @return the group, with the consolidated metadata written
+ * @throws IOException if the metadata cannot be read or written
+ * @throws ZarrException if the metadata of the group cannot be parsed
+ * @throws UnsupportedOperationException if the underlying store does not support listing
+ */
+ public static Group consolidateMetadata(@Nonnull StoreHandle storeHandle) throws IOException, ZarrException {
+ return open(storeHandle, UseConsolidated.IGNORE).consolidateMetadata();
}
+ /**
+ * Opens the group at a storage location, consolidates its metadata and writes the result, in one
+ * call.
+ *
+ * @param path the storage location of the Zarr group
+ * @return the group, with the consolidated metadata written
+ * @throws IOException if the metadata cannot be read or written
+ * @throws ZarrException if the metadata of the group cannot be parsed
+ */
+ public static Group consolidateMetadata(Path path) throws IOException, ZarrException {
+ return consolidateMetadata(new StoreHandle(new FilesystemStore(path)));
+ }
+
+ /**
+ * Opens the group at a storage location, consolidates its metadata and writes the result, in one
+ * call.
+ *
+ * @param path the storage location of the Zarr group
+ * @return the group, with the consolidated metadata written
+ * @throws IOException if the metadata cannot be read or written
+ * @throws ZarrException if the metadata of the group cannot be parsed
+ */
+ public static Group consolidateMetadata(String path) throws IOException, ZarrException {
+ return consolidateMetadata(Paths.get(path));
+ }
+
+ /**
+ * Removes the consolidated metadata of this group, so that its descendants are read from the store
+ * again.
+ *
+ * @return this group, with the consolidated metadata removed
+ * @throws IOException if the metadata cannot be written
+ */
+ public Group dropConsolidatedMetadata() throws IOException {
+ if (metadata.consolidatedMetadata == null) {
+ return this;
+ }
+ return writeMetadata(metadata.withConsolidatedMetadata(null));
+ }
+
+ /**
+ * Collects the metadata documents of all nodes below {@code prefix} into {@code out}, keyed by
+ * their path relative to this group.
+ */
+ private void collectDescendantMetadata(String[] prefix, Map out)
+ throws IOException {
+ List children;
+ try (Stream stream = storeHandle.resolve(prefix).listChildren()) {
+ children = stream.filter(name -> !ZARR_JSON.equals(name)).collect(Collectors.toList());
+ }
+ for (String child : children) {
+ String[] key = Utils.concatArrays(prefix, new String[]{child});
+ ByteBuffer metadataBytes = storeHandle.resolve(key).resolve(ZARR_JSON).read();
+ if (metadataBytes == null) {
+ // Not a node itself, but it may still contain nodes further down.
+ collectDescendantMetadata(key, out);
+ continue;
+ }
+ JsonNode nodeMetadata = makeObjectMapper().readTree(Utils.toArray(metadataBytes));
+ JsonNode nodeTypeNode = nodeMetadata.get("node_type");
+ boolean isGroup = nodeTypeNode != null && GroupMetadata.NODE_TYPE.equals(nodeTypeNode.asText());
+ if (isGroup) {
+ markSubgroupAsConsolidated(nodeMetadata);
+ }
+ out.put(String.join("/", key), nodeMetadata);
+ if (isGroup) {
+ collectDescendantMetadata(key, out);
+ }
+ }
+ }
+
+ /**
+ * Gives the cached metadata of a subgroup an empty consolidated metadata cache of its own. The
+ * empty cache marks the subgroup as covered by the cache being written here, which is where its
+ * entries live. A subgroup that carried a cache of its own loses it in this copy, so that the same
+ * entries are not held twice and cannot drift apart. This mirrors what zarr-python writes.
+ */
+ private static void markSubgroupAsConsolidated(JsonNode nodeMetadata) {
+ if (!(nodeMetadata instanceof ObjectNode)) {
+ return;
+ }
+ ObjectNode metadataObject = (ObjectNode) nodeMetadata;
+ ObjectNode nested = metadataObject.objectNode();
+ nested.put("kind", ConsolidatedMetadata.KIND_INLINE);
+ nested.put("must_understand", false);
+ nested.set("metadata", metadataObject.objectNode());
+ metadataObject.set("consolidated_metadata", nested);
+ }
/**
* Creates a new subgroup with the provided metadata at the specified key.
@@ -302,7 +640,10 @@ public Group updateAttributes(Function attributeMapper)
* @throws IOException if the metadata cannot be serialized
*/
public Group setAttributes(Attributes newAttributes) throws ZarrException, IOException {
- GroupMetadata newGroupMetadata = new GroupMetadata(newAttributes);
+ // The consolidated metadata describes the descendants of this group, which are unaffected by
+ // a change to the attributes of the group itself.
+ GroupMetadata newGroupMetadata =
+ new GroupMetadata(newAttributes, metadata.consolidatedMetadata);
return writeMetadata(newGroupMetadata);
}
diff --git a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java
index 56414a2e..e8f426d0 100644
--- a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java
+++ b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java
@@ -16,21 +16,35 @@ public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata {
public final int zarrFormat = ZARR_FORMAT;
@JsonProperty("node_type")
public final String nodeType = "group";
+
+ /**
+ * An optional cache of the metadata of all descendants of this group, or null if this group has
+ * not been consolidated. See {@link ConsolidatedMetadata} and {@link Group#consolidateMetadata()}.
+ */
+ @Nullable
@JsonProperty("consolidated_metadata")
- public final Object consolidatedMetadata = null;
+ public final ConsolidatedMetadata consolidatedMetadata;
@Nullable
public final Attributes attributes;
public GroupMetadata(@Nullable Attributes attributes) throws ZarrException {
- this(ZARR_FORMAT, NODE_TYPE, attributes);
+ this(ZARR_FORMAT, NODE_TYPE, attributes, null);
+ }
+
+ public GroupMetadata(
+ @Nullable Attributes attributes,
+ @Nullable ConsolidatedMetadata consolidatedMetadata
+ ) throws ZarrException {
+ this(ZARR_FORMAT, NODE_TYPE, attributes, consolidatedMetadata);
}
@JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
public GroupMetadata(
@JsonProperty(value = "zarr_format", required = true) int zarrFormat,
@JsonProperty(value = "node_type", required = true) String nodeType,
- @Nullable @JsonProperty(value = "attributes") Attributes attributes
+ @Nullable @JsonProperty(value = "attributes") Attributes attributes,
+ @Nullable @JsonProperty(value = "consolidated_metadata") ConsolidatedMetadata consolidatedMetadata
) throws ZarrException {
if (zarrFormat != this.zarrFormat) {
throw new ZarrException(
@@ -40,12 +54,17 @@ public GroupMetadata(
throw new ZarrException(
"Expected node type '" + this.nodeType + "', got '" + nodeType + "'.");
}
+ if (consolidatedMetadata != null && !consolidatedMetadata.isInline()) {
+ throw new ZarrException(
+ "Consolidated metadata kind='" + consolidatedMetadata.kind + "' is not supported.");
+ }
this.attributes = attributes;
+ this.consolidatedMetadata = consolidatedMetadata;
}
public static GroupMetadata defaultValue() {
try {
- return new GroupMetadata(ZARR_FORMAT, NODE_TYPE, new Attributes());
+ return new GroupMetadata(ZARR_FORMAT, NODE_TYPE, new Attributes(), null);
} catch (ZarrException e) {
// This should never happen with default values
throw new IllegalStateException(
@@ -53,6 +72,20 @@ public static GroupMetadata defaultValue() {
}
}
+ /**
+ * Returns a copy of this metadata with a different consolidated metadata cache, or without one if
+ * {@code newConsolidatedMetadata} is null.
+ */
+ public GroupMetadata withConsolidatedMetadata(@Nullable ConsolidatedMetadata newConsolidatedMetadata) {
+ try {
+ return new GroupMetadata(zarrFormat, nodeType, attributes, newConsolidatedMetadata);
+ } catch (ZarrException e) {
+ // This should never happen, the format and node type are copied from a valid instance
+ throw new IllegalStateException(
+ "Failed to copy GroupMetadata - this indicates a programming error", e);
+ }
+ }
+
@Override
public @Nonnull Attributes attributes() throws ZarrException {
if (attributes == null) {
diff --git a/src/test/java/dev/zarr/zarrjava/ConsolidatedMetadataTest.java b/src/test/java/dev/zarr/zarrjava/ConsolidatedMetadataTest.java
new file mode 100644
index 00000000..2402ce3a
--- /dev/null
+++ b/src/test/java/dev/zarr/zarrjava/ConsolidatedMetadataTest.java
@@ -0,0 +1,667 @@
+package dev.zarr.zarrjava;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import dev.zarr.zarrjava.core.Attributes;
+import dev.zarr.zarrjava.store.MemoryStore;
+import dev.zarr.zarrjava.store.Store;
+import dev.zarr.zarrjava.store.StoreHandle;
+import dev.zarr.zarrjava.utils.Utils;
+import dev.zarr.zarrjava.v3.ConsolidatedMetadata;
+import dev.zarr.zarrjava.v3.DataType;
+import dev.zarr.zarrjava.v3.Group;
+import dev.zarr.zarrjava.v3.Node;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
+
+import static dev.zarr.zarrjava.core.Node.ZARR_JSON;
+
+/**
+ * Tests for the {@code consolidated_metadata} cache of a v3 group: writing it with
+ * {@link Group#consolidateMetadata()}, answering {@link Group#get} from it, and tolerating caches
+ * this library cannot fully interpret.
+ */
+public class ConsolidatedMetadataTest {
+
+ private static final Set EXPECTED_ENTRIES = new HashSet<>(Arrays.asList(
+ "arr",
+ "sub",
+ "sub/nested",
+ "sub/deep",
+ "sub/deep/deepArray"
+ ));
+
+ /**
+ * The order in which {@link Group#list()} yields the members of the consolidated test hierarchy:
+ * every group before its own descendants, siblings in the order the cache holds them. This is the
+ * order in which zarr-python yields the members of the same group.
+ */
+ private static final List EXPECTED_DEPTH_FIRST = Arrays.asList(
+ "arr",
+ "sub",
+ "sub/deep",
+ "sub/deep/deepArray",
+ "sub/nested"
+ );
+
+ /**
+ * A {@link MemoryStore} that counts how often it is asked to list or read, so that tests can
+ * assert on the number of store operations a group traversal costs.
+ */
+ static final class CountingStore implements Store, Store.ListableStore {
+
+ private final MemoryStore delegate = new MemoryStore();
+ final AtomicInteger listCalls = new AtomicInteger();
+ final AtomicInteger listChildrenCalls = new AtomicInteger();
+ final AtomicInteger readCalls = new AtomicInteger();
+
+ void resetCounters() {
+ listCalls.set(0);
+ listChildrenCalls.set(0);
+ readCalls.set(0);
+ }
+
+ @Override
+ public Stream list(String[] prefix) {
+ listCalls.incrementAndGet();
+ return delegate.list(prefix);
+ }
+
+ @Override
+ public Stream listChildren(String[] prefix) {
+ listChildrenCalls.incrementAndGet();
+ return delegate.listChildren(prefix);
+ }
+
+ @Override
+ public boolean exists(String[] keys) {
+ readCalls.incrementAndGet();
+ return delegate.exists(keys);
+ }
+
+ @Nullable
+ @Override
+ public ByteBuffer get(String[] keys) {
+ readCalls.incrementAndGet();
+ return delegate.get(keys);
+ }
+
+ @Nullable
+ @Override
+ public ByteBuffer get(String[] keys, long start) {
+ readCalls.incrementAndGet();
+ return delegate.get(keys, start);
+ }
+
+ @Nullable
+ @Override
+ public ByteBuffer get(String[] keys, long start, long end) {
+ readCalls.incrementAndGet();
+ return delegate.get(keys, start, end);
+ }
+
+ @Override
+ public void set(String[] keys, ByteBuffer bytes) {
+ delegate.set(keys, bytes);
+ }
+
+ @Override
+ public void delete(String[] keys) {
+ delegate.delete(keys);
+ }
+
+ @Nonnull
+ @Override
+ public StoreHandle resolve(String... keys) {
+ return new StoreHandle(this, keys);
+ }
+
+ @Override
+ public InputStream getInputStream(String[] keys, long start, long end) {
+ readCalls.incrementAndGet();
+ return delegate.getInputStream(keys, start, end);
+ }
+
+ @Override
+ public long getSize(String[] keys) {
+ return delegate.getSize(keys);
+ }
+
+ @Override
+ public String toString() {
+ return "";
+ }
+ }
+
+ /**
+ * A store that cannot be listed, so that its hierarchy cannot be discovered and therefore cannot
+ * be consolidated.
+ */
+ static final class NonListableStore implements Store {
+
+ private final MemoryStore delegate = new MemoryStore();
+
+ @Override
+ public boolean exists(String[] keys) {
+ return delegate.exists(keys);
+ }
+
+ @Nullable
+ @Override
+ public ByteBuffer get(String[] keys) {
+ return delegate.get(keys);
+ }
+
+ @Nullable
+ @Override
+ public ByteBuffer get(String[] keys, long start) {
+ return delegate.get(keys, start);
+ }
+
+ @Nullable
+ @Override
+ public ByteBuffer get(String[] keys, long start, long end) {
+ return delegate.get(keys, start, end);
+ }
+
+ @Override
+ public void set(String[] keys, ByteBuffer bytes) {
+ delegate.set(keys, bytes);
+ }
+
+ @Override
+ public void delete(String[] keys) {
+ delegate.delete(keys);
+ }
+
+ @Nonnull
+ @Override
+ public StoreHandle resolve(String... keys) {
+ return new StoreHandle(this, keys);
+ }
+
+ @Override
+ public InputStream getInputStream(String[] keys, long start, long end) {
+ return delegate.getInputStream(keys, start, end);
+ }
+
+ @Override
+ public long getSize(String[] keys) {
+ return delegate.getSize(keys);
+ }
+
+ @Override
+ public String toString() {
+ return "";
+ }
+ }
+
+ /**
+ * Writes a v3 hierarchy:
+ *
+ * / group
+ * /arr array (chunked)
+ * /sub group
+ * /sub/nested array
+ * /sub/deep group
+ * /sub/deep/deepArray array
+ *
+ */
+ static Group writeTreeV3(StoreHandle storeHandle) throws IOException, ZarrException {
+ Group root = Group.create(storeHandle);
+ byte[] data = new byte[64 * 64];
+ for (int i = 0; i < data.length; i++) {
+ data[i] = (byte) i;
+ }
+ dev.zarr.zarrjava.v3.Array array = root.createArray("arr", b -> b
+ .withShape(64, 64)
+ .withDataType(DataType.UINT8)
+ .withChunkShape(8, 8));
+ array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.BYTE, new int[]{64, 64}, data));
+
+ Group sub = root.createGroup("sub");
+ sub.createArray("nested", b -> b
+ .withShape(8, 8)
+ .withDataType(DataType.UINT8)
+ .withChunkShape(8, 8));
+ Group deep = sub.createGroup("deep");
+ deep.createArray("deepArray", b -> b
+ .withShape(8, 8)
+ .withDataType(DataType.UINT8)
+ .withChunkShape(8, 8));
+ return root;
+ }
+
+ /**
+ * The messages of an exception and all its causes, so that a test can assert on a message that
+ * Jackson has wrapped while parsing.
+ */
+ private static String exceptionMessages(Throwable throwable) {
+ StringBuilder messages = new StringBuilder();
+ for (Throwable current = throwable; current != null; current = current.getCause()) {
+ messages.append(current.getMessage()).append(' ');
+ }
+ return messages.toString();
+ }
+
+ private static ObjectNode readJson(StoreHandle handle) throws IOException {
+ ByteBuffer bytes = handle.resolve(ZARR_JSON).readNonNull();
+ return (ObjectNode) new ObjectMapper().readTree(Utils.toArray(bytes));
+ }
+
+ private static void writeJson(StoreHandle handle, JsonNode json) throws IOException {
+ handle.resolve(ZARR_JSON).set(ByteBuffer.wrap(new ObjectMapper().writeValueAsBytes(json)));
+ }
+
+ @Test
+ public void testConsolidateWritesAllDescendants() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ writeTreeV3(store.resolve()).consolidateMetadata();
+
+ ObjectNode written = readJson(store.resolve());
+ JsonNode consolidated = written.get("consolidated_metadata");
+ Assertions.assertEquals("inline", consolidated.get("kind").asText());
+ Assertions.assertFalse(consolidated.get("must_understand").asBoolean());
+
+ Set keys = new HashSet<>();
+ consolidated.get("metadata").fieldNames().forEachRemaining(keys::add);
+ Assertions.assertEquals(EXPECTED_ENTRIES, keys);
+
+ Assertions.assertEquals("array",
+ consolidated.get("metadata").get("sub/deep/deepArray").get("node_type").asText());
+ }
+
+ @Test
+ public void testConsolidatedEntriesMatchTheNodesThemselves() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ writeTreeV3(store.resolve()).consolidateMetadata();
+
+ JsonNode entries = readJson(store.resolve()).get("consolidated_metadata").get("metadata");
+ for (String key : EXPECTED_ENTRIES) {
+ ObjectNode fromNode = readJson(store.resolve(key.split("/")));
+ fromNode.remove("consolidated_metadata");
+ ObjectNode fromCache = (ObjectNode) entries.get(key).deepCopy();
+ fromCache.remove("consolidated_metadata");
+ Assertions.assertEquals(fromNode, fromCache, "the cached metadata of '" + key
+ + "' must be a verbatim copy of the metadata of the node");
+ }
+ }
+
+ @Test
+ public void testGetIsAnsweredWithoutReadingTheStore() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ writeTreeV3(store.resolve()).consolidateMetadata();
+
+ Group root = Group.open(store.resolve());
+ store.resetCounters();
+
+ Assertions.assertNotNull(root.get("arr"));
+ Assertions.assertNotNull(root.get(new String[]{"sub", "deep", "deepArray"}));
+ Assertions.assertEquals(0, store.readCalls.get(),
+ "a node held by the consolidated metadata must not be read from the store");
+ }
+
+ @Test
+ public void testSubgroupsAreAlsoConsolidated() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ writeTreeV3(store.resolve()).consolidateMetadata();
+
+ Group root = Group.open(store.resolve());
+ store.resetCounters();
+
+ Group sub = (Group) root.get("sub");
+ Assertions.assertNotNull(sub);
+ Group deep = (Group) sub.get("deep");
+ Assertions.assertNotNull(deep);
+ Assertions.assertNotNull(deep.get("deepArray"));
+ Assertions.assertEquals(0, store.readCalls.get(),
+ "walking into a subgroup must keep using the consolidated metadata of the root");
+ }
+
+ @Test
+ public void testListIsAnsweredWithoutTouchingTheStore() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ writeTreeV3(store.resolve()).consolidateMetadata();
+
+ Group root = Group.open(store.resolve());
+ store.resetCounters();
+
+ Assertions.assertEquals(EXPECTED_ENTRIES.size(), root.listAsArray().length);
+ Assertions.assertEquals(0, store.readCalls.get(), "no node metadata may be read again");
+ Assertions.assertEquals(0, store.listCalls.get(),
+ "a consolidated group must not list the store to find its members");
+ Assertions.assertEquals(0, store.listChildrenCalls.get());
+ }
+
+ @Test
+ public void testListIsOrderedDepthFirst() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ writeTreeV3(store.resolve()).consolidateMetadata();
+
+ Group root = Group.open(store.resolve());
+ List keys = new ArrayList<>();
+ for (dev.zarr.zarrjava.core.Node node : root.listAsArray()) {
+ keys.add(String.join("/", ((dev.zarr.zarrjava.core.AbstractNode) node).storeHandle.keys));
+ }
+ Assertions.assertEquals(EXPECTED_DEPTH_FIRST, keys);
+ }
+
+ @Test
+ public void testNodeAddedAfterConsolidatingIsNotFound() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ Group root = writeTreeV3(store.resolve()).consolidateMetadata();
+
+ root.createArray("late", b -> b
+ .withShape(4, 4)
+ .withDataType(DataType.UINT8)
+ .withChunkShape(4, 4));
+
+ Group reopened = Group.open(store.resolve());
+ store.resetCounters();
+ Assertions.assertNull(reopened.get("late"),
+ "the cache is authoritative, so a node it does not hold is reported as absent");
+ Assertions.assertNull(reopened.get("doesNotExist"));
+ Assertions.assertEquals(0, store.readCalls.get(),
+ "a key missing from the cache must not be looked up in the store");
+
+ // Bypassing the stale cache finds the node again.
+ Group fresh = Group.open(store.resolve(), Group.UseConsolidated.IGNORE);
+ Assertions.assertNotNull(fresh.get("late"));
+
+ // So does consolidating again.
+ Assertions.assertNotNull(Group.consolidateMetadata(store.resolve()).get("late"));
+ }
+
+ @Test
+ public void testSubgroupEntriesAreMarkedAsConsolidated() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ writeTreeV3(store.resolve()).consolidateMetadata();
+
+ JsonNode entries = readJson(store.resolve()).get("consolidated_metadata").get("metadata");
+ for (String key : Arrays.asList("sub", "sub/deep")) {
+ JsonNode nested = entries.get(key).get("consolidated_metadata");
+ Assertions.assertNotNull(nested, "the cached metadata of the subgroup '" + key
+ + "' must carry an empty cache, marking it as covered by the cache above it");
+ Assertions.assertEquals("inline", nested.get("kind").asText());
+ Assertions.assertFalse(nested.get("must_understand").asBoolean());
+ Assertions.assertEquals(0, nested.get("metadata").size());
+ }
+ // The subgroups themselves are untouched by consolidating the group above them.
+ Assertions.assertFalse(readJson(store.resolve("sub")).has("consolidated_metadata"));
+ }
+
+ @Test
+ public void testNestedConsolidatedMetadataIsEmptied() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ Group root = writeTreeV3(store.resolve());
+ ((Group) root.get("sub")).consolidateMetadata();
+ root.consolidateMetadata();
+
+ JsonNode entries = readJson(store.resolve()).get("consolidated_metadata").get("metadata");
+ JsonNode nested = entries.get("sub").get("consolidated_metadata");
+ Assertions.assertNotNull(nested, "the key must be kept, so that it stays visible that the"
+ + " subgroup is consolidated");
+ Assertions.assertEquals(0, nested.get("metadata").size(),
+ "the entries of a consolidated subgroup must not be duplicated inside the cache of"
+ + " the group above it");
+
+ // The subgroup keeps its own cache in its own metadata document.
+ Assertions.assertEquals(3,
+ readJson(store.resolve("sub")).get("consolidated_metadata").get("metadata").size());
+ }
+
+ @Test
+ public void testUnknownKindFailsToOpen() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ writeTreeV3(store.resolve()).consolidateMetadata();
+
+ ObjectNode written = readJson(store.resolve());
+ ((ObjectNode) written.get("consolidated_metadata")).put("kind", "something_else");
+ writeJson(store.resolve(), written);
+
+ // zarr-python rejects a cache of an unknown kind rather than ignoring it, so this library does
+ // the same.
+ Exception exception =
+ Assertions.assertThrows(Exception.class, () -> Group.open(store.resolve()));
+ Assertions.assertTrue(exceptionMessages(exception).contains("kind='something_else'"),
+ "the error must name the unsupported kind, got: " + exceptionMessages(exception));
+ }
+
+ @Test
+ public void testUnknownFieldInACachedEntryDoesNotBreakTheGroup() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ writeTreeV3(store.resolve()).consolidateMetadata();
+
+ ObjectNode written = readJson(store.resolve());
+ ObjectNode entry = (ObjectNode) written.get("consolidated_metadata").get("metadata").get("arr");
+ entry.putArray("some_future_field").add("value");
+ writeJson(store.resolve(), written);
+
+ // Opening the group must not fail because of a cache entry it cannot interpret.
+ Group root = Group.open(store.resolve());
+ store.resetCounters();
+ Assertions.assertNotNull(root.get("arr"),
+ "an entry that cannot be parsed must fall back to reading the node itself");
+ Assertions.assertTrue(store.readCalls.get() > 0);
+
+ // The other entries are unaffected.
+ store.resetCounters();
+ Assertions.assertNotNull(root.get(new String[]{"sub", "nested"}));
+ Assertions.assertEquals(0, store.readCalls.get());
+ }
+
+ @Test
+ public void testUnknownFieldSurvivesConsolidation() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ Group root = writeTreeV3(store.resolve());
+
+ ObjectNode arrayMetadata = readJson(store.resolve("arr"));
+ arrayMetadata.putArray("some_future_field").add("value");
+ writeJson(store.resolve("arr"), arrayMetadata);
+
+ root.consolidateMetadata();
+
+ JsonNode cached = readJson(store.resolve())
+ .get("consolidated_metadata").get("metadata").get("arr");
+ Assertions.assertEquals(arrayMetadata, cached,
+ "consolidating must copy the metadata of a node verbatim, including fields this"
+ + " library does not model");
+ }
+
+ @Test
+ public void testIgnoreDropsTheCache() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ writeTreeV3(store.resolve()).consolidateMetadata();
+
+ Group root = Group.open(store.resolve(), Group.UseConsolidated.IGNORE);
+ Assertions.assertNull(root.metadata.consolidatedMetadata,
+ "the cache must be dropped from the metadata held in memory");
+
+ store.resetCounters();
+ Assertions.assertNotNull(root.get("arr"));
+ Assertions.assertTrue(store.readCalls.get() > 0);
+ }
+
+ @Test
+ public void testIgnoreAppliesToTheOpenedGroupOnly() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ Group root = writeTreeV3(store.resolve());
+ ((Group) root.get("sub")).consolidateMetadata();
+ root.consolidateMetadata();
+
+ // The opt-out is not inherited: a subgroup that carries a cache of its own uses it, just as it
+ // would if it had been opened directly. This matches zarr-python, where use_consolidated
+ // applies to the group being opened.
+ Group ignored = Group.open(store.resolve(), Group.UseConsolidated.IGNORE);
+ Group sub = (Group) ignored.get("sub");
+ Assertions.assertNotNull(sub.metadata.consolidatedMetadata);
+
+ store.resetCounters();
+ Assertions.assertNotNull(sub.get(new String[]{"deep", "deepArray"}));
+ Assertions.assertEquals(0, store.readCalls.get());
+ }
+
+ @Test
+ public void testRequireFailsWithoutACache() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ writeTreeV3(store.resolve());
+
+ ZarrException exception = Assertions.assertThrows(ZarrException.class,
+ () -> Group.openConsolidated(store.resolve()));
+ Assertions.assertTrue(exception.getMessage().contains("REQUIRE"), exception.getMessage());
+
+ // With a cache in place the same call succeeds.
+ Group.consolidateMetadata(store.resolve());
+ Group root = Group.openConsolidated(store.resolve());
+ Assertions.assertNotNull(root.metadata.consolidatedMetadata);
+ Assertions.assertNotNull(root.get(new String[]{"sub", "deep", "deepArray"}));
+ }
+
+ @Test
+ public void testStaticConsolidateMetadataOpensAndWrites() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ writeTreeV3(store.resolve());
+
+ Group root = Group.consolidateMetadata(store.resolve());
+ Assertions.assertNotNull(root.metadata.consolidatedMetadata);
+ Assertions.assertEquals(EXPECTED_ENTRIES, root.metadata.consolidatedMetadata.metadata.keySet());
+ Assertions.assertEquals(EXPECTED_ENTRIES.size(),
+ readJson(store.resolve()).get("consolidated_metadata").get("metadata").size());
+
+ // Consolidating a subtree only covers that subtree.
+ Group sub = Group.consolidateMetadata(store.resolve("sub"));
+ Assertions.assertEquals(new HashSet<>(Arrays.asList("nested", "deep", "deep/deepArray")),
+ sub.metadata.consolidatedMetadata.metadata.keySet());
+ }
+
+ @Test
+ public void testConsolidatingANonListableStoreFails() throws IOException, ZarrException {
+ Group group = Group.create(new NonListableStore().resolve());
+ UnsupportedOperationException exception = Assertions.assertThrows(
+ UnsupportedOperationException.class, group::consolidateMetadata,
+ "a store that cannot be listed cannot be consolidated");
+ Assertions.assertTrue(exception.getMessage().contains("NonListableStore"),
+ exception.getMessage());
+ }
+
+ @Test
+ public void testExplicitNullIsParsedAndNotWrittenBack() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ Group root = writeTreeV3(store.resolve());
+
+ ObjectNode written = readJson(store.resolve());
+ written.putNull("consolidated_metadata");
+ writeJson(store.resolve(), written);
+
+ Group reopened = Group.open(store.resolve());
+ Assertions.assertNull(reopened.metadata.consolidatedMetadata);
+
+ reopened.setAttributes(new Attributes().set("a", 1));
+ Assertions.assertFalse(readJson(store.resolve()).has("consolidated_metadata"),
+ "an absent cache must be omitted, never written as null");
+ }
+
+ @Test
+ public void testDropConsolidatedMetadata() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ Group root = writeTreeV3(store.resolve()).consolidateMetadata();
+
+ root.dropConsolidatedMetadata();
+ Assertions.assertNull(root.metadata.consolidatedMetadata);
+ Assertions.assertFalse(readJson(store.resolve()).has("consolidated_metadata"));
+
+ store.resetCounters();
+ Assertions.assertNotNull(root.get("arr"));
+ Assertions.assertTrue(store.readCalls.get() > 0);
+ }
+
+ @Test
+ public void testAttributesUpdateKeepsTheCache() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ Group root = writeTreeV3(store.resolve()).consolidateMetadata();
+
+ root.setAttributes(new Attributes().set("answer", 42));
+
+ ObjectNode written = readJson(store.resolve());
+ Assertions.assertEquals(42, written.get("attributes").get("answer").asInt());
+ Assertions.assertEquals(EXPECTED_ENTRIES.size(),
+ written.get("consolidated_metadata").get("metadata").size(),
+ "changing the attributes of the group does not change its descendants");
+ }
+
+ @Test
+ public void testConsolidatingIsReproducible() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ Group root = writeTreeV3(store.resolve());
+
+ root.consolidateMetadata();
+ byte[] first = Utils.toArray(store.resolve().resolve(ZARR_JSON).readNonNull());
+ Group.open(store.resolve()).consolidateMetadata();
+ byte[] second = Utils.toArray(store.resolve().resolve(ZARR_JSON).readNonNull());
+
+ Assertions.assertArrayEquals(first, second);
+ }
+
+ @Test
+ public void testEntriesAreOrderedByDepthThenName() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ writeTreeV3(store.resolve()).consolidateMetadata();
+
+ List keys = new ArrayList<>();
+ readJson(store.resolve()).get("consolidated_metadata").get("metadata")
+ .fieldNames().forEachRemaining(keys::add);
+ Assertions.assertEquals(
+ Arrays.asList("arr", "sub", "sub/deep", "sub/nested", "sub/deep/deepArray"), keys);
+ }
+
+ @Test
+ public void testConsolidatedMetadataOfAnEmptyGroup() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ Group root = Group.create(store.resolve()).consolidateMetadata();
+
+ Assertions.assertNotNull(root.metadata.consolidatedMetadata);
+ Assertions.assertTrue(root.metadata.consolidatedMetadata.isEmpty());
+ Assertions.assertEquals(0,
+ readJson(store.resolve()).get("consolidated_metadata").get("metadata").size(),
+ "a consolidated group without descendants keeps the key with an empty cache, so that"
+ + " it stays distinguishable from a group that was never consolidated");
+ }
+
+ @Test
+ public void testConsolidatedMetadataIsExposedOnTheMetadata() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ Group root = writeTreeV3(store.resolve()).consolidateMetadata();
+
+ ConsolidatedMetadata consolidated = root.metadata.consolidatedMetadata;
+ Assertions.assertNotNull(consolidated);
+ Assertions.assertEquals(EXPECTED_ENTRIES, consolidated.metadata.keySet());
+ Assertions.assertNotNull(consolidated.get(new String[]{"sub", "deep"}));
+ Assertions.assertEquals(new HashSet<>(Arrays.asList("nested", "deep", "deep/deepArray")),
+ consolidated.sub(new String[]{"sub"}).metadata.keySet());
+ }
+
+ @Test
+ public void testNodeOpenIgnoresTheCacheOfTheGroupItself() throws IOException, ZarrException {
+ CountingStore store = new CountingStore();
+ writeTreeV3(store.resolve()).consolidateMetadata();
+
+ // Opening the group through the generic entry point must give the same, usable group.
+ Group root = (Group) Node.open(store.resolve());
+ Assertions.assertNotNull(root.metadata.consolidatedMetadata);
+ Assertions.assertNotNull(root.get(new String[]{"sub", "nested"}));
+ }
+}
diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java
index 5c22b5fc..b0111b8e 100644
--- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java
+++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java
@@ -8,6 +8,7 @@
import dev.zarr.zarrjava.v2.Group;
import dev.zarr.zarrjava.v3.Array;
import dev.zarr.zarrjava.v3.ArrayMetadataBuilder;
+import dev.zarr.zarrjava.v3.ConsolidatedMetadata;
import dev.zarr.zarrjava.v3.DataType;
import dev.zarr.zarrjava.v3.codec.CodecBuilder;
import org.junit.jupiter.api.Assertions;
@@ -24,6 +25,7 @@
import java.nio.ByteBuffer;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.stream.Stream;
@@ -355,4 +357,37 @@ public void testGroupReadWriteV3() throws Exception {
Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape());
assertIsTestdata(result, dataType);
}
+
+ /**
+ * Checks that the consolidated metadata written by zarr-java is understood by zarr-python and the
+ * other way round.
+ */
+ @Test
+ public void testConsolidatedMetadataReadWriteV3() throws Exception {
+ StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testConsolidatedMetadataV3", "write");
+ StoreHandle storeHandle2 = new FilesystemStore(TESTOUTPUT).resolve("testConsolidatedMetadataV3", "read");
+
+ ConsolidatedMetadataTest.writeTreeV3(storeHandle).consolidateMetadata();
+
+ run_python_script("zarr_python_consolidate.py", storeHandle.toPath().toString(),
+ storeHandle2.toPath().toString());
+
+ dev.zarr.zarrjava.v3.Group group = dev.zarr.zarrjava.v3.Group.open(storeHandle2);
+ ConsolidatedMetadata consolidated = group.metadata.consolidatedMetadata;
+ Assertions.assertNotNull(consolidated, "zarr-java did not pick up the consolidated metadata"
+ + " written by zarr-python");
+ Assertions.assertEquals(
+ Arrays.asList("arr", "sub", "sub/deep", "sub/nested", "sub/deep/deepArray"),
+ new ArrayList<>(consolidated.metadata.keySet()));
+
+ dev.zarr.zarrjava.v3.Array array =
+ (dev.zarr.zarrjava.v3.Array) group.get(new String[]{"sub", "deep", "deepArray"});
+ Assertions.assertNotNull(array);
+ Assertions.assertArrayEquals(new long[]{8, 8}, array.metadata().shape);
+
+ dev.zarr.zarrjava.v3.Array topArray = (dev.zarr.zarrjava.v3.Array) group.get("arr");
+ Assertions.assertNotNull(topArray);
+ Assertions.assertArrayEquals(new int[]{64, 64}, topArray.read().getShape());
+ Assertions.assertEquals(5, group.listAsArray().length);
+ }
}
diff --git a/src/test/python-scripts/zarr_python_consolidate.py b/src/test/python-scripts/zarr_python_consolidate.py
new file mode 100644
index 00000000..28009f21
--- /dev/null
+++ b/src/test/python-scripts/zarr_python_consolidate.py
@@ -0,0 +1,34 @@
+import sys
+from pathlib import Path
+
+import numpy as np
+import zarr
+from zarr.storage import LocalStore
+
+store_path_read = Path(sys.argv[1])
+store_path_write = Path(sys.argv[2])
+
+expected_members = ["arr", "sub", "sub/deep", "sub/deep/deepArray", "sub/nested"]
+
+# Read a hierarchy that zarr-java consolidated.
+g = zarr.open_group(store=LocalStore(store_path_read), zarr_format=3, use_consolidated=True)
+consolidated = g.metadata.consolidated_metadata
+assert consolidated is not None, "zarr-python did not pick up the consolidated metadata"
+# zarr-python re-nests the flat keys when it reads them, so the top level only holds the direct
+# children and the rest lives in the caches it builds for the subgroups.
+assert list(consolidated.metadata.keys()) == ["arr", "sub"], list(consolidated.metadata.keys())
+members = sorted(k for k, _ in g.members(max_depth=None))
+assert members == expected_members, f"got {members}, expected {expected_members}"
+assert g["arr"].shape == (64, 64), g["arr"].shape
+assert g["sub"]["deep"]["deepArray"].shape == (8, 8)
+
+# Write a hierarchy of the same shape and consolidate it, for zarr-java to read.
+g2 = zarr.create_group(store=LocalStore(store_path_write), zarr_format=3)
+g2.attrs["attr"] = "value"
+arr = g2.create_array(name="arr", shape=(64, 64), chunks=(8, 8), dtype="uint8", fill_value=0)
+arr[:] = np.arange(64 * 64, dtype="uint8").reshape(64, 64)
+sub = g2.create_group("sub")
+sub.create_array(name="nested", shape=(8, 8), chunks=(8, 8), dtype="uint8", fill_value=0)
+deep = sub.create_group("deep")
+deep.create_array(name="deepArray", shape=(8, 8), chunks=(8, 8), dtype="uint8", fill_value=0)
+zarr.consolidate_metadata(g2.store)