Consolidated metadata (Zarr V3)

USERGUIDE.md

@@ -288,6 +288,61 @@ Array array = group.createArray(
         .build()
 );
 ```
+### Consolidated Metadata (v3)
+A group can keep a copy of the metadata of all of its descendants inside its own `zarr.json`, so that
+the whole hierarchy can be opened with a single read instead of one read per node. This matters most
+over HTTP and S3, where every node otherwise costs a request.
+
+```java
+// Write the cache. This walks the hierarchy once and stores the metadata of every
+// descendant in the metadata of this group.
+Group root = Group.consolidateMetadata(storeHandle);
+Group root = Group.consolidateMetadata("/data/my.zarr");   // or a path
+
+// Consolidate a subtree instead of the whole hierarchy
+Group.consolidateMetadata(storeHandle.resolve("sub"));
+
+// The same, on a group that is already open
+Group root = Group.open(storeHandle).consolidateMetadata();
+
+// Reads and listings are answered from the cache, without touching the store.
+Group sub = (Group) root.get("sub");
+Array array = (Array) sub.get("nested");
+Node[] everything = root.listAsArray();
+
+// Remove the cache again
+root.dropConsolidatedMetadata();
+```
+
+On opening, the cache is used if the group has one. Pass `UseConsolidated` to say otherwise:
+
+```java
+Group auto = Group.open(storeHandle);                                   // use it if present
+Group required = Group.open(storeHandle, UseConsolidated.REQUIRE);      // fail if absent
+Group required = Group.openConsolidated(storeHandle);                   // the same, shorter
+Group fresh = Group.open(storeHandle, UseConsolidated.IGNORE);          // read every node instead
+```
+
+`UseConsolidated.IGNORE` drops the cache from the metadata held in memory, so writing that metadata
+again - with `setAttributes()` for example - removes the cache from the store as well. The choice
+applies to the group being opened only: a subgroup reached through `get()` uses a cache of its own if
+it has one.
+
+The cache is written in the same format as `zarr.consolidate_metadata()` in zarr-python, and behaves
+the same way on reading, so both libraries can read each other's output.
+
+**The cache is authoritative.** While a group is using its cache, `get()` and `list()` answer from it
+alone and never fall back to the store. A key the cache does not hold is reported as absent.
+
+**The cache is a snapshot.** Nothing invalidates it when a node is added, removed or changed
+afterwards, so it has to be written again after modifying the hierarchy. Until then a node added
+afterwards is invisible, and a node changed afterwards is served as it was. Open the group with
+`UseConsolidated.IGNORE` if in doubt.
+
+Consolidated metadata is a Zarr v3 feature here; the v2 `.zmetadata` file is not supported. Note that
+consolidated metadata is not part of the Zarr v3 specification, so other implementations may not
+support it; `consolidateMetadata()` logs a warning saying so, as zarr-python does.
+
 ### Hierarchical Example
 ```java
 Group root = Group.create(

src/main/java/dev/zarr/zarrjava/v3/ConsolidatedMetadata.java

@@ -0,0 +1,168 @@
+package dev.zarr.zarrjava.v3;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.JsonNode;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * An optional cache of the metadata of all descendants of a group, stored inside that group's own
+ * {@code zarr.json} under the {@code consolidated_metadata} key. It allows a reader to open a whole
+ * hierarchy with a single request instead of one request per node.
+ * <p>
+ * The keys of {@link #metadata} are flat, {@code "/"}-joined paths relative to the group holding the
+ * cache, for example {@code "ocean"} and {@code "ocean/salinity"}.
+ * <p>
+ * A cache of a kind other than {@link #KIND_INLINE} makes the group unopenable, which is what
+ * zarr-python does as well.
+ * <p>
+ * The cached node metadata is deliberately kept as raw {@link JsonNode} rather than as parsed
+ * {@link ArrayMetadata} / {@link GroupMetadata}. The cache is declared with
+ * {@code must_understand: false}, so a reader that cannot interpret an entry has to ignore it rather
+ * than fail. Parsing entries eagerly would mean that a single node written by another implementation
+ * with a field this library does not model would make the whole group unopenable. Keeping the raw
+ * JSON also lets {@link Group#consolidateMetadata()} copy each node's metadata verbatim, so the cache
+ * never silently loses information that is present in the node's own {@code zarr.json}.
+ * <p>
+ * The cache is a snapshot taken at the time of consolidation. Nothing invalidates it when a
+ * descendant changes, so {@link Group#consolidateMetadata()} has to be re-run after modifying the
+ * hierarchy.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public final class ConsolidatedMetadata {
+
+    /**
+     * The only cache kind defined so far: the metadata is stored inline in the group's metadata
+     * document. A cache of any other kind is rejected by this library.
+     */
+    public static final String KIND_INLINE = "inline";
+
+    @Nonnull
+    @JsonProperty("kind")
+    public final String kind;
+
+    @JsonProperty("must_understand")
+    public final boolean mustUnderstand;
+
+    /**
+     * The cached metadata documents, keyed by their {@code "/"}-joined path relative to the group
+     * holding this cache.
+     */
+    @Nonnull
+    @JsonProperty("metadata")
+    public final Map<String, JsonNode> metadata;
+
+    @JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
+    public ConsolidatedMetadata(
+            @Nullable @JsonProperty("kind") String kind,
+            @Nullable @JsonProperty("must_understand") Boolean mustUnderstand,
+            @Nullable @JsonProperty("metadata") Map<String, JsonNode> metadata
+    ) {
+        this.kind = kind == null ? KIND_INLINE : kind;
+        this.mustUnderstand = mustUnderstand != null && mustUnderstand;
+        this.metadata = metadata == null
+                ? Collections.emptyMap()
+                : Collections.unmodifiableMap(new LinkedHashMap<>(metadata));
+    }
+
+    public ConsolidatedMetadata(@Nonnull Map<String, JsonNode> metadata) {
+        this(KIND_INLINE, false, metadata);
+    }
+
+    /**
+     * An empty inline cache, used for a consolidated subgroup whose own entries have been hoisted
+     * into the cache of an ancestor.
+     */
+    public static ConsolidatedMetadata empty() {
+        return new ConsolidatedMetadata(Collections.emptyMap());
+    }
+
+    /**
+     * Whether this cache is stored inline and can therefore be used by this library. A group whose
+     * cache is not inline cannot be opened, see {@link GroupMetadata}.
+     */
+    @JsonIgnore
+    public boolean isInline() {
+        return KIND_INLINE.equals(kind);
+    }
+
+    @JsonIgnore
+    public boolean isEmpty() {
+        return metadata.isEmpty();
+    }
+
+    /**
+     * Returns the cached metadata document for a node, or null if this cache does not hold it.
+     *
+     * @param key the path of the node relative to the group holding this cache
+     */
+    @Nullable
+    public JsonNode get(String[] key) {
+        if (!isInline()) {
+            return null;
+        }
+        return metadata.get(String.join("/", key));
+    }
+
+    /**
+     * Returns the keys of this cache with every group listed before its own descendants, and
+     * siblings in the order they are held in this cache. This is the order in which zarr-python
+     * yields the members of a consolidated group, because it walks the nested representation it
+     * builds when reading the cache.
+     */
+    public List<String> depthFirstKeys() {
+        Map<String, List<String>> childrenByParent = new LinkedHashMap<>();
+        for (String key : metadata.keySet()) {
+            int lastSlash = key.lastIndexOf('/');
+            String parent = lastSlash < 0 ? "" : key.substring(0, lastSlash);
+            childrenByParent.computeIfAbsent(parent, unused -> new ArrayList<>()).add(key);
+        }
+        Set<String> ordered = new LinkedHashSet<>();
+        appendDepthFirst("", childrenByParent, ordered);
+        // A key whose parent is missing from the cache is not reachable from the group, so it is not
+        // visited above. Append it, so that a malformed cache hides nothing.
+        ordered.addAll(metadata.keySet());
+        return new ArrayList<>(ordered);
+    }
+
+    private static void appendDepthFirst(String parent, Map<String, List<String>> childrenByParent,
+                                        Set<String> out) {
+        List<String> children = childrenByParent.get(parent);
+        if (children == null) {
+            return;
+        }
+        for (String child : children) {
+            out.add(child);
+            appendDepthFirst(child, childrenByParent, out);
+        }
+    }
+
+    /**
+     * Returns the entries below {@code prefix} with the prefix stripped from their keys, so that the
+     * result can serve as the cache of the subgroup at {@code prefix}.
+     */
+    public ConsolidatedMetadata sub(String[] prefix) {
+        if (!isInline()) {
+            return empty();
+        }
+        String keyPrefix = String.join("/", prefix) + "/";
+        Map<String, JsonNode> sub = new LinkedHashMap<>();
+        for (Map.Entry<String, JsonNode> entry : metadata.entrySet()) {
+            if (entry.getKey().startsWith(keyPrefix)) {
+                sub.put(entry.getKey().substring(keyPrefix.length()), entry.getValue());
+            }
+        }
+        return new ConsolidatedMetadata(sub);
+    }
+}