Custom hierarchies – other API concept, for now without extended meshes and particles path

.github/workflows/linux.yml

@@ -300,6 +300,7 @@ jobs:
         find ../../../build/samples/      \
             ! -path '*thetaMode*'         \
             ! -path '/*many_iterations/*' \
+            ! -path '/*custom_hierarchy/*' \
             ! -name 'profiling.json'      \
             ! -name '*config.json'        \
             -iname '*.json'               \

CMakeLists.txt

@@ -402,6 +402,7 @@ include(${openPMD_SOURCE_DIR}/cmake/dependencies/pybind11.cmake)
 set(CORE_SOURCE
         src/config.cpp
         src/ChunkInfo.cpp
+        src/CustomHierarchy.cpp
         src/Dataset.cpp
         src/Datatype.cpp
         src/Error.cpp
@@ -430,6 +431,7 @@ set(CORE_SOURCE
         src/backend/BaseRecord.cpp
         src/backend/BaseRecordComponent.cpp
         src/backend/Container.cpp
+        src/backend/HierarchyVisitor.cpp
         src/backend/MeshRecordComponent.cpp
         src/backend/PatchRecord.cpp
         src/backend/PatchRecordComponent.cpp
@@ -867,6 +869,7 @@ if(openPMD_BUILD_TESTING)
             list(APPEND ${out_list}
                 test/Files_Core/automatic_variable_encoding.cpp
                 test/Files_Core/read_nonexistent_attribute.cpp
+                test/Files_Core/custom_hierarchy.cpp
             )
         endif()
     endmacro()

include/openPMD/CustomHierarchy.hpp

@@ -0,0 +1,268 @@
+/* Copyright 2023 Franz Poeschel
+ *
+ * This file is part of openPMD-api.
+ *
+ * openPMD-api is free software: you can redistribute it and/or modify
+ * it under the terms of of either the GNU General Public License or
+ * the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * openPMD-api is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License and the GNU Lesser General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * and the GNU Lesser General Public License along with openPMD-api.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#include "openPMD/Error.hpp"
+#include "openPMD/IO/AbstractIOHandler.hpp"
+#include "openPMD/Mesh.hpp"
+#include "openPMD/ParticleSpecies.hpp"
+#include "openPMD/RecordComponent.hpp"
+#include "openPMD/auxiliary/Export.hpp"
+#include "openPMD/auxiliary/TypeTraits.hpp"
+#include "openPMD/backend/Attributable.hpp"
+#include "openPMD/backend/Container.hpp"
+#include "openPMD/backend/Writable.hpp"
+
+#include <stdexcept>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+namespace openPMD
+{
+class CustomHierarchy;
+namespace internal
+{
+    struct CustomHierarchyData : ContainerData<CustomHierarchy>
+    {
+        bool has_been_read = false;
+    };
+} // namespace internal
+
+class CustomDataset : public RecordComponent
+{
+    template <typename>
+    friend class ConvertibleContainer;
+    friend class CustomHierarchy;
+    friend class Attributable;
+};
+
+/*
+ * This is its own class, so the return value of asContainerOf() is also
+ * convsertible again.
+ */
+template <typename MappedType>
+class ConvertibleContainer : public Container<MappedType>
+{
+    template <typename>
+    friend class ConvertibleContainer;
+    friend class CustomHierarchy;
+
+protected:
+    using Container_t = Container<MappedType>;
+    using Data_t = internal::ContainerData<MappedType>;
+    static_assert(
+        std::is_base_of_v<typename Container_t::ContainerData, Data_t>);
+
+    using Container_t::Container_t;
+
+private:
+    explicit ConvertibleContainer() = default;
+    ConvertibleContainer(Attributable::NoInit)
+        : Container_t(Attributable::NoInit{})
+    {}
+
+public:
+    auto isDataset() -> bool;
+    auto asDataset() -> CustomDataset;
+
+    auto datasets() -> ConvertibleContainer<CustomDataset>;
+    auto subgroups() -> CustomHierarchy;
+
+    void refresh();
+};
+
+namespace traits
+{
+    template <>
+    struct ElementAccessPolicy<CustomHierarchy>
+    {
+        static void call(CustomHierarchy &);
+        static void call(Container<CustomHierarchy> const &);
+    };
+
+    namespace detail
+    {
+        template <typename Container, typename Iterator>
+        void emplace_object_as_customly_managed(Container &cont, Iterator &it)
+        {
+            auto cast_to_custom_hierarchy = [&it]() {
+                using MappedType = std::remove_cv_t<
+                    std::remove_reference_t<decltype(it->second)>>;
+                if constexpr (std::is_same_v<MappedType, CustomHierarchy>)
+                {
+                    return it->second;
+                }
+                else
+                {
+                    return it->second.customHierarchies();
+                }
+            };
+            auto &writable = it->second.writable();
+
+            // These should be different
+            auto child_shared_data = &it->second.Attributable::get();
+            auto parent_shared_data = &cont.Attributable::get();
+            if (child_shared_data == parent_shared_data)
+            {
+                throw std::runtime_error(
+                    "Trying to emplace object as its own child");
+            }
+
+            // These might be different, but might also be the same
+            //
+            // For an explanation, ref. the documentation of
+            // Writable::attributable: This is a pointer back to the first
+            // created Attributable instance linking this Writable. There might
+            // be multiple Attributable objects linking the same backend
+            // Writable object when opening multiple "views" on the same backend
+            // object, e.g. when a scalar Record is at the same time a
+            // RecordComponent, or when reopening an object as a
+            // CustomHierarchy.
+            //
+            // Since CustomHierarchy performs no memory management by default,
+            // we must ensure that the backpointer in Writable::attributable
+            // remains valid when the frontend instance pointed by
+            // Writable::attributable *is* the CustomHierarchy instance (happens
+            // when it is the first frontend object created for that backend
+            // object).
+            auto backpointer = writable.attributable;
+            auto emplaced_pointer = it->second.m_attri.get();
+            auto *group_metadata =
+                (**cont.m_attri).m_writable.objectType.requireGroup();
+            if (backpointer == emplaced_pointer)
+            {
+                group_metadata
+                    ->m_children_managed_as_custom_hierarchy[it->first] =
+                    // NO move!! The iterator must stay alive
+                    std::make_shared<CustomHierarchy>(
+                        cast_to_custom_hierarchy());
+            }
+        }
+    } // namespace detail
+
+    template <>
+    struct GenerationPolicy<CustomHierarchy>
+    {
+        constexpr static bool is_noop = false;
+        template <typename Container, typename Iterator>
+        void operator()(Container &cont, Iterator &it)
+        {
+            detail::emplace_object_as_customly_managed<Container, Iterator>(
+                cont, it);
+        }
+    };
+
+    template <>
+    struct GenerationPolicy<CustomDataset>
+    {
+        constexpr static bool is_noop = false;
+        template <typename Container, typename Iterator>
+        void operator()(Container &cont, Iterator &it)
+        {
+            detail::emplace_object_as_customly_managed<Container, Iterator>(
+                cont, it);
+        }
+    };
+} // namespace traits
+
+class CustomHierarchy : public ConvertibleContainer<CustomHierarchy>
+{
+    friend class Iteration;
+    friend class Container<CustomHierarchy>;
+    friend class Attributable;
+    friend struct traits::ElementAccessPolicy<CustomHierarchy>;
+    template <typename, typename, typename>
+    friend class Container;
+
+private:
+    using Parent_t = ConvertibleContainer<CustomHierarchy>;
+    using Container_t = typename Parent_t::Container_t;
+    using Data_t = internal::CustomHierarchyData;
+
+    std::shared_ptr<Data_t> m_customHierarchyData;
+
+    inline void setData(std::shared_ptr<Data_t> customHierarchyData)
+    {
+        m_customHierarchyData = std::move(customHierarchyData);
+        ConvertibleContainer<CustomHierarchy>::setData(m_customHierarchyData);
+    }
+
+    inline auto get() -> Data_t &
+    {
+        return *m_customHierarchyData;
+    }
+
+    [[nodiscard]] inline auto get() const -> Data_t const &
+    {
+        return *m_customHierarchyData;
+    }
+
+protected:
+    CustomHierarchy(NoInit);
+    CustomHierarchy(std::shared_ptr<internal::SharedAttributableData> other);
+    CustomHierarchy(Attributable const &other);
+
+    void flush_internal(
+        internal::FlushParams const &, std::vector<std::string> currentPath);
+    void flush(std::string const &path, internal::FlushParams const &);
+
+    /**
+     * @brief Link with parent.
+     *
+     * @param w The Writable representing the parent.
+     */
+    void linkHierarchy(Writable &w) override;
+
+public:
+    CustomHierarchy();
+
+    CustomHierarchy(CustomHierarchy const &other) = default;
+    CustomHierarchy(CustomHierarchy &&other) = default;
+
+    CustomHierarchy &operator=(CustomHierarchy const &) = default;
+    CustomHierarchy &operator=(CustomHierarchy &&) = default;
+
+    inline operator CustomDataset()
+    {
+        return asDataset();
+    }
+
+    // TODO should we automatically read upon returning / instantiating a
+    // CustomHierarchy object? i.e. upon Attributable::customHierarchy() and
+    // CustomHierarchy::operator[]().
+    //
+    // set max_recursion_depth = 0 for infinite cycling
+    // recursion depth includes the current object
+    // recursion will not continue expanding into regions that are already known
+    // (hence not transitively expand into unknown subregions of known regions)
+    auto read(size_t max_recursion_depth = 1) -> CustomHierarchy &;
+
+    void printRecursively();
+
+private:
+    void printRecursively(std::string indent);
+
+protected:
+    OPENPMDAPI_EXPORT void
+    visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override;
+};
+} // namespace openPMD

include/openPMD/Iteration.hpp

@@ -153,14 +153,14 @@ namespace internal
 
 class Meshes : public Container<Mesh>
 {
-public:
-    void visitHierarchy(HierarchyVisitor &v, bool recursive) override;
+protected:
+    void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override;
 };
 
 class Particles : public Container<ParticleSpecies>
 {
-public:
-    void visitHierarchy(HierarchyVisitor &v, bool recursive) override;
+protected:
+    void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override;
 };
 
 /** @brief  Logical compilation of data from one snapshot (e.g. a single
@@ -301,8 +301,6 @@ class Iteration
     [[deprecated("This attribute is no longer set by the openPMD-api.")]] bool
     closedByWriter() const;
 
-    void visitHierarchy(HierarchyVisitor &v, bool recursive) override;
-
     Meshes meshes{};
     Particles particles{};
 
@@ -478,6 +476,8 @@ class Iteration
 protected:
     void scientificDefaults_impl(
         internal::WriteOrRead, OpenpmdStandard) override;
+
+    void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override;
 }; // Iteration
 
 namespace traits
@@ -486,8 +486,8 @@ namespace traits
     struct GenerationPolicy<Iteration>
     {
         constexpr static bool is_noop = false;
-        template <typename Iterator>
-        void operator()(Iterator &it)
+        template <typename Container, typename Iterator>
+        void operator()(Container &, Iterator &it)
         {
             it->second.get().m_iterationIndex = it->first;
         }
@@ -544,7 +544,7 @@ class IndexedIteration : public Iteration
 
 class Iterations : public Container<Iteration, Iteration::IterationIndex_t>
 {
-public:
-    void visitHierarchy(HierarchyVisitor &v, bool recursive) override;
+protected:
+    void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override;
 };
 } // namespace openPMD

include/openPMD/Mesh.hpp

@@ -327,8 +327,6 @@ class Mesh : public BaseRecord<MeshRecordComponent>
         typename = std::enable_if_t<std::is_floating_point<T>::value>>
     Mesh &setTimeOffset(T timeOffset);
 
-    void visitHierarchy(HierarchyVisitor &v, bool recursive) override;
-
 private:
     Mesh();
 
@@ -340,6 +338,8 @@ class Mesh : public BaseRecord<MeshRecordComponent>
 protected:
     void scientificDefaults_impl(
         internal::WriteOrRead, OpenpmdStandard) override;
+
+    void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override;
 }; // Mesh
 
 template <typename T>

include/openPMD/ParticlePatches.hpp

@@ -38,11 +38,12 @@ class ParticlePatches : public Container<PatchRecord>
     size_t numPatches() const;
     ~ParticlePatches() override = default;
 
-    void visitHierarchy(HierarchyVisitor &v, bool recursive) override;
-
 private:
     ParticlePatches() = default;
     void read();
+
+protected:
+    void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override;
 }; // ParticlePatches
 
 } // namespace openPMD