Merge pull request LLNL#1047 from LLNL/feature/nselliott/layout-proto…

…cols

Add new sidre Group output protocols for metadata

src/axom/sidre/core/Buffer.cpp

@@ -257,12 +257,7 @@ void Buffer::print(std::ostream& os) const
  */
 void Buffer::exportTo(conduit::Node& data_holder)
 {
-  data_holder["id"] = m_index;
-
-  if(isDescribed())
-  {
-    data_holder["schema"] = m_node.schema().to_json();
-  }
+  exportMetadata(data_holder);
 
   // If Buffer is allocated, export it's node's data
   if(isAllocated())
@@ -279,6 +274,23 @@ void Buffer::exportTo(conduit::Node& data_holder)
   }
 }
 
+/*
+ *************************************************************************
+ *
+ * Export metadata of the buffer into a conduit node
+ *
+ *************************************************************************
+ */
+void Buffer::exportMetadata(conduit::Node& data_holder)
+{
+  data_holder["id"] = m_index;
+
+  if(isDescribed())
+  {
+    data_holder["schema"] = m_node.schema().to_json();
+  }
+}
+
 /*
  *************************************************************************
  *

src/axom/sidre/core/Buffer.hpp

@@ -251,6 +251,11 @@ class Buffer
    */
   void exportTo(conduit::Node& data_holder);
 
+  /*!
+   * \brief Exports Buffer's metadata to a Conduit node.
+   */
+  void exportMetadata(conduit::Node& data_holder);
+
   /*!
    * \brief Import Buffer's state from a Conduit node.
    */

src/axom/sidre/core/Group.cpp

@@ -1208,6 +1208,43 @@ bool Group::createNativeLayout(Node& n, const Attribute* attr) const
   return hasSavedViews;
 }
 
+/*
+ *************************************************************************
+ *
+ * Copy Group layout to the given Conduit node, including only the metadata
+ * for Views and not the actual data the Views hold.
+ *
+ *************************************************************************
+ */
+
+void Group::createNoDataLayout(Node& n, const Attribute* attr) const
+{
+  n.set(DataType::object());
+
+  // Dump the group's views
+  for(auto& view : views())
+  {
+    // Check that the view's name is not also a child group name
+    SLIC_CHECK_MSG(m_is_list || !hasChildGroup(view.getName()),
+                   SIDRE_GROUP_LOG_PREPEND
+                     << "'" << view.getName()
+                     << "' is the name of both a group and a view.");
+
+    if(attr == nullptr || view.hasAttributeValue(attr))
+    {
+      conduit::Node& child_node = m_is_list ? n.append() : n[view.getName()];
+      view.copyMetadataToNode(child_node);
+    }
+  }
+
+  // Recursively dump the child groups
+  for(auto& group : groups())
+  {
+    conduit::Node& child_node = m_is_list ? n.append() : n[group.getName()];
+    group.createNoDataLayout(child_node, attr);
+  }
+}
+
 /*
  *************************************************************************
  *
@@ -1490,6 +1527,14 @@ void Group::save(const std::string& path,
     n["sidre_group_name"] = m_name;
     conduit::relay::io::save(n, path, "json");
   }
+  else if(protocol == "sidre_layout_json")
+  {
+    Node n;
+    exportWithoutBufferData(n["sidre"], attr);
+    ds->saveAttributeLayout(n["sidre/attribute"]);
+    n["sidre_group_name"] = m_name;
+    conduit::relay::io::save(n, path, "conduit_json");
+  }
   else if(protocol == "conduit_hdf5")
   {
     Node n;
@@ -1505,6 +1550,13 @@ void Group::save(const std::string& path,
     n["sidre_group_name"] = m_name;
     conduit::relay::io::save(n, path, protocol);
   }
+  else if(protocol == "conduit_layout_json")
+  {
+    Node n;
+    createNoDataLayout(n, attr);
+    n["sidre_group_name"] = m_name;
+    conduit::relay::io::save(n, path, "conduit_json");
+  }
   else
   {
     SLIC_ERROR(SIDRE_GROUP_LOG_PREPEND << "Invalid protocol '" << protocol
@@ -1975,7 +2027,9 @@ Group* Group::detachGroup(IndexType idx)
  *
  *************************************************************************
  */
-bool Group::exportTo(conduit::Node& result, const Attribute* attr) const
+bool Group::exportTo(conduit::Node& result,
+                     const Attribute* attr,
+                     bool export_buffer) const
 {
   result.set(DataType::object());
   // TODO - This implementation will change in the future.  We want to write
@@ -2007,7 +2061,14 @@ bool Group::exportTo(conduit::Node& result, const Attribute* attr) const
       std::ostringstream oss;
       oss << "buffer_id_" << *s_it;
       Node& n_buffer = bnode.fetch(oss.str());
-      getDataStore()->getBuffer(*s_it)->exportTo(n_buffer);
+      if(export_buffer)
+      {
+        getDataStore()->getBuffer(*s_it)->exportTo(n_buffer);
+      }
+      else
+      {
+        getDataStore()->getBuffer(*s_it)->exportMetadata(n_buffer);
+      }
     }
   }
 
@@ -2077,6 +2138,21 @@ bool Group::exportTo(conduit::Node& result,
   return hasSavedViews;
 }
 
+/*
+ *************************************************************************
+ *
+ * Exports the contents of the Group, excluding the data arrays held
+ * by any Buffers attached to the Views.
+ *
+ *************************************************************************
+ */
+
+bool Group::exportWithoutBufferData(conduit::Node& result,
+                                    const Attribute* attr) const
+{
+  return exportTo(result, attr, false);
+}
+
 /*
  *************************************************************************
  *

src/axom/sidre/core/Group.hpp

@@ -1153,7 +1153,7 @@ class Group
   void copyToConduitNode(Node& n) const;
 
   /*!
-   * \brief Copy data Group native layout to given Conduit node.
+   * \brief Copy Group's native layout to given Conduit node.
    *
    * The native layout is a Conduit Node hierarchy that maps the Conduit Node
    * data
@@ -1167,6 +1167,21 @@ class Group
    */
   bool createNativeLayout(Node& n, const Attribute* attr = nullptr) const;
 
+  /*!
+   * \brief Copy Group's layout to given Conduit node without data
+   *
+   * This method copies only a metadata version of the Group's hierarchical
+   * layout to a Conduit Node.  All of the Groups and Views in the
+   * hierarchy will be represented, but the actual data held by the Views
+   * will not be present.  For every View, the Conduit schema describing
+   * its datatype will be copied but not the data.  This is intended to
+   * provide an object that can be sent to a readable output format where
+   * the overall layout of the hierarchy can be seen without sending large
+   * arrays or other data to the output.
+   *
+   */
+  void createNoDataLayout(Node& n, const Attribute* attr = nullptr) const;
+
   /*!
    * \brief Copy data Group external layout to given Conduit node.
    *
@@ -1587,10 +1602,15 @@ class Group
    *
    * Note: This is for the "sidre_{zzz}" protocols.
    *
+   * \param export_buffers  Optional parameter, if set to false, the data
+   *                       arrays owned by Buffers will not be copied.
+   *
    * \return True if the group or any of its children have saved Views,
    * false otherwise.
    */
-  bool exportTo(conduit::Node& result, const Attribute* attr) const;
+  bool exportTo(conduit::Node& result,
+                const Attribute* attr,
+                bool export_buffers = true) const;
 
   /*!
    * \brief Private method to copy Group to Conduit Node.
@@ -1605,6 +1625,12 @@ class Group
                 const Attribute* attr,
                 std::set<IndexType>& buffer_indices) const;
 
+  /*!
+   * \brief Private method  exports the Group to a conduit Node without
+   * the data held by Buffers, enabling a save that shows the layout only
+   */
+  bool exportWithoutBufferData(conduit::Node& result, const Attribute* attr) const;
+
   /*!
    * \brief Private method to build a Group hierarchy from Conduit Node.
    *

src/axom/sidre/core/View.cpp

@@ -858,6 +858,20 @@ void View::createNativeLayout(Node& n) const
   n.set_external(m_node.schema(), data_ptr);
 }
 
+/*
+ *************************************************************************
+ *
+ * Copy the metadata for the View.
+ *
+ *************************************************************************
+ */
+void View::copyMetadataToNode(Node& n) const
+{
+  n["state"] = getStateStringName(m_state);
+  n["schema"] = m_schema.to_json();
+  n["is_applied"] = m_is_applied;
+}
+
 /*
  *************************************************************************
  *

src/axom/sidre/core/View.hpp

@@ -908,6 +908,14 @@ class View
    */
   void createNativeLayout(Node& n) const;
 
+  /*!
+   * \brief Copy metadata of the View to the given Conduit node
+   *
+   * The View's datatype schema and state information will be copied into the
+   * node, but not the data held by the View.
+   */
+  void copyMetadataToNode(Node& n) const;
+
   /*!
    * \brief Change the name of this View.
    *

src/axom/sidre/docs/sphinx/file_io.rst

@@ -121,3 +121,31 @@ Protocols for file I/O
     data in a basic JSON hierarchy with no type information beyond
     what can be interpreted implicitly. When loading, the library will allocate 
     the largest supported bit width for integer or floating-point values.
+
+``sidre_layout_json``
+
+    This is one of two protocols that is provided as a debugging aid and not
+    to support the full save and load features of Sidre. It is intended to
+    provide user-readable JSON output that shows the full layout of the
+    Sidre hierarchy, but it excludes the data arrays held by the buffers.
+    With the exception of those data arrays, it matches the output layout
+    of the ``sidre_conduit_json`` protocol. This protocol should only be
+    used with ``save``, as it does not output all of the data that would be
+    needed for a successful ``load``.
+
+``conduit_layout_json``
+
+    This is the second protocol that is a debugging aid without supporting
+    the full save and load features of Sidre. Its JSON output writes the
+    Sidre hierarchy layout in way that matches the ``condut_json`` protocol
+    while again excluding the data arrays from the Views. This should only be
+    used with ``save`` and not load.
+
+    Another way to create file output that has the data arrays fully or 
+    partially removed is to use the ``convert_sidre_protocol`` tool that is
+    built as part of the build of Sidre. This tool takes as input a root file
+    that was created by ``IOManager`` with the ``sidre_hdf5`` protocol
+    and converts it to output in another selected protocol. This tool has a
+    has a command-line option to strip all or part of the data arrays
+    from the output. Unlike this tool, the ``*_layout_*`` protocols
+    described above do not require usage of ``IOManager``.

src/axom/sidre/tests/sidre_group.cpp

@@ -2729,6 +2729,55 @@ TEST(sidre_group, save_load_preserve_contents)
   DataStore::setConduitDefaultMessageHandlers();
 }
 
+//------------------------------------------------------------------------------
+TEST(sidre_group, save_layout_protocols)
+{
+  // Tests calls to save using sidre_layout_json and conduit_layout_json
+  // protocols. No loads are attempted as these protocols are used for
+  // save only.
+
+  DataStore::setConduitSLICMessageHandlers();
+
+  const std::string file_path_base("sidre_save_layout_protocols_");
+  DataStore ds;
+
+  Group* flds = ds.getRoot()->createGroup("fields");
+
+  Group* ga = flds->createGroup("a");
+  Group* gb = flds->createGroup("b");
+  Group* gc = flds->createGroup("c");
+  int ndata = 10;
+
+  ga->createViewScalar<conduit::int64>("i0", 100);
+  ga->createViewScalar<conduit::float64>("d0", 3000.00);
+  gb->createViewString("s0", "foo");
+
+  gc->createViewAndAllocate("int10", DataType::int64(ndata));
+  conduit::int64* data_ptr = gc->getView("int10")->getArray();
+  for(int i = 0; i < ndata; ++i)
+  {
+    data_ptr[i] = (conduit::int64)i;
+  }
+
+  int extdatasize = 20;
+  std::vector<conduit::float64> extdata(extdatasize);
+  gc->createView("ext20", FLOAT64_ID, extdatasize)->setExternalDataPtr(&extdata[0]);
+  for(int i = 0; i < extdatasize; ++i)
+  {
+    extdata[i] = (conduit::float64)(i / 3.0);
+  }
+
+  gc->createView("empty_view");
+
+  std::string file_path = file_path_base + "sidre_layout_json";
+  ds.getRoot()->save(file_path, "sidre_layout_json");
+  file_path = file_path_base + "conduit_layout_json";
+  ds.getRoot()->save(file_path, "conduit_layout_json");
+
+  //restore conduit default errors
+  DataStore::setConduitDefaultMessageHandlers();
+}
+
 //------------------------------------------------------------------------------
 TEST(sidre_group, import_conduit)
 {