From e8ae38b65ea81dc81d2fc3aebcf6c1679436a2e5 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Thu, 10 Nov 2022 17:34:18 +0100 Subject: [PATCH] Python bindings for Frame (#343) * Make python code a proper module * Reorganize CMake config for shared libraries * Refactor library and dict generation in cmake config * Add basic version of python bindings for working with Frames --- .gitignore | 1 + CMakeLists.txt | 2 + include/podio/Frame.h | 25 +++ include/podio/ROOTFrameReader.h | 5 + include/podio/SIOBlock.h | 5 + include/podio/SIOFrameReader.h | 5 + python/CMakeLists.txt | 29 ++- python/EventStore.py | 139 +------------ python/__init__.py.in | 1 + python/podio/EventStore.py | 137 +++++++++++++ python/podio/base_reader.py | 44 ++++ python/podio/frame.py | 178 ++++++++++++++++ python/podio/frame_iterator.py | 58 ++++++ python/{ => podio}/generator_utils.py | 0 python/{ => podio}/podio_config_reader.py | 2 +- python/podio/root_io.py | 28 +++ python/podio/sio_io.py | 25 +++ .../test_ClassDefinitionValidator.py | 4 +- python/{ => podio}/test_EventStore.py | 2 +- python/{ => podio}/test_EventStoreRoot.py | 4 +- python/{ => podio}/test_EventStoreSio.py | 8 +- python/podio/test_Frame.py | 95 +++++++++ python/{ => podio}/test_MemberParser.py | 2 +- python/podio/test_Reader.py | 70 +++++++ python/podio/test_ReaderRoot.py | 14 ++ python/podio/test_ReaderSio.py | 16 ++ python/podio/test_utils.py | 6 + python/podio_class_generator.py | 4 +- src/CMakeLists.txt | 194 ++++++++++-------- src/ROOTFrameReader.cc | 11 +- src/SIOBlock.cc | 10 + src/SIOFrameReader.cc | 4 + src/root_selection.xml | 6 + src/sio_selection.xml | 6 + tests/CMakeLists.txt | 9 +- tests/read.py | 2 +- tests/write_frame.h | 1 + tools/podio-dump | 2 +- 38 files changed, 909 insertions(+), 245 deletions(-) create mode 100644 python/__init__.py.in create mode 100644 python/podio/EventStore.py create mode 100644 python/podio/base_reader.py create mode 100644 python/podio/frame.py create mode 100644 python/podio/frame_iterator.py rename python/{ => podio}/generator_utils.py (100%) rename python/{ => podio}/podio_config_reader.py (99%) create mode 100644 python/podio/root_io.py create mode 100644 python/podio/sio_io.py rename python/{ => podio}/test_ClassDefinitionValidator.py (99%) rename python/{ => podio}/test_EventStore.py (98%) rename python/{ => podio}/test_EventStoreRoot.py (94%) rename python/{ => podio}/test_EventStoreSio.py (86%) create mode 100644 python/podio/test_Frame.py rename python/{ => podio}/test_MemberParser.py (99%) create mode 100644 python/podio/test_Reader.py create mode 100644 python/podio/test_ReaderRoot.py create mode 100644 python/podio/test_ReaderSio.py create mode 100644 python/podio/test_utils.py create mode 100644 src/root_selection.xml create mode 100644 src/sio_selection.xml diff --git a/.gitignore b/.gitignore index be893c085..568832339 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,4 @@ spack* # Populated by cmake before build /include/podio/podioVersion.h +/python/podio/__init__.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 175085b87..c87330a8c 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -146,6 +146,8 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/podioVersion.in.h ${CMAKE_CURRENT_SOURCE_DIR}/include/podio/podioVersion.h ) install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/podio/podioVersion.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/podio ) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/python/__init__.py.in + ${CMAKE_CURRENT_SOURCE_DIR}/python/podio/__init__.py) #--- add license files --------------------------------------------------------- install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE diff --git a/include/podio/Frame.h b/include/podio/Frame.h index fc31b4cbe..5f2357dae 100644 --- a/include/podio/Frame.h +++ b/include/podio/Frame.h @@ -27,6 +27,10 @@ using EnableIfCollection = typename std::enable_if_t>; template using EnableIfCollectionRValue = typename std::enable_if_t && !std::is_lvalue_reference_v>; +/// Alias template for enabling overloads for r-values +template +using EnableIfRValue = typename std::enable_if_t>; + namespace detail { /** The minimal interface for raw data types */ @@ -152,6 +156,14 @@ class Frame { template Frame(std::unique_ptr); + /** Frame constructor from (almost) arbitrary raw data. + * + * This r-value overload is mainly present for enabling the python bindings, + * where cppyy seems to strip the std::unique_ptr somewhere in the process + */ + template > + Frame(FrameDataT&&); + // The frame is a non-copyable type Frame(const Frame&) = delete; Frame& operator=(const Frame&) = delete; @@ -167,6 +179,11 @@ class Frame { template > const CollT& get(const std::string& name) const; + /** Get a collection from the Frame. This is the pointer-to-base version for + * type-erased access (e.g. python interface) + */ + const podio::CollectionBase* get(const std::string& name) const; + /** (Destructively) move a collection into the Frame and get a const reference * back for further use */ @@ -259,6 +276,10 @@ template Frame::Frame(std::unique_ptr data) : m_self(std::make_unique>(std::move(data))) { } +template +Frame::Frame(FrameDataT&& data) : Frame(std::make_unique(std::move(data))) { +} + template const CollT& Frame::get(const std::string& name) const { const auto* coll = dynamic_cast(m_self->get(name)); @@ -270,6 +291,10 @@ const CollT& Frame::get(const std::string& name) const { return emptyColl; } +const podio::CollectionBase* Frame::get(const std::string& name) const { + return m_self->get(name); +} + void Frame::put(std::unique_ptr coll, const std::string& name) { const auto* retColl = m_self->put(std::move(coll), name); if (!retColl) { diff --git a/include/podio/ROOTFrameReader.h b/include/podio/ROOTFrameReader.h index 4eca496be..1850a0b02 100644 --- a/include/podio/ROOTFrameReader.h +++ b/include/podio/ROOTFrameReader.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -70,10 +71,14 @@ class ROOTFrameReader { /// Returns number of entries for the given name unsigned getEntries(const std::string& name) const; + /// Get the build version of podio that has been used to write the current file podio::version::Version currentFileVersion() const { return m_fileVersion; } + /// Get the names of all the availalable Frame categories in the current file(s) + std::vector getAvailableCategories() const; + private: /** * Helper struct to group together all the necessary state to read / process a diff --git a/include/podio/SIOBlock.h b/include/podio/SIOBlock.h index fdc4bd8a8..7c9b2a462 100644 --- a/include/podio/SIOBlock.h +++ b/include/podio/SIOBlock.h @@ -14,6 +14,7 @@ #include #include #include +#include namespace podio { @@ -230,6 +231,10 @@ class SIOFileTOCRecord { */ PositionType getPosition(const std::string& name, unsigned iEntry = 0) const; + /** Get all the record names that are stored in this TOC record + */ + std::vector getRecordNames() const; + private: friend struct SIOFileTOCRecordBlock; diff --git a/include/podio/SIOFrameReader.h b/include/podio/SIOFrameReader.h index 11b1676b6..d7a2c5e8c 100644 --- a/include/podio/SIOFrameReader.h +++ b/include/podio/SIOFrameReader.h @@ -9,6 +9,7 @@ #include #include +#include #include namespace podio { @@ -44,10 +45,14 @@ class SIOFrameReader { void openFile(const std::string& filename); + /// Get the build version of podio that has been used to write the current file podio::version::Version currentFileVersion() const { return m_fileVersion; } + /// Get the names of all the availalable Frame categories in the current file(s) + std::vector getAvailableCategories() const; + private: void readPodioHeader(); diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index fb6fc46b7..54547e833 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -2,19 +2,28 @@ SET(podio_PYTHON_INSTALLDIR python) SET(podio_PYTHON_INSTALLDIR ${podio_PYTHON_INSTALLDIR} PARENT_SCOPE) SET(podio_PYTHON_DIR ${CMAKE_CURRENT_LIST_DIR} PARENT_SCOPE) -file(GLOB to_install *.py figure.txt) - -# remove test_*.py file from being installed -foreach(file_path ${to_install}) - get_filename_component(file_name ${file_path} NAME) - string(REGEX MATCH test_.*\\.py$ FOUND_PY_TEST ${file_name}) - if (NOT "${FOUND_PY_TEST}" STREQUAL "") - list(REMOVE_ITEM to_install "${file_path}") - endif() -endforeach() +set(to_install + podio_class_generator.py + figure.txt + EventStore.py) install(FILES ${to_install} DESTINATION ${podio_PYTHON_INSTALLDIR}) +if(ENABLE_SIO) + install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/podio + DESTINATION ${podio_PYTHON_INSTALLDIR} + REGEX test_.*\\.py$ EXCLUDE # Do not install test files + PATTERN __pycache__ EXCLUDE # Or pythons caches + ) +else() + install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/podio + DESTINATION ${podio_PYTHON_INSTALLDIR} + REGEX test_.*\\.py$ EXCLUDE # Do not install test files + PATTERN __pycache__ EXCLUDE # Or pythons caches + REGEX .*sio.*\\.py$ EXCLUDE # All things sio related + ) +endif() + #--- install templates --------------------------------------------------------- install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/templates DESTINATION ${podio_PYTHON_INSTALLDIR}) diff --git a/python/EventStore.py b/python/EventStore.py index c1a2df3d4..5607f2da0 100644 --- a/python/EventStore.py +++ b/python/EventStore.py @@ -1,137 +1,6 @@ -"""Python EventStore for reading files with podio generated datamodels""" +"""Legacy import wrapper for EventStore.""" +import warnings +warnings.warn("You are using the legacy EventStore import. Switch to 'from podio import EventStore'", FutureWarning) -from ROOT import gSystem -gSystem.Load("libpodioPythonStore") # noqa: E402 -from ROOT import podio # noqa: E402 # pylint: disable=wrong-import-position - - -def size(self): - """Override size function that can be attached as __len__ method to - collections""" - return self.size() - - -def getitem(self, key): - """Override getitem function that can be attached as __getitem__ method to - collections (see below why this is necessary sometimes)""" - return self.at(key) - - -class EventStore: - '''Interface to events in an podio root file. - Example of use: - events = EventStore(["example.root", "example1.root"]) - for iev, store in islice(enumerate(events), 0, 2): - particles = store.get("GenParticle"); - for i, p in islice(enumerate(particles), 0, 5): - print "particle ", i, p.ID(), p.P4().Pt - ''' - - def __init__(self, filenames): - '''Create an event list from the podio root file. - Parameters: - filenames: list of root files - you can of course provide a list containing a single - root file. you could use the glob module to get all - files matching a wildcard pattern. - ''' - if isinstance(filenames, str): - filenames = (filenames,) - self.files = filenames - self.stores = [] - self.current_store = None - for fname in self.files: - store = podio.PythonEventStore(fname) - if store.isZombie(): - raise ValueError(fname + ' does not exist.') - store.name = fname - if self.current_store is None: - self.current_store = store - self.stores.append((store.getEntries(), store)) - - def __str__(self): - result = "Content:" - for item in self.current_store.getCollectionNames(): - result += f"\n\t{item}" - return result - - def get(self, name): - '''Returns a collection. - Parameters: - name: name of the collection in the podio root file. - ''' - coll = self.current_store.get(name) - # adding length function - coll.__len__ = size - # enabling the use of [] notation on the collection - # cppyy defines the __getitem__ method if the underlying c++ class has an operator[] - # method. For some reason they do not conform to the usual signature and only - # pass one argument to the function they call. Here we simply check if we have to - # define the __getitem__ for the collection. - if not hasattr(coll, '__getitem__'): - coll.__getitem__ = getitem - return coll - - def collections(self): - """Return list of all collection names.""" - return [str(c) for c in self.current_store.getCollectionNames()] - - def metadata(self): - """Get the metadata of the current event as GenericParameters""" - return self.current_store.getEventMetaData() - - def isValid(self): - """Check if the EventStore is in a valid state""" - return self.current_store is not None and self.current_store.isValid() - - # def __getattr__(self, name): - # '''missing attributes are taken from self.current_store''' - # if name != 'current_store': - # return getattr(self.current_store, name) - # else: - # return None - - def current_filename(self): - '''Returns the name of the current file.''' - if self.current_store is None: - return None - return self.current_store.fname - - def __enter__(self): - return self - - def __exit__(self, exception_type, exception_val, trace): - for store in self.stores: - store[1].close() - - def __iter__(self): - '''iterate on events in the tree. - ''' - for _, store in self.stores: - self.current_store = store - for _ in range(store.getEntries()): - yield store - store.endOfEvent() - - def __getitem__(self, evnum): - '''Get event number evnum''' - current_store = None - rel_evnum = evnum - for nev, store in self.stores: - if rel_evnum < nev: - current_store = store - break - rel_evnum -= nev - if current_store is None: - raise ValueError('event number too large: ' + str(evnum)) - self.current_store = current_store - self.current_store.goToEvent(rel_evnum) - return self - - def __len__(self): - '''Returns the total number of events in all files.''' - nevts_all_files = 0 - for nev, _ in self.stores: - nevts_all_files += nev - return nevts_all_files +from podio import EventStore # noqa: F401 # pylint: disable=wrong-import-position, unused-import diff --git a/python/__init__.py.in b/python/__init__.py.in new file mode 100644 index 000000000..e1f12ec32 --- /dev/null +++ b/python/__init__.py.in @@ -0,0 +1 @@ +__version__ = '@podio_VERSION@' diff --git a/python/podio/EventStore.py b/python/podio/EventStore.py new file mode 100644 index 000000000..c1a2df3d4 --- /dev/null +++ b/python/podio/EventStore.py @@ -0,0 +1,137 @@ +"""Python EventStore for reading files with podio generated datamodels""" + + +from ROOT import gSystem +gSystem.Load("libpodioPythonStore") # noqa: E402 +from ROOT import podio # noqa: E402 # pylint: disable=wrong-import-position + + +def size(self): + """Override size function that can be attached as __len__ method to + collections""" + return self.size() + + +def getitem(self, key): + """Override getitem function that can be attached as __getitem__ method to + collections (see below why this is necessary sometimes)""" + return self.at(key) + + +class EventStore: + '''Interface to events in an podio root file. + Example of use: + events = EventStore(["example.root", "example1.root"]) + for iev, store in islice(enumerate(events), 0, 2): + particles = store.get("GenParticle"); + for i, p in islice(enumerate(particles), 0, 5): + print "particle ", i, p.ID(), p.P4().Pt + ''' + + def __init__(self, filenames): + '''Create an event list from the podio root file. + Parameters: + filenames: list of root files + you can of course provide a list containing a single + root file. you could use the glob module to get all + files matching a wildcard pattern. + ''' + if isinstance(filenames, str): + filenames = (filenames,) + self.files = filenames + self.stores = [] + self.current_store = None + for fname in self.files: + store = podio.PythonEventStore(fname) + if store.isZombie(): + raise ValueError(fname + ' does not exist.') + store.name = fname + if self.current_store is None: + self.current_store = store + self.stores.append((store.getEntries(), store)) + + def __str__(self): + result = "Content:" + for item in self.current_store.getCollectionNames(): + result += f"\n\t{item}" + return result + + def get(self, name): + '''Returns a collection. + Parameters: + name: name of the collection in the podio root file. + ''' + coll = self.current_store.get(name) + # adding length function + coll.__len__ = size + # enabling the use of [] notation on the collection + # cppyy defines the __getitem__ method if the underlying c++ class has an operator[] + # method. For some reason they do not conform to the usual signature and only + # pass one argument to the function they call. Here we simply check if we have to + # define the __getitem__ for the collection. + if not hasattr(coll, '__getitem__'): + coll.__getitem__ = getitem + return coll + + def collections(self): + """Return list of all collection names.""" + return [str(c) for c in self.current_store.getCollectionNames()] + + def metadata(self): + """Get the metadata of the current event as GenericParameters""" + return self.current_store.getEventMetaData() + + def isValid(self): + """Check if the EventStore is in a valid state""" + return self.current_store is not None and self.current_store.isValid() + + # def __getattr__(self, name): + # '''missing attributes are taken from self.current_store''' + # if name != 'current_store': + # return getattr(self.current_store, name) + # else: + # return None + + def current_filename(self): + '''Returns the name of the current file.''' + if self.current_store is None: + return None + return self.current_store.fname + + def __enter__(self): + return self + + def __exit__(self, exception_type, exception_val, trace): + for store in self.stores: + store[1].close() + + def __iter__(self): + '''iterate on events in the tree. + ''' + for _, store in self.stores: + self.current_store = store + for _ in range(store.getEntries()): + yield store + store.endOfEvent() + + def __getitem__(self, evnum): + '''Get event number evnum''' + current_store = None + rel_evnum = evnum + for nev, store in self.stores: + if rel_evnum < nev: + current_store = store + break + rel_evnum -= nev + if current_store is None: + raise ValueError('event number too large: ' + str(evnum)) + self.current_store = current_store + self.current_store.goToEvent(rel_evnum) + return self + + def __len__(self): + '''Returns the total number of events in all files.''' + nevts_all_files = 0 + for nev, _ in self.stores: + nevts_all_files += nev + return nevts_all_files diff --git a/python/podio/base_reader.py b/python/podio/base_reader.py new file mode 100644 index 000000000..78549a5a4 --- /dev/null +++ b/python/podio/base_reader.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Python module for defining the basic reader interface that is used by the +backend specific bindings""" + + +from podio.frame_iterator import FrameCategoryIterator + + +class BaseReaderMixin: + """Mixin class the defines the base interface of the readers. + + The backend specific readers inherit from here and have to initialize the + following members: + - _reader: The actual reader that is able to read frames + """ + + def __init__(self): + """Initialize common members. + + In inheriting classes this needs to be called **after** the _reader has been + setup. + """ + self._categories = tuple(s.data() for s in self._reader.getAvailableCategories()) + + @property + def categories(self): + """Get the available categories from this reader. + + Returns: + tuple(str): The names of the available categories from this reader + """ + return self._categories + + def get(self, category): + """Get an iterator with access functionality for a given category. + + Args: + category (str): The name of the desired category + + Returns: + FrameCategoryIterator: The iterator granting access to all Frames of the + desired category + """ + return FrameCategoryIterator(self._reader, category) diff --git a/python/podio/frame.py b/python/podio/frame.py new file mode 100644 index 000000000..903ccf41d --- /dev/null +++ b/python/podio/frame.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Module for the python bindings of the podio::Frame""" + +# pylint: disable-next=import-error # gbl is a dynamic module from cppyy +from cppyy.gbl import std + +import ROOT +# NOTE: It is necessary that this can be found on the ROOT_INCLUDE_PATH +ROOT.gInterpreter.LoadFile('podio/Frame.h') # noqa: E402 +from ROOT import podio # noqa: E402 # pylint: disable=wrong-import-position + + +def _determine_supported_parameter_types(lang): + """Determine the supported types for the parameters. + + Args: + lang (str): Language for which the type names should be returned. Either + 'c++' or 'py'. + + Returns: + tuple (str): the tuple with the string representation of all **c++** + classes that are supported + """ + types_tuple = podio.SupportedGenericDataTypes() + n_types = std.tuple_size[podio.SupportedGenericDataTypes].value + + # Get the python types with the help of cppyy and the STL + py_types = (type(std.get[i](types_tuple)).__name__ for i in range(n_types)) + if lang == 'py': + return tuple(py_types) + if lang == 'c++': + # Map of types that need special care when going from python to c++ + py_to_cpp_type_map = { + 'str': 'std::string' + } + # Convert them to the corresponding c++ types + return tuple(py_to_cpp_type_map.get(t, t) for t in py_types) + + raise ValueError(f"lang needs to be 'py' or 'c++' (got {lang})") + + +SUPPORTED_PARAMETER_TYPES = _determine_supported_parameter_types('c++') +SUPPORTED_PARAMETER_PY_TYPES = _determine_supported_parameter_types('py') + + +class Frame: + """Frame class that serves as a container of collection and meta data.""" + + # Map that is necessary for easier disambiguation of parameters that are + # available with more than one type under the same name. Maps a python type to + # a c++ vector of the corresponding type + _py_to_cpp_type_map = { + pytype: f'std::vector<{cpptype}>' for (pytype, cpptype) in zip(SUPPORTED_PARAMETER_PY_TYPES, + SUPPORTED_PARAMETER_TYPES) + } + + def __init__(self, data=None): + """Create a Frame. + + Args: + data (FrameData, optional): Almost arbitrary FrameData, e.g. from file + """ + # Explicitly check for None here, to not return empty Frames on nullptr data + if data is not None: + self._frame = podio.Frame(data) + else: + self._frame = podio.Frame() + + self._collections = tuple(str(s) for s in self._frame.getAvailableCollections()) + self._param_key_types = self._init_param_keys() + + @property + def collections(self): + """Get the available collection (names) from this Frame. + + Returns: + tuple(str): The names of the available collections from this Frame. + """ + return self._collections + + def get(self, name): + """Get a collection from the Frame by name. + + Args: + name (str): The name of the desired collection + + Returns: + collection (podio.CollectionBase): The collection stored in the Frame + + Raises: + KeyError: If the collection with the name is not available + """ + collection = self._frame.get(name) + if not collection: + raise KeyError + return collection + + @property + def parameters(self): + """Get the available parameter names from this Frame. + + Returns: + tuple (str): The names of the available parameters from this Frame. + """ + return tuple(self._param_key_types.keys()) + + def get_parameter(self, name, as_type=None): + """Get the parameter stored under the given name. + + Args: + name (str): The name of the parameter + as_type (str, optional): Type specifier to disambiguate between + parameters with the same name but different types. If there is only + one parameter with a given name, this argument is ignored + + Returns: + int, float, str or list of those: The value of the stored parameter + + Raises: + KeyError: If no parameter is stored under the given name + ValueError: If there are multiple parameters with the same name, but + multiple types and no type specifier to disambiguate between them + has been passed. + + """ + def _get_param_value(par_type, name): + par_value = self._frame.getParameter[par_type](name) + if len(par_value) > 1: + return list(par_value) + return par_value[0] + + # This access already raises the KeyError if there is no such parameter + par_type = self._param_key_types[name] + # Exactly one parameter, nothing more to do here + if len(par_type) == 1: + return _get_param_value(par_type[0], name) + + if as_type is None: + raise ValueError(f'{name} parameter has {len(par_type)} different types available, ' + 'but no as_type argument to disambiguate') + + req_type = self._py_to_cpp_type_map.get(as_type, None) + if req_type is None: + raise ValueError(f'as_type value {as_type} cannot be mapped to a valid parameter type') + + if req_type not in par_type: + raise ValueError(f'{name} parameter is not available as type {as_type}') + + return _get_param_value(req_type, name) + + def _init_param_keys(self): + """Initialize the param keys dict for easier lookup of the available parameters. + + NOTE: This depends on a "side channel" that is usually reserved for the + writers but is currently still in the public interface of the Frame + + Returns: + dict: A dictionary mapping each key to the corresponding c++ type + """ + params = self._frame.getGenericParametersForWrite() # this is the iffy bit + keys_dict = {} + for par_type in SUPPORTED_PARAMETER_TYPES: + keys = params.getKeys[par_type]() + for key in keys: + # Make sure to convert to a python string here to not have a dangling + # reference here for the key. + key = str(key) + # In order to support the use case of having the same key for multiple + # types create a list of available types for the key, so that we can + # disambiguate later. Storing a vector here, and check later how + # many elements there actually are to decide whether to return a single + # value or a list + if key not in keys_dict: + keys_dict[key] = [f'std::vector<{par_type}>'] + else: + keys_dict[key].append(f'std::vector<{par_type}>') + + return keys_dict diff --git a/python/podio/frame_iterator.py b/python/podio/frame_iterator.py new file mode 100644 index 000000000..b3d9925ad --- /dev/null +++ b/python/podio/frame_iterator.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Module defining the Frame iterator used by the Reader interface""" + +# pylint: disable-next=import-error # gbl is a dynamic module from cppyy +from cppyy.gbl import std +from podio.frame import Frame + + +class FrameCategoryIterator: + """Iterator for iterating over all Frames of a given category available from a + reader as well as accessing specific entries + """ + + def __init__(self, reader, category): + """Construct the iterator from the reader and the cateogry. + + Args: + reader (Reader): Any podio reader offering access to Frames + category (str): The category name of the Frames to be iterated over + """ + self._reader = reader + self._category = category + + def __iter__(self): + """The trivial implementaion for the iterator protocol.""" + return self + + def __next__(self): + """Get the next available Frame or stop.""" + frame_data = self._reader.readNextEntry(self._category) + if frame_data: + return Frame(std.move(frame_data)) + + raise StopIteration + + def __len__(self): + """Get the number of available Frames for the passed category.""" + return self._reader.getEntries(self._category) + + def __getitem__(self, entry): + """Get a specific entry. + + Args: + entry (int): The entry to access + """ + # Handle python negative indexing to start from the end + if entry < 0: + entry = self._reader.getEntries(self._category) + entry + + if entry < 0: + # If we are below 0 now, we do not have enough entries to serve the request + raise IndexError + + frame_data = self._reader.readEntry(self._category, entry) + if frame_data: + return Frame(std.move(frame_data)) + + raise IndexError diff --git a/python/generator_utils.py b/python/podio/generator_utils.py similarity index 100% rename from python/generator_utils.py rename to python/podio/generator_utils.py diff --git a/python/podio_config_reader.py b/python/podio/podio_config_reader.py similarity index 99% rename from python/podio_config_reader.py rename to python/podio/podio_config_reader.py index 8f81b45a2..14c34d0f5 100644 --- a/python/podio_config_reader.py +++ b/python/podio/podio_config_reader.py @@ -6,7 +6,7 @@ import warnings import yaml -from generator_utils import MemberVariable, DefinitionError, BUILTIN_TYPES, DataModel +from podio.generator_utils import MemberVariable, DefinitionError, BUILTIN_TYPES, DataModel class MemberParser: diff --git a/python/podio/root_io.py b/python/podio/root_io.py new file mode 100644 index 000000000..2a37906aa --- /dev/null +++ b/python/podio/root_io.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +"""Python module for reading root files containing podio Frames""" + +from podio.base_reader import BaseReaderMixin + +from ROOT import gSystem +gSystem.Load('libpodioRootIO') # noqa: E402 +from ROOT import podio # noqa: E402 # pylint: disable=wrong-import-position + +Writer = podio.ROOTFrameWriter + + +class Reader(BaseReaderMixin): + """Reader class for reading podio root files.""" + + def __init__(self, filenames): + """Create a reader that reads from the passed file(s). + + Args: + filenames (str or list[str]): file(s) to open and read data from + """ + if isinstance(filenames, str): + filenames = (filenames,) + + self._reader = podio.ROOTFrameReader() + self._reader.openFiles(filenames) + + super().__init__() diff --git a/python/podio/sio_io.py b/python/podio/sio_io.py new file mode 100644 index 000000000..24ce24df3 --- /dev/null +++ b/python/podio/sio_io.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Python module for reading sio files containing podio Frames""" + +from podio.base_reader import BaseReaderMixin + +from ROOT import gSystem +gSystem.Load('libpodioSioIO') # noqa: 402 +from ROOT import podio # noqa: 402 # pylint: disable=wrong-import-position + +Writer = podio.SIOFrameWriter + + +class Reader(BaseReaderMixin): + """Reader class for readion podio SIO files.""" + + def __init__(self, filename): + """Create a reader that reads from the passed file. + + Args: + filename (str): File to open and read data from + """ + self._reader = podio.SIOFrameReader() + self._reader.openFile(filename) + + super().__init__() diff --git a/python/test_ClassDefinitionValidator.py b/python/podio/test_ClassDefinitionValidator.py similarity index 99% rename from python/test_ClassDefinitionValidator.py rename to python/podio/test_ClassDefinitionValidator.py index 5ddad9488..5db93c195 100644 --- a/python/test_ClassDefinitionValidator.py +++ b/python/podio/test_ClassDefinitionValidator.py @@ -8,8 +8,8 @@ import unittest from copy import deepcopy -from podio_config_reader import ClassDefinitionValidator, MemberVariable, DefinitionError -from generator_utils import DataModel +from podio.podio_config_reader import ClassDefinitionValidator, MemberVariable, DefinitionError +from podio.generator_utils import DataModel def make_dm(components, datatypes, options=None): diff --git a/python/test_EventStore.py b/python/podio/test_EventStore.py similarity index 98% rename from python/test_EventStore.py rename to python/podio/test_EventStore.py index 3202f4843..a8e4fb965 100644 --- a/python/test_EventStore.py +++ b/python/podio/test_EventStore.py @@ -1,6 +1,6 @@ """Unit tests for the EventStore class""" -from EventStore import EventStore +from podio.EventStore import EventStore class EventStoreBaseTestCaseMixin: diff --git a/python/test_EventStoreRoot.py b/python/podio/test_EventStoreRoot.py similarity index 94% rename from python/test_EventStoreRoot.py rename to python/podio/test_EventStoreRoot.py index 4acf74950..e9c334e4e 100644 --- a/python/test_EventStoreRoot.py +++ b/python/podio/test_EventStoreRoot.py @@ -6,8 +6,8 @@ from ROOT import TFile -from EventStore import EventStore -from test_EventStore import EventStoreBaseTestCaseMixin +from podio.EventStore import EventStore +from podio.test_EventStore import EventStoreBaseTestCaseMixin class EventStoreRootTestCase(EventStoreBaseTestCaseMixin, unittest.TestCase): diff --git a/python/test_EventStoreSio.py b/python/podio/test_EventStoreSio.py similarity index 86% rename from python/test_EventStoreSio.py rename to python/podio/test_EventStoreSio.py index 409039d95..1859fecf6 100644 --- a/python/test_EventStoreSio.py +++ b/python/podio/test_EventStoreSio.py @@ -4,11 +4,9 @@ import unittest import os -from EventStore import EventStore -from test_EventStore import EventStoreBaseTestCaseMixin - - -SKIP_SIO_TESTS = os.environ.get('SKIP_SIO_TESTS', '1') == '1' +from podio.EventStore import EventStore +from podio.test_EventStore import EventStoreBaseTestCaseMixin +from podio.test_utils import SKIP_SIO_TESTS @unittest.skipIf(SKIP_SIO_TESTS, "no SIO support") diff --git a/python/podio/test_Frame.py b/python/podio/test_Frame.py new file mode 100644 index 000000000..1390e08ce --- /dev/null +++ b/python/podio/test_Frame.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Unit tests for python bindings of podio::Frame""" + +import unittest + +from podio.frame import Frame +# using root_io as that should always be present regardless of which backends are built +from podio.root_io import Reader + +# The expected collections in each frame +EXPECTED_COLL_NAMES = { + 'arrays', 'WithVectorMember', 'info', 'fixedWidthInts', 'mcparticles', + 'moreMCs', 'mcParticleRefs', 'hits', 'hitRefs', 'clusters', 'refs', 'refs2', + 'OneRelation', 'userInts', 'userDoubles', 'WithNamespaceMember', + 'WithNamespaceRelation', 'WithNamespaceRelationCopy' + } +# The expected parameter names in each frame +EXPECTED_PARAM_NAMES = {'anInt', 'UserEventWeight', 'UserEventName', 'SomeVectorData'} + + +class FrameTest(unittest.TestCase): + """General unittests for for python bindings of the Frame""" + def test_frame_invalid_access(self): + """Check that the advertised exceptions are raised on invalid access.""" + # Creat an empty Frame here + frame = Frame() + with self.assertRaises(KeyError): + _ = frame.get('NonExistantCollection') + + with self.assertRaises(KeyError): + _ = frame.get_parameter('NonExistantParameter') + + +class FrameReadTest(unittest.TestCase): + """Unit tests for the Frame python bindings for Frames read from file. + + NOTE: The assumption is that the Frame has been written by tests/write_frame.h + """ + def setUp(self): + """Open the file and read in the first frame internally. + + Reading only one event/Frame of each category here as looping and other + basic checks are already handled by the Reader tests + """ + reader = Reader('example_frame.root') + self.event = reader.get('events')[0] + self.other_event = reader.get('other_events')[7] + + def test_frame_collections(self): + """Check that all expected collections are available.""" + self.assertEqual(set(self.event.collections), EXPECTED_COLL_NAMES) + self.assertEqual(set(self.other_event.collections), EXPECTED_COLL_NAMES) + + # Not going over all collections here, as that should all be covered by the + # c++ test cases; Simply picking a few and doing some basic tests + mc_particles = self.event.get('mcparticles') + self.assertEqual(mc_particles.getValueTypeName(), 'ExampleMC') + self.assertEqual(len(mc_particles), 10) + self.assertEqual(len(mc_particles[0].daughters()), 4) + + mc_particle_refs = self.event.get('mcParticleRefs') + self.assertTrue(mc_particle_refs.isSubsetCollection()) + self.assertEqual(len(mc_particle_refs), 10) + + fixed_w_ints = self.event.get('fixedWidthInts') + self.assertEqual(len(fixed_w_ints), 3) + # Python has no concept of fixed width integers... + max_vals = fixed_w_ints[0] + self.assertEqual(max_vals.fixedInteger64(), 2**63 - 1) + self.assertEqual(max_vals.fixedU64(), 2**64 - 1) + + def test_frame_parameters(self): + """Check that all expected parameters are available.""" + self.assertEqual(set(self.event.parameters), EXPECTED_PARAM_NAMES) + self.assertEqual(set(self.other_event.parameters), EXPECTED_PARAM_NAMES) + + self.assertEqual(self.event.get_parameter('anInt'), 42) + self.assertEqual(self.other_event.get_parameter('anInt'), 42 + 107) + + self.assertEqual(self.event.get_parameter('UserEventWeight'), 0) + self.assertEqual(self.other_event.get_parameter('UserEventWeight'), 100. * 107) + + self.assertEqual(self.event.get_parameter('UserEventName'), ' event_number_0') + self.assertEqual(self.other_event.get_parameter('UserEventName'), ' event_number_107') + + with self.assertRaises(ValueError): + # Parameter name is available with multiple types + _ = self.event.get_parameter('SomeVectorData') + + with self.assertRaises(ValueError): + # Parameter not available as float (only int and string) + _ = self.event.get_parameter('SomeVectorData', as_type='float') + + self.assertEqual(self.event.get_parameter('SomeVectorData', as_type='int'), [1, 2, 3, 4]) + self.assertEqual(self.event.get_parameter('SomeVectorData', as_type='str'), ["just", "some", "strings"]) diff --git a/python/test_MemberParser.py b/python/podio/test_MemberParser.py similarity index 99% rename from python/test_MemberParser.py rename to python/podio/test_MemberParser.py index 572c33cd0..c7e4ec2bb 100644 --- a/python/test_MemberParser.py +++ b/python/podio/test_MemberParser.py @@ -6,7 +6,7 @@ import unittest -from podio_config_reader import MemberParser, DefinitionError +from podio.podio_config_reader import MemberParser, DefinitionError class MemberParserTest(unittest.TestCase): diff --git a/python/podio/test_Reader.py b/python/podio/test_Reader.py new file mode 100644 index 000000000..ff35ca0d4 --- /dev/null +++ b/python/podio/test_Reader.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Unit tests for podio readers""" + + +class ReaderTestCaseMixin: + """Common unittests for readers. + + Inheriting actual test cases have to inhert from this and unittest.TestCase. + All test cases assume that the files are produced with the tests/write_frame.h + functionaltiy. The following members have to be setup and initialized by the + inheriting test cases: + - reader: a podio reader + """ + def test_categories(self): + """Make sure that the categories are as expected""" + reader_cats = self.reader.categories + self.assertEqual(len(reader_cats), 2) + + for cat in ('events', 'other_events'): + self.assertTrue(cat in reader_cats) + + def test_frame_iterator_valid_category(self): + """Check that the returned iterators returned by Reader.get behave as expected.""" + # NOTE: very basic iterator tests only, content tests are done elsewhere + frames = self.reader.get('other_events') + self.assertEqual(len(frames), 10) + + i = 0 + for frame in self.reader.get('events'): + # Rudimentary check here only to see whether we got the right frame + self.assertEqual(frame.get_parameter('UserEventName'), f' event_number_{i}') + i += 1 + self.assertEqual(i, 10) + + # Out of bound access should not work + with self.assertRaises(IndexError): + _ = frames[10] + with self.assertRaises(IndexError): + _ = frames[-11] + + # Again only rudimentary checks + frame = frames[7] + self.assertEqual(frame.get_parameter('UserEventName'), ' event_number_107') + # Valid negative indexing + frame = frames[-2] + self.assertEqual(frame.get_parameter('UserEventName'), ' event_number_108') + # jumping back again also works + frame = frames[3] + self.assertEqual(frame.get_parameter('UserEventName'), ' event_number_103') + + # Looping starts from where we left, i.e. here we have 6 frames left + i = 0 + for _ in frames: + i += 1 + self.assertEqual(i, 6) + + def test_frame_iterator_invalid_category(self): + """Make sure non existant Frames are handled gracefully""" + non_existant = self.reader.get('non-existant') + self.assertEqual(len(non_existant), 0) + + # Indexed access should obviously not work + with self.assertRaises(IndexError): + _ = non_existant[0] + + # Loops should never be entered + i = 0 + for _ in non_existant: + i += 1 + self.assertEqual(i, 0) diff --git a/python/podio/test_ReaderRoot.py b/python/podio/test_ReaderRoot.py new file mode 100644 index 000000000..5ad7d119b --- /dev/null +++ b/python/podio/test_ReaderRoot.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +"""Python unit tests for the ROOT backend (using Frames)""" + +import unittest + +from podio.root_io import Reader +from podio.test_Reader import ReaderTestCaseMixin + + +class RootReaderTestCase(ReaderTestCaseMixin, unittest.TestCase): + """Test cases for root input files""" + def setUp(self): + """Setup the corresponding reader""" + self.reader = Reader('example_frame.root') diff --git a/python/podio/test_ReaderSio.py b/python/podio/test_ReaderSio.py new file mode 100644 index 000000000..72a8c0d0e --- /dev/null +++ b/python/podio/test_ReaderSio.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""Python unit tests for the SIO backend (using Frames)""" + +import unittest + +from podio.sio_io import Reader +from podio.test_Reader import ReaderTestCaseMixin +from podio.test_utils import SKIP_SIO_TESTS + + +@unittest.skipIf(SKIP_SIO_TESTS, "no SIO support") +class SioReaderTestCase(ReaderTestCaseMixin, unittest.TestCase): + """Test cases for root input files""" + def setUp(self): + """Setup the corresponding reader""" + self.reader = Reader('example_frame.sio') diff --git a/python/podio/test_utils.py b/python/podio/test_utils.py new file mode 100644 index 000000000..2c5e282b6 --- /dev/null +++ b/python/podio/test_utils.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 +"""Utilities for python unittests""" + +import os + +SKIP_SIO_TESTS = os.environ.get('SKIP_SIO_TESTS', '1') == '1' diff --git a/python/podio_class_generator.py b/python/podio_class_generator.py index a4b587d4e..42187e8b0 100755 --- a/python/podio_class_generator.py +++ b/python/podio_class_generator.py @@ -15,8 +15,8 @@ import jinja2 -from podio_config_reader import PodioConfigReader -from generator_utils import DataType, DefinitionError +from podio.podio_config_reader import PodioConfigReader +from podio.generator_utils import DataType, DefinitionError THIS_DIR = os.path.dirname(os.path.abspath(__file__)) TEMPLATE_DIR = os.path.join(THIS_DIR, 'templates') diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 696831613..f8e320763 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,12 +1,71 @@ -# This is needed for older ROOTs which do not understand -# target usage requirements +# Helper function for adding two targets. A shared library and a corresponding +# ROOT dictionary that is necessary for e.g. python bindings. +# Arguments: +# libname The base name for the library (and target) +# headers The header files that should be passed to dictionary generation +# sources The source files for the shared libraries +# selection The selection xml passed to the dictionary generation +# +# The function creates the following targets +# The shared library. Also available under the podio:: alias +# This is not linked against anything and has include directories set +# to podio only. So some target_link_libraries are most likely to be +# done outside +# Dict The dictionary shared library. This is linked against podio::podio +# and the necessary ROOT libraries +# +# Additionally the following files are generated by root +# - DictDict.rootmap +# - Dict_rdict.pcm +# these files have to be installed to the same directory as the dictionary shared +# library +FUNCTION(PODIO_ADD_LIB_AND_DICT libname headers sources selection ) + # shared library + add_library(${libname} SHARED ${sources}) + add_library(podio::${libname} ALIAS ${libname}) + target_include_directories(${libname} PUBLIC + $ + $) -SET(sources + # dictionary + set(dictname ${libname}Dict) + add_library(${dictname} SHARED) + target_include_directories(${dictname} PUBLIC + $ + $) + target_link_libraries(${dictname} PUBLIC podio::${libname} podio::podio ROOT::Core ROOT::Tree) + PODIO_GENERATE_DICTIONARY(${dictname} ${headers} SELECTION ${selection} + OPTIONS --library ${CMAKE_SHARED_LIBRARY_PREFIX}${dictname}${CMAKE_SHARED_LIBRARY_SUFFIX} + ) + # prevent generating dictionary twice + set_target_properties(${dictname}-dictgen PROPERTIES EXCLUDE_FROM_ALL TRUE) + target_sources(${dictname} PRIVATE ${dictname}.cxx) +ENDFUNCTION() + + +# --- Core podio library and dictionary without I/O +SET(core_sources CollectionIDTable.cc GenericParameters.cc ASCIIWriter.cc EventStore.cc) +SET(core_headers + ${CMAKE_SOURCE_DIR}/include/podio/CollectionBase.h + ${CMAKE_SOURCE_DIR}/include/podio/CollectionIDTable.h + ${CMAKE_SOURCE_DIR}/include/podio/EventStore.h + ${CMAKE_SOURCE_DIR}/include/podio/ICollectionProvider.h + ${CMAKE_SOURCE_DIR}/include/podio/IReader.h + ${CMAKE_SOURCE_DIR}/include/podio/ObjectID.h + ${CMAKE_SOURCE_DIR}/include/podio/UserDataCollection.h + ${CMAKE_SOURCE_DIR}/include/podio/podioVersion.h + ) + +PODIO_ADD_LIB_AND_DICT(podio "${core_headers}" "${core_sources}" selection.xml) +target_compile_options(podio PRIVATE -pthread) + + +# --- Root I/O functionality and corresponding dictionary SET(root_sources rootUtils.h ROOTWriter.cc @@ -15,106 +74,81 @@ SET(root_sources ROOTFrameReader.cc ) -SET(sio_sources - SIOReader.cc - SIOWriter.cc - SIOBlockUserData.cc - SIOBlock.cc - SIOFrameWriter.cc - SIOFrameReader.cc - SIOFrameData.cc -) - -SET(python_sources - IOHelpers.cc - PythonEventStore.cc +SET(root_headers + ${CMAKE_SOURCE_DIR}/include/podio/ROOTFrameReader.h + ${CMAKE_SOURCE_DIR}/include/podio/ROOTFrameWriter.h ) -# Main Library, no external dependencies -add_library(podio SHARED ${sources}) -add_library(podio::podio ALIAS podio) -target_include_directories(podio PUBLIC - $ - $) -target_compile_options(podio PRIVATE -pthread) - -# Root dependency, mostly IO -add_library(podioRootIO SHARED ${root_sources}) -add_library(podio::podioRootIO ALIAS podioRootIO) +PODIO_ADD_LIB_AND_DICT(podioRootIO "${root_headers}" "${root_sources}" root_selection.xml) target_link_libraries(podioRootIO PUBLIC podio::podio ROOT::Core ROOT::RIO ROOT::Tree) -target_include_directories(podioRootIO PUBLIC - $ - $) - -# Dict Library -add_library(podioDict SHARED) -add_library(podio::podioDict ALIAS podioDict) -target_include_directories(podioDict PUBLIC - $ - $) -target_link_libraries(podioDict PUBLIC podio::podio ROOT::Core ROOT::Tree) - -SET(headers - ${CMAKE_SOURCE_DIR}/include/podio/CollectionBase.h - ${CMAKE_SOURCE_DIR}/include/podio/CollectionIDTable.h - ${CMAKE_SOURCE_DIR}/include/podio/EventStore.h - ${CMAKE_SOURCE_DIR}/include/podio/ICollectionProvider.h - ${CMAKE_SOURCE_DIR}/include/podio/IReader.h - ${CMAKE_SOURCE_DIR}/include/podio/ObjectID.h - ${CMAKE_SOURCE_DIR}/include/podio/UserDataCollection.h - ${CMAKE_SOURCE_DIR}/include/podio/podioVersion.h - ) -PODIO_GENERATE_DICTIONARY(podioDict ${headers} SELECTION selection.xml - OPTIONS --library ${CMAKE_SHARED_LIBRARY_PREFIX}podioDict${CMAKE_SHARED_LIBRARY_SUFFIX} - ) -# prevent generating dictionary twice -set_target_properties(podioDict-dictgen PROPERTIES EXCLUDE_FROM_ALL TRUE) -target_sources(podioDict PRIVATE podioDict.cxx) -add_library(podioPythonStore SHARED ${python_sources}) -target_link_libraries(podioPythonStore podio podioRootIO) -LIST(APPEND INSTALL_LIBRARIES podioPythonStore) +# --- Python EventStore for enabling (legacy) python bindings +SET(python_sources + IOHelpers.cc + PythonEventStore.cc + ) -add_library(podioPythonStoreDict SHARED) -target_include_directories(podioPythonStoreDict PUBLIC - $ - $ -) -target_link_libraries(podioPythonStoreDict PUBLIC podioPythonStore) SET(python_headers ${CMAKE_SOURCE_DIR}/include/podio/PythonEventStore.h ) -PODIO_GENERATE_DICTIONARY(podioPythonStoreDict ${python_headers} SELECTION python_selection.xml - OPTIONS --library ${CMAKE_SHARED_LIBRARY_PREFIX}podioPythonStoreDict${CMAKE_SHARED_MODULE_SUFFIX}) -set_target_properties(podioPythonStoreDict-dictgen PROPERTIES EXCLUDE_FROM_ALL TRUE) -target_sources(podioPythonStoreDict PRIVATE podioPythonStoreDict.cxx) +PODIO_ADD_LIB_AND_DICT(podioPythonStore "${python_headers}" "${python_sources}" python_selection.xml) +target_link_libraries(podioPythonStore PUBLIC podio::podio) +target_link_libraries(podioPythonStore PRIVATE podio::podioRootIO) -# SIO I/O library +# --- SIO I/O functionality and corresponding dictionary if(ENABLE_SIO) - add_library(podioSioIO SHARED ${sio_sources}) - add_library(podio::podioSioIO ALIAS podioSioIO) - - target_include_directories(podioSioIO PUBLIC - $ - $) + SET(sio_sources + SIOReader.cc + SIOWriter.cc + SIOBlockUserData.cc + SIOBlock.cc + SIOFrameWriter.cc + SIOFrameReader.cc + SIOFrameData.cc + ) + + SET(sio_headers + ${CMAKE_SOURCE_DIR}/include/podio/SIOFrameReader.h + ${CMAKE_SOURCE_DIR}/include/podio/SIOFrameWriter.h + ) + + PODIO_ADD_LIB_AND_DICT(podioSioIO "${sio_headers}" "${sio_sources}" sio_selection.xml) target_link_libraries(podioSioIO PUBLIC podio::podio SIO::sio ${CMAKE_DL_LIBS} ${PODIO_FS_LIBS}) - # also make the python EventStore understand SIO - target_link_libraries(podioPythonStore podioSioIO) + # Make sure the legacy python bindings know about the SIO backend + target_link_libraries(podioPythonStore PRIVATE podioSioIO) target_compile_definitions(podioPythonStore PRIVATE PODIO_ENABLE_SIO=1) - LIST(APPEND INSTALL_LIBRARIES podioSioIO) + LIST(APPEND INSTALL_LIBRARIES podioSioIO podioSioIODict) endif() -install(TARGETS podio podioDict podioPythonStoreDict podioRootIO ${INSTALL_LIBRARIES} +# --- Install everything +install(TARGETS podio podioDict podioPythonStore podioPythonStoreDict podioRootIO podioRootIODict ${INSTALL_LIBRARIES} EXPORT podioTargets DESTINATION "${CMAKE_INSTALL_LIBDIR}") -install(DIRECTORY ${PROJECT_SOURCE_DIR}/include/podio DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") +# Only install the necessary headers +if (ENABLE_SIO) + install(DIRECTORY ${PROJECT_SOURCE_DIR}/include/podio DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") +else() + install(DIRECTORY ${PROJECT_SOURCE_DIR}/include/podio DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + REGEX SIO.*\\.h$ EXCLUDE ) +endif() + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/podioDictDict.rootmap ${CMAKE_CURRENT_BINARY_DIR}/libpodioDict_rdict.pcm + ${CMAKE_CURRENT_BINARY_DIR}/podioRootIODictDict.rootmap + ${CMAKE_CURRENT_BINARY_DIR}/libpodioRootIODict_rdict.pcm ${CMAKE_CURRENT_BINARY_DIR}/podioPythonStoreDictDict.rootmap ${CMAKE_CURRENT_BINARY_DIR}/libpodioPythonStoreDict_rdict.pcm DESTINATION "${CMAKE_INSTALL_LIBDIR}") + +if (ENABLE_SIO) + install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/podioSioIODictDict.rootmap + ${CMAKE_CURRENT_BINARY_DIR}/libpodioSioIODict_rdict.pcm + DESTINATION "${CMAKE_INSTALL_LIBDIR}" + ) +endif() diff --git a/src/ROOTFrameReader.cc b/src/ROOTFrameReader.cc index ce761bd4d..93ac7d9e8 100644 --- a/src/ROOTFrameReader.cc +++ b/src/ROOTFrameReader.cc @@ -214,7 +214,7 @@ void ROOTFrameReader::openFiles(const std::vector& filenames) { // Do some work up front for setting up categories and setup all the chains // and record the available categories. The rest of the setup follows on // demand when the category is first read - m_availCategories = getAvailableCategories(m_metaChain.get()); + m_availCategories = ::podio::getAvailableCategories(m_metaChain.get()); for (const auto& cat : m_availCategories) { auto [it, _] = m_categories.try_emplace(cat, std::make_unique(cat.c_str())); for (const auto& fn : filenames) { @@ -231,6 +231,15 @@ unsigned ROOTFrameReader::getEntries(const std::string& name) const { return 0; } +std::vector ROOTFrameReader::getAvailableCategories() const { + std::vector cats; + cats.reserve(m_categories.size()); + for (const auto& [cat, _] : m_categories) { + cats.emplace_back(cat); + } + return cats; +} + std::tuple, std::vector>> createCollectionBranches(TChain* chain, const podio::CollectionIDTable& idTable, const std::vector& collInfo) { diff --git a/src/SIOBlock.cc b/src/SIOBlock.cc index c521b694c..55fa62871 100644 --- a/src/SIOBlock.cc +++ b/src/SIOBlock.cc @@ -222,6 +222,16 @@ SIOFileTOCRecord::PositionType SIOFileTOCRecord::getPosition(const std::string& return 0; } +std::vector SIOFileTOCRecord::getRecordNames() const { + std::vector cats; + cats.reserve(m_recordMap.size()); + for (const auto& [cat, _] : m_recordMap) { + cats.emplace_back(cat); + } + + return cats; +} + void SIOFileTOCRecordBlock::read(sio::read_device& device, sio::version_type) { int size; device.data(size); diff --git a/src/SIOFrameReader.cc b/src/SIOFrameReader.cc index 0ad6d281e..7b87d31c3 100644 --- a/src/SIOFrameReader.cc +++ b/src/SIOFrameReader.cc @@ -73,6 +73,10 @@ std::unique_ptr SIOFrameReader::readEntry(const std::string& name, return readNextEntry(name); } +std::vector SIOFrameReader::getAvailableCategories() const { + return m_tocRecord.getRecordNames(); +} + unsigned SIOFrameReader::getEntries(const std::string& name) const { return m_tocRecord.getNRecords(name); } diff --git a/src/root_selection.xml b/src/root_selection.xml new file mode 100644 index 000000000..41fd14a8a --- /dev/null +++ b/src/root_selection.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/sio_selection.xml b/src/sio_selection.xml new file mode 100644 index 000000000..78d43d3ca --- /dev/null +++ b/src/sio_selection.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b67d80585..c82ba3092 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -164,15 +164,18 @@ if (TARGET read_sio) set_property(TEST check_benchmark_outputs_sio PROPERTY DEPENDS read_timed_sio write_timed_sio) endif() -add_test( NAME pyunittest COMMAND python -m unittest discover -s ${CMAKE_SOURCE_DIR}/python) +add_test( NAME pyunittest COMMAND python -m unittest discover -s ${CMAKE_SOURCE_DIR}/python/podio) set_property(TEST pyunittest PROPERTY ENVIRONMENT LD_LIBRARY_PATH=${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/src:$:$ENV{LD_LIBRARY_PATH} PYTHONPATH=${CMAKE_SOURCE_DIR}/python:$ENV{PYTHONPATH} - ROOT_INCLUDE_PATH=${CMAKE_SOURCE_DIR}/tests/datamodel + ROOT_INCLUDE_PATH=${CMAKE_SOURCE_DIR}/tests/datamodel:${CMAKE_SOURCE_DIR}/include:$ENV{ROOT_INCLUDE_PATH} SKIP_SIO_TESTS=$> ) -set_property(TEST pyunittest PROPERTY DEPENDS write) +set_property(TEST pyunittest PROPERTY DEPENDS write write_frame_root) +if (TARGET write_sio) + set_property(TEST pyunittest PROPERTY DEPENDS write_sio write_frame_sio) +endif() # Customize CTest to potentially disable some of the tests with known problems configure_file(CTestCustom.cmake ${CMAKE_BINARY_DIR}/CTestCustom.cmake) diff --git a/tests/read.py b/tests/read.py index 63026b957..3428b3c60 100644 --- a/tests/read.py +++ b/tests/read.py @@ -4,7 +4,7 @@ from __future__ import absolute_import, unicode_literals, print_function -from EventStore import EventStore +from podio.EventStore import EventStore if __name__ == '__main__': diff --git a/tests/write_frame.h b/tests/write_frame.h index f7ccdc267..e72ae6b50 100644 --- a/tests/write_frame.h +++ b/tests/write_frame.h @@ -366,6 +366,7 @@ podio::Frame makeFrame(int iFrame) { frame.putParameter("UserEventWeight", 100.f * iFrame); frame.putParameter("UserEventName", " event_number_" + std::to_string(iFrame)); frame.putParameter("SomeVectorData", {1, 2, 3, 4}); + frame.putParameter("SomeVectorData", {"just", "some", "strings"}); return frame; } diff --git a/tools/podio-dump b/tools/podio-dump index ab4efbaf9..fcae97b1f 100755 --- a/tools/podio-dump +++ b/tools/podio-dump @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """podio-dump tool to dump contents of podio files""" -from EventStore import EventStore +from podio.EventStore import EventStore def dump_evt_overview(event, ievt):