Split util tests, organize more string functions

src/libexpr/eval-cache.cc

@@ -4,6 +4,8 @@
 #include "eval.hh"
 #include "eval-inline.hh"
 #include "store-api.hh"
+// Need specialization involving `SymbolStr` just in this one module.
+#include "strings-inline.hh"
 
 namespace nix::eval_cache {
 

src/libutil/strings-inline.hh

@@ -4,6 +4,21 @@
 
 namespace nix {
 
+template<class C>
+C tokenizeString(std::string_view s, std::string_view separators)
+{
+    C result;
+    auto pos = s.find_first_not_of(separators, 0);
+    while (pos != s.npos) {
+        auto end = s.find_first_of(separators, pos + 1);
+        if (end == s.npos)
+            end = s.size();
+        result.insert(result.end(), std::string(s, pos, end - pos));
+        pos = s.find_first_not_of(separators, end);
+    }
+    return result;
+}
+
 template<class C>
 std::string concatStringsSep(const std::string_view sep, const C & ss)
 {
@@ -28,4 +43,30 @@ std::string concatStringsSep(const std::string_view sep, const C & ss)
     return s;
 }
 
+template<class C>
+std::string dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss)
+{
+    size_t size = 0;
+
+    // TODO? remove to make sure we don't rely on the empty item ignoring behavior,
+    //       or just get rid of this function by understanding the remaining calls.
+    // for (auto & i : ss) {
+    //     // Make sure we don't rely on the empty item ignoring behavior
+    //     assert(!i.empty());
+    //     break;
+    // }
+
+    // need a cast to string_view since this is also called with Symbols
+    for (const auto & s : ss)
+        size += sep.size() + std::string_view(s).size();
+    std::string s;
+    s.reserve(size);
+    for (auto & i : ss) {
+        if (s.size() != 0)
+            s += sep;
+        s += i;
+    }
+    return s;
+}
+
 } // namespace nix

src/libutil/strings.cc

@@ -1,12 +1,15 @@
 #include <string>
 
 #include "strings-inline.hh"
-#include "util.hh"
 
 namespace nix {
 
-template std::string concatStringsSep(std::string_view, const Strings &);
-template std::string concatStringsSep(std::string_view, const StringSet &);
+template std::list<std::string> tokenizeString(std::string_view s, std::string_view separators);
+template std::set<std::string> tokenizeString(std::string_view s, std::string_view separators);
+template std::vector<std::string> tokenizeString(std::string_view s, std::string_view separators);
+
+template std::string concatStringsSep(std::string_view, const std::list<std::string> &);
+template std::string concatStringsSep(std::string_view, const std::set<std::string> &);
 template std::string concatStringsSep(std::string_view, const std::vector<std::string> &);
 
 typedef std::string_view strings_2[2];
@@ -16,4 +19,8 @@ template std::string concatStringsSep(std::string_view, const strings_3 &);
 typedef std::string_view strings_4[4];
 template std::string concatStringsSep(std::string_view, const strings_4 &);
 
+template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::list<std::string> &);
+template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set<std::string> &);
+template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector<std::string> &);
+
 } // namespace nix

src/libutil/strings.hh

@@ -8,6 +8,18 @@
 
 namespace nix {
 
+/**
+ * String tokenizer.
+ *
+ * See also `basicSplitString()`, which preserves empty strings between separators, as well as at the start and end.
+ */
+template<class C>
+C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r");
+
+extern template std::list<std::string> tokenizeString(std::string_view s, std::string_view separators);
+extern template std::set<std::string> tokenizeString(std::string_view s, std::string_view separators);
+extern template std::vector<std::string> tokenizeString(std::string_view s, std::string_view separators);
+
 /**
  * Concatenate the given strings with a separator between the elements.
  */
@@ -18,4 +30,20 @@ extern template std::string concatStringsSep(std::string_view, const std::list<s
 extern template std::string concatStringsSep(std::string_view, const std::set<std::string> &);
 extern template std::string concatStringsSep(std::string_view, const std::vector<std::string> &);
 
+/**
+ * Ignore any empty strings at the start of the list, and then concatenate the
+ * given strings with a separator between the elements.
+ *
+ * @deprecated This function exists for historical reasons. You probably just
+ *             want to use `concatStringsSep`.
+ */
+template<class C>
+[[deprecated(
+    "Consider removing the empty string dropping behavior. If acceptable, use concatStringsSep instead.")]] std::string
+dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss);
+
+extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::list<std::string> &);
+extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set<std::string> &);
+extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector<std::string> &);
+
 }

src/libutil/util.cc

@@ -53,24 +53,6 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss)
 //////////////////////////////////////////////////////////////////////
 
 
-template<class C> C tokenizeString(std::string_view s, std::string_view separators)
-{
-    C result;
-    auto pos = s.find_first_not_of(separators, 0);
-    while (pos != s.npos) {
-        auto end = s.find_first_of(separators, pos + 1);
-        if (end == s.npos) end = s.size();
-        result.insert(result.end(), std::string(s, pos, end - pos));
-        pos = s.find_first_not_of(separators, end);
-    }
-    return result;
-}
-
-template Strings tokenizeString(std::string_view s, std::string_view separators);
-template StringSet tokenizeString(std::string_view s, std::string_view separators);
-template std::vector<std::string> tokenizeString(std::string_view s, std::string_view separators);
-
-
 std::string chomp(std::string_view s)
 {
     size_t i = s.find_last_not_of(" \n\r\t");

src/libutil/util.hh

@@ -28,49 +28,11 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss);
 MakeError(FormatError, Error);
 
 
-/**
- * String tokenizer.
- */
-template<class C> C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r");
-
-
-/**
- * Ignore any empty strings at the start of the list, and then concatenate the
- * given strings with a separator between the elements.
- *
- * @deprecated This function exists for historical reasons. You probably just
- *             want to use `concatStringsSep`.
- */
-template<class C>
-[[deprecated("Consider removing the empty string dropping behavior. If acceptable, use concatStringsSep instead.")]]
-std::string dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss)
-{
-    size_t size = 0;
-
-    // TODO? remove to make sure we don't rely on the empty item ignoring behavior,
-    //       or just get rid of this function by understanding the remaining calls.
-    // for (auto & i : ss) {
-    //     // Make sure we don't rely on the empty item ignoring behavior
-    //     assert(!i.empty());
-    //     break;
-    // }
-
-    // need a cast to string_view since this is also called with Symbols
-    for (const auto & s : ss) size += sep.size() + std::string_view(s).size();
-    std::string s;
-    s.reserve(size);
-    for (auto & i : ss) {
-        if (s.size() != 0) s += sep;
-        s += i;
-    }
-    return s;
-}
-
-template<class ... Parts>
-auto concatStrings(Parts && ... parts)
+template<class... Parts>
+auto concatStrings(Parts &&... parts)
     -> std::enable_if_t<(... && std::is_convertible_v<Parts, std::string_view>), std::string>
 {
-    std::string_view views[sizeof...(parts)] = { parts... };
+    std::string_view views[sizeof...(parts)] = {parts...};
     return concatStringsSep({}, views);
 }