libexpr: split yacc prologue & epilogue (NFC)

This is an NFC PR that splits epilogue & prologue from parser.

As mentioned in #8812, we can add static checking tools & auto
formatting for these files, but if it is written .y directly, the clang
parser cannot understand the code is actually "C++".

src/libexpr/lexer-prologue.cpp

@@ -0,0 +1,93 @@
+#ifdef __clang__
+#pragma clang diagnostic ignored "-Wunneeded-internal-declaration"
+#endif
+
+#include <boost/lexical_cast.hpp>
+
+#include "nixexpr.hh"
+#include "parser-tab.hh"
+
+using namespace nix;
+
+namespace nix {
+
+static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data)
+{
+    return data->state.positions.add(data->origin, loc.first_line, loc.first_column);
+}
+
+#define CUR_POS makeCurPos(*yylloc, data)
+
+// backup to recover from yyless(0)
+thread_local YYLTYPE prev_yylloc;
+
+static void initLoc(YYLTYPE * loc)
+{
+    loc->first_line = loc->last_line = 1;
+    loc->first_column = loc->last_column = 1;
+}
+
+static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
+{
+    prev_yylloc = *loc;
+
+    loc->first_line = loc->last_line;
+    loc->first_column = loc->last_column;
+
+    for (size_t i = 0; i < len; i++) {
+        switch (*s++) {
+        case '\r':
+            if (*s == '\n') { /* cr/lf */
+                i++;
+                s++;
+            }
+            /* fall through */
+        case '\n':
+            ++loc->last_line;
+            loc->last_column = 1;
+            break;
+        default:
+            ++loc->last_column;
+        }
+    }
+}
+
+// we make use of the fact that the parser receives a private copy of the input
+// string and can munge around in it.
+static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
+{
+    char * result = s;
+    char * t = s;
+    char c;
+    // the input string is terminated with *two* NULs, so we can safely take
+    // *one* character after the one being checked against.
+    while ((c = *s++)) {
+        if (c == '\\') {
+            c = *s++;
+            if (c == 'n')
+                *t = '\n';
+            else if (c == 'r')
+                *t = '\r';
+            else if (c == 't')
+                *t = '\t';
+            else
+                *t = c;
+        } else if (c == '\r') {
+            /* Normalise CR and CR/LF into LF. */
+            *t = '\n';
+            if (*s == '\n')
+                s++; /* cr/lf */
+        } else
+            *t = c;
+        t++;
+    }
+    return {result, size_t(t - result)};
+}
+
+}
+
+#define YY_USER_INIT initLoc(yylloc)
+#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng);
+
+#define PUSH_STATE(state) yy_push_state(state, yyscanner)
+#define POP_STATE() yy_pop_state(yyscanner)

src/libexpr/lexer.l

@@ -15,98 +15,7 @@
 
 
 %{
-#ifdef __clang__
-#pragma clang diagnostic ignored "-Wunneeded-internal-declaration"
-#endif
-
-#include <boost/lexical_cast.hpp>
-
-#include "nixexpr.hh"
-#include "parser-tab.hh"
-
-using namespace nix;
-
-namespace nix {
-
-static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data)
-{
-    return data->state.positions.add(data->origin, loc.first_line, loc.first_column);
-}
-
-#define CUR_POS makeCurPos(*yylloc, data)
-
-// backup to recover from yyless(0)
-thread_local YYLTYPE prev_yylloc;
-
-static void initLoc(YYLTYPE * loc)
-{
-    loc->first_line = loc->last_line = 1;
-    loc->first_column = loc->last_column = 1;
-}
-
-static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
-{
-    prev_yylloc = *loc;
-
-    loc->first_line = loc->last_line;
-    loc->first_column = loc->last_column;
-
-    for (size_t i = 0; i < len; i++) {
-       switch (*s++) {
-       case '\r':
-           if (*s == '\n') { /* cr/lf */
-               i++;
-               s++;
-           }
-           /* fall through */
-       case '\n':
-           ++loc->last_line;
-           loc->last_column = 1;
-           break;
-       default:
-           ++loc->last_column;
-       }
-    }
-}
-
-
-// we make use of the fact that the parser receives a private copy of the input
-// string and can munge around in it.
-static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
-{
-    char * result = s;
-    char * t = s;
-    char c;
-    // the input string is terminated with *two* NULs, so we can safely take
-    // *one* character after the one being checked against.
-    while ((c = *s++)) {
-        if (c == '\\') {
-            c = *s++;
-            if (c == 'n') *t = '\n';
-            else if (c == 'r') *t = '\r';
-            else if (c == 't') *t = '\t';
-            else *t = c;
-        }
-        else if (c == '\r') {
-            /* Normalise CR and CR/LF into LF. */
-            *t = '\n';
-            if (*s == '\n') s++; /* cr/lf */
-        }
-        else *t = c;
-        t++;
-    }
-    return {result, size_t(t - result)};
-}
-
-
-}
-
-#define YY_USER_INIT initLoc(yylloc)
-#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng);
-
-#define PUSH_STATE(state) yy_push_state(state, yyscanner)
-#define POP_STATE() yy_pop_state(yyscanner)
-
+#include "lexer-prologue.cpp"
 %}
 
 

src/libexpr/local.mk

@@ -28,10 +28,10 @@ libexpr_LDFLAGS_PROPAGATED = $(BDW_GC_LIBS)
 
 libexpr_ORDER_AFTER := $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
-$(d)/parser-tab.cc $(d)/parser-tab.hh: $(d)/parser.y
+$(d)/parser-tab.cc $(d)/parser-tab.hh: $(d)/parser.y $(d)/parser-prologue.cpp $(d)/parser-epilogue.cpp
 	$(trace-gen) bison -v -o $(libexpr_DIR)/parser-tab.cc $< -d
 
-$(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l
+$(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l $(d)/lexer-prologue.cpp
 	$(trace-gen) flex --outfile $(libexpr_DIR)/lexer-tab.cc --header-file=$(libexpr_DIR)/lexer-tab.hh $<
 
 clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh

src/libexpr/parser-epilogue.cpp

@@ -0,0 +1,178 @@
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "parser-tab.hh"
+
+#include "lexer-tab.hh"
+
+#include "eval.hh"
+#include "filetransfer.hh"
+#include "fetchers.hh"
+#include "store-api.hh"
+#include "flake/flake.hh"
+
+namespace nix {
+
+unsigned long Expr::nrExprs = 0;
+
+Expr * EvalState::parse(
+    char * text, size_t length, Pos::Origin origin, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv)
+{
+    yyscan_t scanner;
+    ParseData data{
+        .state = *this,
+        .symbols = symbols,
+        .basePath = basePath,
+        .origin = {origin},
+    };
+
+    yylex_init(&scanner);
+    yy_scan_buffer(text, length, scanner);
+    int res = yyparse(scanner, &data);
+    yylex_destroy(scanner);
+
+    if (res)
+        throw ParseError(data.error.value());
+
+    data.result->bindVars(*this, staticEnv);
+
+    return data.result;
+}
+
+SourcePath resolveExprPath(const SourcePath & path)
+{
+    /* If `path' is a symlink, follow it.  This is so that relative
+       path references work. */
+    auto path2 = path.resolveSymlinks();
+
+    /* If `path' refers to a directory, append `/default.nix'. */
+    if (path2.lstat().type == InputAccessor::tDirectory)
+        return path2 + "default.nix";
+
+    return path2;
+}
+
+Expr * EvalState::parseExprFromFile(const SourcePath & path)
+{
+    return parseExprFromFile(path, staticBaseEnv);
+}
+
+Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv)
+{
+    auto buffer = path.readFile();
+    // readFile hopefully have left some extra space for terminators
+    buffer.append("\0\0", 2);
+    return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv);
+}
+
+Expr *
+EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv)
+{
+    auto s = make_ref<std::string>(std::move(s_));
+    s->append("\0\0", 2);
+    return parse(s->data(), s->size(), Pos::String{.source = s}, basePath, staticEnv);
+}
+
+Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath)
+{
+    return parseExprFromString(std::move(s), basePath, staticBaseEnv);
+}
+
+Expr * EvalState::parseStdin()
+{
+    // Activity act(*logger, lvlTalkative, "parsing standard input");
+    auto buffer = drainFD(0);
+    // drainFD should have left some extra space for terminators
+    buffer.append("\0\0", 2);
+    auto s = make_ref<std::string>(std::move(buffer));
+    return parse(s->data(), s->size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv);
+}
+
+SourcePath EvalState::findFile(const std::string_view path)
+{
+    return findFile(searchPath, path);
+}
+
+SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos)
+{
+    for (auto & i : searchPath.elements) {
+        auto suffixOpt = i.prefix.suffixIfPotentialMatch(path);
+
+        if (!suffixOpt)
+            continue;
+        auto suffix = *suffixOpt;
+
+        auto rOpt = resolveSearchPathPath(i.path);
+        if (!rOpt)
+            continue;
+        auto r = *rOpt;
+
+        Path res = suffix == "" ? r : concatStrings(r, "/", suffix);
+        if (pathExists(res))
+            return CanonPath(canonPath(res));
+    }
+
+    if (hasPrefix(path, "nix/"))
+        return CanonPath(concatStrings(corepkgsPrefix, path.substr(4)));
+
+    debugThrow(
+        ThrownError(
+            {.msg = hintfmt(
+                 evalSettings.pureEval
+                     ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
+                     : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
+                 path),
+             .errPos = positions[pos]}),
+        0, 0);
+}
+
+std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Path & value0)
+{
+    auto & value = value0.s;
+    auto i = searchPathResolved.find(value);
+    if (i != searchPathResolved.end())
+        return i->second;
+
+    std::optional<std::string> res;
+
+    if (EvalSettings::isPseudoUrl(value)) {
+        try {
+            auto storePath =
+                fetchers::downloadTarball(store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath;
+            res = {store->toRealPath(storePath)};
+        } catch (FileTransferError & e) {
+            logWarning({.msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)});
+            res = std::nullopt;
+        }
+    }
+
+    else if (hasPrefix(value, "flake:")) {
+        experimentalFeatureSettings.require(Xp::Flakes);
+        auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false);
+        debug("fetching flake search path element '%s''", value);
+        auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath;
+        res = {store->toRealPath(storePath)};
+    }
+
+    else {
+        auto path = absPath(value);
+        if (pathExists(path))
+            res = {path};
+        else {
+            logWarning({.msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", value)});
+            res = std::nullopt;
+        }
+    }
+
+    if (res)
+        debug("resolved search path element '%s' to '%s'", value, *res);
+    else
+        debug("failed to resolve search path element '%s'", value);
+
+    searchPathResolved[value] = res;
+    return res;
+}
+
+}