feat/improve test setup (#132)

README.md

@@ -93,22 +93,18 @@ Optional: [install direnv](https://github.com/direnv/direnv#basic-installation),
 
 ## Development
 
-#### Just
+#### Running tests
 
-Nobody wants to memorize complex inputs for doing things in the terminal. For that reason, we use `just` as a command runner to save and run common tasks. By `just` writing:
+To run all tests you can use `nix flake check`, it will build all packages and run all tests.
 
-```bash
-just
-```
-
-It will describe available commands (commands are defined in it's respective `Justfile`, so have a look)!
+To execute a specific test you can use `, test -h` which will provide more information.
 
 #### Formatting
 
-Formatting will be run via `pre-commit` hook if you are in the `nix shell`, otherwise you can manually format using the `format` command like so:
+You can manually format using the `format` command like so:
 
 ```bash
-just fmt
+, fmt
 ```
 
 ## Applications

flake.nix

@@ -73,7 +73,6 @@
           ./nix
           ./packages
           ./modules
-          ./tests
         ];
         systems = [
           "x86_64-linux"

modules/clients/execution/default.nix

@@ -2,6 +2,6 @@
   imports = [
     ./erigon.nix
     ./geth.nix
-    ./nethermind.nix
+    ./nethermind
   ];
 }

modules/clients/execution/nethermind/basic.test.nix

@@ -0,0 +1,41 @@
+{
+  systems = ["x86_64-linux"];
+
+  module = {pkgs, ...}: let
+    jwtSecret = pkgs.writeText "jwt-secret" "315228a30b238d15df0bedd570a3e1d21bb3f92588168a26127c2090497cf4b6";
+  in {
+    name = "basic";
+
+    nodes = {
+      basicConf = {
+        # see: https://docs.nethermind.io/nethermind/first-steps-with-nethermind/system-requirements
+        virtualisation.cores = 2;
+        virtualisation.memorySize = 8192;
+
+        services.ethereum.nethermind.sepolia = {
+          enable = true;
+          args = {
+            config = "sepolia";
+            modules.JsonRpc.JwtSecretFile = "${jwtSecret}";
+            modules.Metrics.Enabled = true;
+            modules.Metrics.ExposePort = 1313;
+          };
+        };
+      };
+    };
+
+    testScript = ''
+      start_all()
+
+      with subtest("Minimal (settings = null) config test"):
+          basicConf.wait_for_unit("nethermind-sepolia.service")
+
+          # TODO: Finish properly these tests once PR is merged in upstream https://github.com/NethermindEth/nethermind/pull/4320
+          # basicConf.wait_for_open_port(30303)
+          # basicConf.wait_for_open_port(8545)
+
+          # out = basicConf.succeed("systemctl status nethermind-sepolia.service")
+          # print(out)
+    '';
+  };
+}

modules/clients/execution/nethermind.nix → modules/clients/execution/nethermind/default.nix

@@ -231,7 +231,7 @@ in {
         nethermindName: let
           serviceName = "nethermind-${nethermindName}";
 
-          modulesLib = import ../../lib.nix {inherit lib pkgs;};
+          modulesLib = import ../../../lib.nix {inherit lib pkgs;};
           inherit (modulesLib) mkArgs baseServiceConfig;
         in
           cfg: let

modules/default.nix

@@ -1,4 +1,9 @@
-{lib, ...}: {
+{
+  imports = [
+    ./testing.nix
+  ];
+
+  # create a default nixos module which mixes in all modules
   flake.nixosModules.default = {
     imports = [
       ./clients

modules/testing.nix

@@ -0,0 +1,145 @@
+{
+  self,
+  inputs,
+  ...
+}: {
+  perSystem = {
+    lib,
+    config,
+    system,
+    ...
+  }: let
+    cfg = config.testing;
+
+    # create a custom nixpkgs with our flake packages available
+    pkgs = import inputs.nixpkgs {
+      inherit system;
+      overlays = [
+        self.overlays.default
+      ];
+    };
+  in {
+    ########################################
+    ## Interface
+    ########################################
+    options.testing.checks = with lib;
+      mkOption {
+        type = types.attrsOf types.package;
+        default = {};
+      };
+
+    ########################################
+    ## Implementation
+    ########################################
+    config.testing.checks = with lib; let
+      # import the testing framework
+      nixos-lib = import (pkgs.path + "/nixos/lib") {};
+
+      # traverse the filesystem and capture any files with `.test.nix` suffix
+      eachTest =
+        filterAttrs
+        (_: (hasSuffix ".test.nix"))
+        (fs.flattenTree {
+          tree = fs.rakeLeaves ./.;
+        });
+
+      # examine the `systems` attribute of each test, filtering out any that do not support the current system
+      eachTestForSystem = with lib;
+        filterAttrs
+        (_: v: elem system v.systems)
+        (mapAttrs (_: import) eachTest);
+    in
+      mapAttrs'
+      (name: test: let
+        # cleanup test name to be of the form `test-<module>-<test>`
+        components = reverseList (splitString "." name);
+        testName = "test-${elemAt components 2}-${elemAt components 1}";
+      in
+        nameValuePair testName
+        (nixos-lib.runTest {
+          hostPkgs = pkgs;
+          # speed up evaluation by skipping docs
+          defaults.documentation.enable = lib.mkDefault false;
+
+          # make self available in test modules and our custom pkgs
+          node.specialArgs = {inherit self pkgs;};
+
+          # import all of our flake nixos modules by default
+          defaults.imports = [
+            self.nixosModules.default
+          ];
+
+          # import the test module
+          imports = [test.module];
+        })
+        .config
+        .result)
+      eachTestForSystem;
+
+    ########################################
+    ## Commands
+    ########################################
+    config.mission-control.scripts = let
+      category = "Testing";
+    in {
+      test = {
+        inherit category;
+        description = "Build and run a test";
+        exec = ''
+          set -euo pipefail
+
+          Help() {
+               # Display Help
+               echo "  Build and run a test"
+               echo
+               echo "  Usage:"
+               echo "    , test <name>"
+               echo "    , test <name> --interactive"
+               echo "    , test -s <system> <name>"
+               echo
+               echo "  Arguments:"
+               echo "    <name> If a test package is called 'test-nethermind-basic' then <name> should be 'nethermind-basic'."
+               echo
+               echo "  Options:"
+               echo "    -h --help          Show this screen."
+               echo "    -s --system        Specify the target platform [default: x84_64-linux]."
+               echo "    -i --interactive   Run the test interactively."
+               echo
+          }
+
+          ARGS=$(getopt -o ihs: --long interactive,help,system: -n ', test' -- "$@")
+          eval set -- "$ARGS"
+
+          SYSTEM="x86_64-linux"
+          DRIVER_ARGS=()
+
+          while [ $# -gt 0 ]; do
+            case "$1" in
+                -i | --interactive) DRIVER_ARGS+=("--interactive"); shift;;
+                -s | --system) SYSTEM="$2"; shift 2;;
+                -h | --help) Help; exit 0;;
+                -- ) shift; break;;
+                * ) break;;
+            esac
+          done
+
+          if [ $# -eq 0 ]; then
+            # No test name has been provided
+            Help
+            exit 1
+          fi
+
+          NAME="$1"
+          shift
+
+          # build the test driver
+          nix build ".#checks.$SYSTEM.test-$NAME.driver"
+
+          # run the test driver, passing any remaining arguments
+          set -x
+          ./result/bin/nixos-test-driver "''${DRIVER_ARGS[@]}"
+        '';
+      };
+    };
+  };
+}

nix/checks.nix

@@ -24,9 +24,9 @@ in {
             touch $out
           '';
       }
-      # add integration tests for our custom nixosModules
-      // config.tests
+      # mix in tests
+      // config.testing.checks
       # merge in the package derivations to force a build of all packages during a `nix flake check`
-      // self'.packages;
+      // (with lib; mapAttrs' (n: nameValuePair "package-${n}") self'.packages);
   };
 }

nix/lib/default.nix

@@ -1,5 +1,6 @@
 lib: let
+  fs = import ./fs.nix lib;
   flake = import ./flake.nix lib;
 in {
-  inherit flake;
+  inherit fs flake;
 }

nix/lib/fs.nix

@@ -0,0 +1,45 @@
+lib: rec {
+  flattenTree = {
+    tree,
+    separator ? ".",
+  }: let
+    op = sum: path: val: let
+      pathStr = builtins.concatStringsSep separator path;
+    in
+      if builtins.isPath val
+      then
+        (sum
+          // {
+            "${pathStr}" = val;
+          })
+      else if builtins.isAttrs val
+      then
+        # recurse into that attribute set
+        (recurse sum path val)
+      else
+        # ignore that value
+        sum;
+
+    recurse = sum: path: val:
+      builtins.foldl'
+      (sum: key: op sum (path ++ [key]) val.${key})
+      sum
+      (builtins.attrNames val);
+  in
+    recurse {} [] tree;
+
+  rakeLeaves = dirPath: let
+    collect = file: type: {
+      name = lib.removeSuffix ".nix" file;
+      value = let
+        path = dirPath + "/${file}";
+      in
+        if (type == "regular")
+        then path
+        else rakeLeaves path;
+    };
+
+    files = builtins.readDir dirPath;
+  in
+    lib.filterAttrs (n: v: v != {}) (lib.mapAttrs' collect files);
+}