From e8082785073a0d23a09a114745365442e81492e4 Mon Sep 17 00:00:00 2001 From: zmrocze Date: Sat, 2 Dec 2023 22:13:50 +0100 Subject: [PATCH 01/10] Add populate-cache-effect --- effects/populate-cache/default.nix | 179 +++++++++++++++++++++++++++++ flake.lock | 129 +++++++++++++++++++++ flake.nix | 6 + 3 files changed, 314 insertions(+) create mode 100644 effects/populate-cache/default.nix diff --git a/effects/populate-cache/default.nix b/effects/populate-cache/default.nix new file mode 100644 index 0000000..42dba17 --- /dev/null +++ b/effects/populate-cache/default.nix @@ -0,0 +1,179 @@ +{inputs, withSystem, ...}: +let + attic-client = inputs.attic.packages."x86_64-linux".attic-client; + cachix = withSystem "x86_64-linux" ({pkgs, ...}: pkgs.cachix); + in +{ + inputs, + lib, + withSystem, + config, + ... +}: { + imports = [ + inputs.hercules-ci-effects.flakeModule + ]; + + options = { + populate-cache-effect = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Enables HerculesCI effects populating some external cache."; + }; + attic-client-pkg = lib.mkOption { + type = lib.types.package; + description = "Version of the attic-client package to use on \"x86_64-linux\"."; + default = attic-client; + }; + cachix-pkg = lib.mkOption { + type = lib.types.package; + description = "Version of the cachix package to use on \"x86_64-linux\"."; + default = cachix; + }; + caches = lib.mkOption { + description = " + An attribute set, each `name: value` pair translates to an effect under + onPush.default.outputs.effects.populate-cache-effect.name + "; + example = " + { + our-cachix = { + type = \"cachix\"; + secretName = \"our-cachix-token\"; + branches = [ \"master\" ]; + packages = [ pkgs.hello ]; + }; + } + "; + type = lib.types.attrsOf (lib.types.submodule ( + {name, ...}: { + options = { + name = lib.mkOption { + type = lib.types.str; + default = name; + description = '' + Name of the effect. By default it's the attribute name. + ''; + }; + type = lib.mkOption { + type = lib.types.enum ["attic" "cachix"]; + description = "A string \"attic\" or \"cachix\"."; + }; + packages = lib.mkOption { + type = with lib.types; listOf package; + description = "List of packages to push to the cache."; + example = "[ pkgs.hello ]"; + }; + secretName = lib.mkOption { + type = lib.types.str; + description = '' + Name of the HerculesCI secret. See [HerculesCI docs](https://docs.hercules-ci.com/hercules-ci-agent/secrets-json). + The secrets "data" field should contain given data: + + ``` + "data": { + "name": "my-cache-name", + "token": "ey536428341723812", + "endpoint": "https://my-cache-name.com" + } + ``` + + The "endpoint" field is needed for Attic cache. With Cachix cache the "endpoint" field is not read and can be absent. + ''; + }; + branches = lib.mkOption { + type = with lib.types; listOf str; + description = '' + Branches on which we'd like to execute the effect. + ''; + }; + }; + } + )); + }; + }; + }; + + config = let + # file with all the package paths written line by line + # nixpkgs -> [derivation] -> derivation + packagesFile = pkgs: packages: + pkgs.writeText "pushed-paths" + (lib.strings.concatStringsSep "\n" (builtins.map builtins.toString packages)); + + mkAtticPushEffect = { + cacheOptions, + branch, + }: + withSystem "x86_64-linux" ( + { + hci-effects, + pkgs, + ... + }: let + pushEffect = hci-effects.mkEffect { + inputs = [attic-client-pkg]; + secretsMap = { + token-file = "${cacheOptions.secretName}"; + }; + userSetupScript = '' + attic login \ + server-name \ + $(readSecretString token-file .endpoint) \ + $(readSecretString token-file .token) + ''; + effectScript = '' + cat ${packagesFile pkgs cacheOptions.packages} | xargs -s 4096 attic push server-name:$(readSecretString token-file .name) + ''; + }; + in + hci-effects.runIf (builtins.elem branch cacheOptions.branches) pushEffect + ); + + mkCachixPushEffect = { + cacheOptions, + branch, + }: + withSystem "x86_64-linux" ( + { + hci-effects, + pkgs, + ... + }: let + pushEffect = hci-effects.mkEffect { + inputs = [cachix]; + secretsMap = { + token-file = "${cacheOptions.secretName}"; + }; + userSetupScript = '' + cachix authtoken $(readSecretString token-file .token) + ''; + effectScript = '' + cat ${packagesFile pkgs cacheOptions.packages} | cachix push $(readSecretString token-file .name) + ''; + }; + in + hci-effects.runIf (builtins.elem branch cacheOptions.branches) pushEffect + ); + in + lib.mkIf config.populate-cache-effect.enable { + herculesCI = herculesConfig: { + onPush.default.outputs.effects.populate-cache-effect = + lib.attrsets.mapAttrs' (_: cacheOptions: { + inherit (cacheOptions) name; + value = builtins.getAttr "${cacheOptions.type}" { + attic = mkAtticPushEffect { + inherit cacheOptions; + inherit (herculesConfig.config.repo) branch; + }; + cachix = mkCachixPushEffect { + inherit cacheOptions; + inherit (herculesConfig.config.repo) branch; + }; + }; + }) + config.populate-cache-effect.caches; + }; + }; +} diff --git a/flake.lock b/flake.lock index 54ad27d..c526bb0 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,75 @@ { "nodes": { + "attic": { + "inputs": { + "crane": "crane", + "flake-compat": "flake-compat", + "flake-utils": "flake-utils", + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-stable": "nixpkgs-stable" + }, + "locked": { + "lastModified": 1698258239, + "narHash": "sha256-qnhoYYIJ0L/P7H/f56lQUEvpzNlXh4sxuHpRERV+B44=", + "owner": "zhaofengli", + "repo": "attic", + "rev": "e9918bc6be268da6fa97af6ced15193d8a0421c0", + "type": "github" + }, + "original": { + "owner": "zhaofengli", + "repo": "attic", + "type": "github" + } + }, + "crane": { + "inputs": { + "flake-compat": [ + "attic", + "flake-compat" + ], + "flake-utils": [ + "attic", + "flake-utils" + ], + "nixpkgs": [ + "attic", + "nixpkgs" + ], + "rust-overlay": "rust-overlay" + }, + "locked": { + "lastModified": 1677892403, + "narHash": "sha256-/Wi0L1spSWLFj+UQxN3j0mPYMoc7ZoAujpUF/juFVII=", + "owner": "ipetkov", + "repo": "crane", + "rev": "105e27adb70a9890986b6d543a67761cbc1964a2", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-parts": { "inputs": { "nixpkgs-lib": [ @@ -19,6 +89,21 @@ "type": "indirect" } }, + "flake-utils": { + "locked": { + "lastModified": 1667395993, + "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1697723726, @@ -35,11 +120,55 @@ "type": "github" } }, + "nixpkgs-stable": { + "locked": { + "lastModified": 1685004253, + "narHash": "sha256-AbVL1nN/TDicUQ5wXZ8xdLERxz/eJr7+o8lqkIOVuaE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "3e01645c40b92d29f3ae76344a6d654986a91a91", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.05", + "repo": "nixpkgs", + "type": "github" + } + }, "root": { "inputs": { + "attic": "attic", "flake-parts": "flake-parts", "nixpkgs": "nixpkgs" } + }, + "rust-overlay": { + "inputs": { + "flake-utils": [ + "attic", + "crane", + "flake-utils" + ], + "nixpkgs": [ + "attic", + "crane", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1675391458, + "narHash": "sha256-ukDKZw922BnK5ohL9LhwtaDAdCsJL7L6ScNEyF1lO9w=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "383a4acfd11d778d5c2efcf28376cbd845eeaedf", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 1d314f9..46d4513 100644 --- a/flake.nix +++ b/flake.nix @@ -3,6 +3,12 @@ inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; + inputs.attic = { + url = "github:zhaofengli/attic"; + inputs = { + nixpkgs.follows = "nixpkgs"; + }; + }; outputs = inputs@{ self, nixpkgs, flake-parts, ... }: flake-parts.lib.mkFlake { inherit inputs; } From 2bb0b93916acdaf4ba45a05c9f12fac00bd66c63 Mon Sep 17 00:00:00 2001 From: zmrocze Date: Thu, 7 Dec 2023 19:27:22 +0100 Subject: [PATCH 02/10] Remove attic flake input & Rename to push & small fixes --- effects/populate-cache/default.nix | 37 +++++---- flake.lock | 129 ----------------------------- flake.nix | 6 -- 3 files changed, 22 insertions(+), 150 deletions(-) diff --git a/effects/populate-cache/default.nix b/effects/populate-cache/default.nix index 42dba17..b5f23e5 100644 --- a/effects/populate-cache/default.nix +++ b/effects/populate-cache/default.nix @@ -1,40 +1,47 @@ -{inputs, withSystem, ...}: -let - attic-client = inputs.attic.packages."x86_64-linux".attic-client; - cachix = withSystem "x86_64-linux" ({pkgs, ...}: pkgs.cachix); - in { inputs, lib, withSystem, config, ... -}: { +}: let + pkgs-x86_64-linux = withSystem "x86_64-linux" ({pkgs, ...}: pkgs); + in + { imports = [ inputs.hercules-ci-effects.flakeModule ]; options = { - populate-cache-effect = { + push-cache-effect = { enable = lib.mkOption { type = lib.types.bool; default = false; - description = "Enables HerculesCI effects populating some external cache."; + description = '' + Enables an effect that pushes certain outputs to a different binary cache. + + Hercules CI normally pushes everything to the cache(s) configured on the agent. This effect supplements that behavior by letting you push a subset of those to a different cache. + Note that it only pushes the output closure, and not the closures of build dependencies used during the build stage of the CI job. (Unless those closures happen to also be part of the output or "runtime" closure) + ''; }; attic-client-pkg = lib.mkOption { type = lib.types.package; - description = "Version of the attic-client package to use on \"x86_64-linux\"."; - default = attic-client; + description = '' + Version of the attic-client package to use on \"x86_64-linux\". + + Hint: You can use `attic.packages.x86_64-linux.attic-client` from the attic flake. + ''; + default = pkgs-x86_64-linux.attic-client or (throw "push-cache-effect.attic-client-pkg: It seems that attic hasn't been packaged in Nixpkgs (yet?). Please check or set