Python bindings: add Value type

.github/workflows/ci.yml

@@ -20,6 +20,9 @@ jobs:
       with:
         fetch-depth: 0
     - uses: cachix/install-nix-action@v19
+      with:
+        # The sandbox would be disabled by default on Darwin
+        extra_nix_config: "sandbox = true"
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - uses: cachix/cachix-action@v12
       if: needs.check_secrets.outputs.cachix == 'true'

flake.nix

@@ -309,7 +309,13 @@
             name = "nix-${version}";
             inherit version;
 
-            src = self;
+            src = builtins.path {
+              name = "source-nix";
+              path = ./.;
+              filter = path: type:
+                path != toString ./python
+                && path != toString ./flake.nix;
+            };
 
             VERSION_SUFFIX = versionSuffix;
 
@@ -410,6 +416,11 @@
               postUnpack = "sourceRoot=$sourceRoot/perl";
             });
 
+            passthru.python-bindings = final.callPackage ./python {
+              inherit self system;
+              python = final.python3;
+            };
+
             meta.platforms = lib.platforms.unix;
           };
 
@@ -465,6 +476,10 @@
         # Perl bindings for various platforms.
         perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings);
 
+        pythonBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.python-bindings);
+        # TODO: recurseIntoAttrs or combine multiple tests into a single one
+        pythonBindingsTests = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.python-bindings.tests.example-buildPythonApplication);
+
         # Binary tarball for various platforms, containing a Nix store
         # with the closure of 'nix' package, and the second half of
         # the installation script.
@@ -585,6 +600,7 @@
       checks = forAllSystems (system: {
         binaryTarball = self.hydraJobs.binaryTarball.${system};
         perlBindings = self.hydraJobs.perlBindings.${system};
+        pythonBindings = self.hydraJobs.pythonBindings.${system};
         installTests = self.hydraJobs.installTests.${system};
         nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system};
       } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
@@ -665,6 +681,7 @@
             (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) //
             {
               default = self.devShells.${system}.native-stdenvPackages;
+              python = self.packages.${system}.nix.python-bindings.shell;
             }
         );
   };

python/.gitignore

@@ -0,0 +1,3 @@
+build
+# For clang-tools
+.cache

python/Makefile

@@ -0,0 +1,49 @@
+# This Makefile is only used for development of the Python bindings, it is not
+# used in the Nix build.
+# The reason this exists is to make it easier to develop the Python bindings in
+# tandem with the main Nix.
+# The default `make` (defaults to `make build`) calls the main Nix projects
+# `make install` before calling the Python bindings' `meson compile`, therefore
+# ensuring that the needed Nix dynamic libraries are up-to-date
+
+builddir=build
+
+.PHONY: build
+build: nix-install setup-done
+	meson compile -C $(builddir)
+
+.PHONY: test
+test: nix-install setup-done
+	meson test -C $(builddir) -v
+
+.PHONY: clean
+clean:
+	rm -rf $(builddir)
+
+# We include the main Nix projects Makefile.config file to know the $(libdir)
+# variable, which is where Nix is installed in, which we can then use to setup
+# the meson build
+include ../Makefile.config
+
+# We need the file to exist though
+../Makefile.config:
+	@# Throw a good error message in case ./configure hasn't been run yet
+	@[[ -e ../config.status ]] || ( echo "The main Nix project needs to be configured first, see https://nixos.org/manual/nix/stable/contributing/hacking.html" && exit 1 )
+	@# If ./configure is done, we can create the file ourselves
+	$(MAKE) -C .. Makefile.config
+
+.PHONY: setup
+setup: nix-install
+	@# Make meson be able to find the locally-installed Nix
+	PKG_CONFIG_PATH=$(libdir)/pkgconfig:$$PKG_CONFIG_PATH meson setup $(builddir)
+
+.PHONY: setup-done
+setup-done:
+	@# A better error message in case the build directory doesn't exist yet
+	@[[ -e $(builddir) ]] || ( echo "Run 'make setup' once to configure the project build directory" && exit 1 )
+
+.PHONY: nix-install
+nix-install:
+	@# The python bindings don't technically need an _entire_ Nix installation,
+	@# but it seems non-trivial to pick out only exactly the files it actually needs
+	$(MAKE) -C .. install

python/README.md

@@ -0,0 +1,11 @@
+# Python Bindings
+
+This directory contains experimental Python bindings to a small subset of Nix's functionality. These bindings are very fast since they link to the necessary dynamic libraries directly, without having to call the Nix CLI for every operation.
+
+Originally these bindings were created by [@Mic92](https://github.com/Mic92) and called [Pythonix](https://github.com/Mic92/pythonix). However, after Nix changed the internal library interface multiple times, therefore requiring Pythonix to be updated, it wasn't worth the third-party time investment anymore and the Pythonix repository was archived. At that point it only worked for Nix versions below 2.4. By having these bindings upstream, they can be updated right along the C++ code that defines the library interface.
+
+## Documentation
+
+See [index.md](./doc/index.md), which is also rendered in the HTML manual.
+
+To hack on these bindings, see [hacking.md](./doc/hacking.md), also rendered in the HTML manual.

python/default.nix

@@ -0,0 +1,98 @@
+{
+  self,
+  system,
+  lib,
+  python,
+  boost,
+  gdb,
+  clang-tools,
+  pkg-config,
+  ninja,
+  meson,
+  nix,
+  mkShell,
+  enableDebugging,
+  recurseIntoAttrs,
+  isShell ? false,
+}:
+let
+  _python = python;
+in
+let
+  python = _python.override { self = enableDebugging _python; };
+  # Extracts tests/init.sh
+  testScripts = nix.overrideAttrs (old: {
+    name = "nix-test-scripts-${old.version}";
+    outputs = [ "out" ];
+    separateDebugInfo = false;
+    buildPhase = ''
+      make tests/{init.sh,common/vars-and-functions.sh}
+    '';
+    script = ''
+      pushd ${placeholder "out"}/libexec >/dev/null
+      source init.sh
+      popd >/dev/null
+    '';
+    passAsFile = [ "script" ];
+    installPhase = ''
+      rm -rf "$out"
+      mkdir -p "$out"/{libexec/common,share/bash}
+      cp tests/init.sh "$out"/libexec
+      cp tests/common/vars-and-functions.sh "$out"/libexec/common
+
+      cp "$scriptPath" "$out"/share/bash/nix-test.sh
+    '';
+    dontFixup = true;
+  });
+in
+python.pkgs.buildPythonPackage {
+  name = "nix";
+  format = "other";
+
+  src = builtins.path {
+    path = ./.;
+    filter = path: type:
+      path == toString ./meson.build
+      || path == toString ./tests.py
+      || path == toString ./test.sh
+      || lib.hasPrefix (toString ./src) path;
+  };
+
+
+  strictDeps = true;
+
+  nativeBuildInputs = [
+    ninja
+    pkg-config
+    (meson.override { python3 = python; })
+  ] ++ lib.optional (!isShell) nix;
+
+  buildInputs = nix.propagatedBuildInputs ++ [
+    boost
+  ] ++ lib.optional (!isShell) nix;
+
+  mesonBuildType = "release";
+
+  doInstallCheck = true;
+  TEST_SCRIPTS = testScripts;
+  installCheckPhase = "meson test -v";
+
+  passthru = {
+    exampleEnv = python.withPackages (p: [ nix.python-bindings ]);
+    tests = {
+      example-buildPythonApplication = import ./examples/buildPythonApplication {
+        inherit nix system testScripts;
+      };
+    };
+    shell = mkShell {
+      packages = [
+        clang-tools
+        gdb
+      ];
+      TEST_SCRIPTS = testScripts;
+      inputsFrom = [
+        (nix.python-bindings.override { isShell = true; })
+      ];
+    };
+  };
+}

python/doc/api.md

@@ -0,0 +1,3 @@
+# Experimental Python Bindings API
+
+This is the API!

python/doc/hacking.md

@@ -0,0 +1,3 @@
+# Python Bindings Hacking
+
+This is how to hack on the bindings

python/doc/index.md

@@ -0,0 +1,32 @@
+# Experimental Python Bindings
+
+Nix comes with minimal experimental Python bindings that link directly to the necessary dynamic libraries, making them very fast.
+
+## Trying it out
+
+The easiest way to try out the bindings is using the provided example environment:
+
+```
+$ nix run github:NixOS/nix#nix.python-bindings.exampleEnv
+Python 3.10.8 (main, Oct 11 2022, 11:35:05) [GCC 11.3.0] on linux
+Type "help", "copyright", "credits" or "license" for more information.
+>>> import nix
+>>> nix.eval('"Hello ${name}!"', vars=dict(name="Python"))
+'Hello Python!'
+```
+
+For the available functions and their interfaces, see the API section.
+
+## Build integration
+
+In the future these Python bindings will be available from Nixpkgs as `python3Packages.nix`.
+
+Until then the Python bindings are only available from the Nix derivation via the `python-bindings` [passthru attribute](https://nixos.org/manual/nixpkgs/stable/#var-stdenv-passthru). Without any modifications, this derivation is built for the default Python 3 version from the Nixpkgs version used to build Nix. This Python version might not match the Python version of the project you're trying to use them in. Therefore it is recommended to override the bindings with the correct Python version using
+
+```
+nix.python-bindings.override {
+  python = myPythonVersion;
+}
+```
+
+For complete examples, see https://github.com/NixOS/nix/tree/master/python/examples

python/examples/buildPythonApplication/default.nix

@@ -0,0 +1,37 @@
+{
+  system ? builtins.currentSystem,
+  pkgs ? import (fetchTarball {
+    url = "https://github.com/NixOS/nixpkgs/archive/545c7a31e5dedea4a6d372712a18e00ce097d462.tar.gz";
+    sha256 = "1dbsi2ccq8x0hyl8n0hisigj8q19amvj9irzfbgy4b3szb6x2y6l";
+  }) {
+    config = {};
+    overlays = [];
+    inherit system;
+  },
+  nix ? (import ../../..).default,
+  testScripts,
+}:
+let
+  python = pkgs.python3;
+  nixBindings = nix.python-bindings.override { inherit python; };
+in python.pkgs.buildPythonApplication {
+  pname = "hello-nix";
+  version = "0.1";
+  src = builtins.path {
+    path = ./.;
+    filter = path: type:
+      pkgs.lib.hasPrefix (toString ./hello) path
+      || path == toString ./setup.py;
+  };
+  propagatedBuildInputs = [ nixBindings ];
+  doInstallCheck = true;
+  nativeCheckInputs = [
+    nix
+  ];
+  installCheckPhase = ''
+    (
+      source ${testScripts}/share/bash/nix-test.sh
+      $out/bin/hello-nix
+    )
+  '';
+}

python/examples/buildPythonApplication/hello/__init__.py

@@ -0,0 +1,4 @@
+import nix
+
+def greet():
+    print("Evaluating 1 + 1 in Nix gives: " + str(nix.eval("1 + 1")))

python/examples/buildPythonApplication/setup.py

@@ -0,0 +1,12 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="hello-nix",
+    version="0.1",
+    packages=find_packages(),
+    entry_points={
+        'console_scripts': [
+            'hello-nix = hello:greet',
+        ],
+    },
+)

python/include/nix/python/value.hh

@@ -0,0 +1,26 @@
+#pragma once
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include "eval.hh"
+
+namespace nix::python {
+
+typedef struct {
+    PyObject_HEAD
+    // TODO: (performance) flatten this pointer, while making sure
+    //       initialization is correct,
+    //       or allocate these python objects with boehmgc
+    nix::RootValue *value;
+
+    inline nix::RootValue & operator *() {
+        return *value;
+    }
+} ValuePyObject;
+
+extern PyTypeObject ValuePyType;
+
+ValuePyObject *allocValuePyObject(nix::EvalState * state);
+
+}

python/meson.build

@@ -0,0 +1,18 @@
+project('python-nix', 'cpp',
+  version : '0.1.8',
+  license : 'LGPL-2.0',
+)
+
+python_mod = import('python')
+py_installation = python_mod.find_installation()
+
+nix_expr_dep = dependency('nix-expr', required: true)
+nix_main_dep = dependency('nix-main', required: true)
+
+subdir('src')
+
+fs = import('fs')
+
+env = environment()
+env.prepend('PYTHONPATH', fs.parent(nix_bindings.full_path()))
+test('python test', find_program('bash'), args : files('test.sh'), env : env)