From cc1118993719d250d10468e90aa36d3c5bdef5e9 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <silvan.mosberger@tweag.io>
Date: Thu, 2 Feb 2023 21:06:59 +0100
Subject: [PATCH] python-bindings: Incremental building

Can now do this:

$ nix develop .#python
$ cd python
$ make

Also reuses the common.sh/init.sh testing scripts
---
 Makefile                             |  3 +-
 flake.nix                            | 52 +++++-----------------------
 python/.gitignore                    |  1 +
 python/Makefile                      |  9 +++++
 python/default.nix                   | 40 +++++++++++++++++++++
 python/local.mk                      |  8 +++++
 python/meson.build                   | 14 ++++++--
 python/src/internal/nix-to-python.hh |  2 +-
 python/src/internal/python-to-nix.hh |  2 +-
 python/src/meson.build               |  2 +-
 python/src/python-module.cc          |  2 +-
 python/test.sh                       |  7 ++++
 12 files changed, 90 insertions(+), 52 deletions(-)
 create mode 100644 python/Makefile
 create mode 100644 python/default.nix
 create mode 100644 python/local.mk
 create mode 100644 python/test.sh

diff --git a/Makefile b/Makefile
index c1a1ce2c7386..70f543e8ae42 100644
--- a/Makefile
+++ b/Makefile
@@ -21,7 +21,8 @@ makefiles = \
   misc/upstart/local.mk \
   doc/manual/local.mk \
   tests/local.mk \
-  tests/plugins/local.mk
+  tests/plugins/local.mk \
+  python/local.mk
 
 -include Makefile.config
 
diff --git a/flake.nix b/flake.nix
index 7b3c7f9cc396..e4892d200090 100644
--- a/flake.nix
+++ b/flake.nix
@@ -388,49 +388,10 @@
               postUnpack = "sourceRoot=$sourceRoot/perl";
             });
 
-            passthru.python-bindings =
-              let
-                python = final.python3;
-              in python.pkgs.buildPythonPackage {
-                name = "nix";
-                format = "other";
-
-                src = self;
-                postUnpack = "sourceRoot=$sourceRoot/python";
-
-                nativeBuildInputs = [
-                  final.buildPackages.ninja
-                  (final.buildPackages.meson.override { python3 = python; })
-                  final.buildPackages.pkgconfig
-                  final.buildPackages.nix
-                ];
-
-                strictDeps = true;
-
-                buildInputs = [
-                  final.nix
-                  final.boost
-                ];
-
-                installCheckPhase = ''
-                  export TEST_ROOT=$(mktemp -d)
-                  export NIX_STATE_DIR=$TEST_ROOT/var/nix
-                  export NIX_STORE_DIR=$TEST_ROOT/store
-                  export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
-                  export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
-
-                  export NIX_CONF_DIR=$TEST_ROOT/etc
-                  mkdir "$NIX_CONF_DIR"
-                  cat > "$NIX_CONF_DIR"/nix.conf <<EOF
-                  sandbox = false
-                  substituters =
-                  EOF
-
-                  export PYTHONPATH=$out/${python.sitePackages}
-
-                  ninja test
-                '';
-              };
+            passthru.python-bindings = final.callPackage ./python {
+              inherit self system;
+              python = final.python3;
+            };
 
             meta.platforms = systems;
           };
@@ -768,7 +729,10 @@
               '';
           }
         )
-        // { default = self.devShells.${system}.stdenv; }
+        // {
+          default = self.devShells.${system}.stdenv;
+          python = self.packages.${system}.nix.python-bindings.shell;
+        }
       );
 
   };
diff --git a/python/.gitignore b/python/.gitignore
index 16d3c4dbbfec..7194ea7277ee 100644
--- a/python/.gitignore
+++ b/python/.gitignore
@@ -1 +1,2 @@
 .cache
+build
diff --git a/python/Makefile b/python/Makefile
new file mode 100644
index 000000000000..0e78c216beaf
--- /dev/null
+++ b/python/Makefile
@@ -0,0 +1,9 @@
+build:
+	$(MAKE) -C .. python/build
+	meson compile -C build
+
+test:
+	$(MAKE) -C .. python/build
+	meson test -C build -v
+
+.PHONY: build test
diff --git a/python/default.nix b/python/default.nix
new file mode 100644
index 000000000000..c7a929801bac
--- /dev/null
+++ b/python/default.nix
@@ -0,0 +1,40 @@
+{ self, system, lib, python, ninja, meson, nix, mkShell }:
+python.pkgs.buildPythonPackage {
+  name = "nix";
+  format = "other";
+
+  src = self;
+
+  strictDeps = true;
+
+  nativeBuildInputs = lib.optionals (nix != null) nix.nativeBuildInputs ++ [
+    ninja
+    (meson.override { python3 = python; })
+    nix
+  ];
+
+  buildInputs = lib.optionals (nix != null) nix.buildInputs ++ [
+    nix
+  ];
+
+  passthru.shell = mkShell {
+    inputsFrom = [
+      self.devShells.${system}.stdenv
+      (nix.python-bindings.override { nix = null; })
+    ];
+  };
+
+  # We need to be able to generate tests/common.sh, which requires running
+  # `make`, which requires having run `autoreconf` and `./configure`.
+  # So we propagate `autoreconfHook` from nix.nativeBuildInputs for that to
+  # work, but after that we also need to cd into the python directory and run the
+  # meson configure phase for the python bindings
+  configurePhase = ''
+    ./configure
+    cd python
+    mesonConfigurePhase
+  '';
+
+  doInstallCheck = true;
+  installCheckPhase = "meson test -v";
+}
diff --git a/python/local.mk b/python/local.mk
new file mode 100644
index 000000000000..aeb1509f2431
--- /dev/null
+++ b/python/local.mk
@@ -0,0 +1,8 @@
+pythonbuild=$(d)/build
+
+clean-files += $(pythonbuild)
+
+$(pythonbuild): src/libexpr/nix-expr.pc $(libdir)/libnixexpr.so src/libmain/nix-main.pc $(libdir)/libnixmain.so
+	rm -rf $(pythonbuild)
+	PKG_CONFIG_PATH=$(libdir)/pkgconfig:$$PKG_CONFIG_PATH meson setup $(pythonbuild) python
+	touch $(pythonbuild)
diff --git a/python/meson.build b/python/meson.build
index 30678c45e8e2..77a26ecf050b 100644
--- a/python/meson.build
+++ b/python/meson.build
@@ -8,7 +8,15 @@ python_dep = dependency('python3', required : true)
 nix_expr_dep = dependency('nix-expr', required: true)
 nix_main_dep = dependency('nix-main', required: true)
 
-python = python_mod.find_python()
-test('python test', python, args : files('tests.py'))
-
 subdir('src')
+
+fs = import('fs')
+
+nix_root = fs.parent(meson.project_source_root())
+run_command('make', '-C', nix_root, 'tests/common.sh', check: true)
+
+env = environment()
+env.prepend('PYTHONPATH', fs.parent(pythonix.full_path()))
+bash = find_program('bash')
+test('python test', bash, args : files('test.sh'), env : env)
+
diff --git a/python/src/internal/nix-to-python.hh b/python/src/internal/nix-to-python.hh
index 27849fd03290..a6266fa90f3d 100644
--- a/python/src/internal/nix-to-python.hh
+++ b/python/src/internal/nix-to-python.hh
@@ -2,7 +2,7 @@
 
 #include <Python.h>
 
-#include <nix/config.h>
+#include <config.h>
 
 #include <eval.hh>
 
diff --git a/python/src/internal/python-to-nix.hh b/python/src/internal/python-to-nix.hh
index c0ddaf012e17..db14b7f7533c 100644
--- a/python/src/internal/python-to-nix.hh
+++ b/python/src/internal/python-to-nix.hh
@@ -1,7 +1,7 @@
 #pragma once
 
 #include <Python.h>
-#include <nix/config.h>
+#include <config.h>
 
 #include <eval.hh>
 #include <optional>
diff --git a/python/src/meson.build b/python/src/meson.build
index 499af79b3692..2eba9de02ab2 100644
--- a/python/src/meson.build
+++ b/python/src/meson.build
@@ -5,7 +5,7 @@ src = [
   'python-module.cc',
 ]
 
-python_mod.extension_module('nix', src,
+pythonix = python_mod.extension_module('nix', src,
     dependencies : [python_dep, nix_expr_dep, nix_main_dep],
     install: true,
     install_dir: python_mod.sysconfig_path('platlib'),
diff --git a/python/src/python-module.cc b/python/src/python-module.cc
index 3f042cee9c0e..25806929bdbe 100644
--- a/python/src/python-module.cc
+++ b/python/src/python-module.cc
@@ -3,7 +3,7 @@
 #include "internal/eval.hh"
 #include "internal/ptr.hh"
 
-#include <nix/config.h>
+#include <config.h>
 
 #include <eval.hh>
 #include <globals.hh>
diff --git a/python/test.sh b/python/test.sh
new file mode 100644
index 000000000000..ca13ce5f6ead
--- /dev/null
+++ b/python/test.sh
@@ -0,0 +1,7 @@
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+cd "$SCRIPT_DIR"/../tests
+
+source init.sh
+
+python "$SCRIPT_DIR"/tests.py