diff --git a/flake.nix b/flake.nix index a4ee80b32c8..635dce2192b 100644 --- a/flake.nix +++ b/flake.nix @@ -331,7 +331,7 @@ name = "nix-${version}"; inherit version; - src = sourceByRegexInverted [ "tests/nixos/.*" "tests/installer/.*" ] self; + src = sourceByRegexInverted [ "tests/nixos/.*" "tests/installer/.*" "python/*" "flake.nix" ] self; VERSION_SUFFIX = versionSuffix; outputs = [ "out" "dev" "doc" ]; @@ -439,6 +439,8 @@ postUnpack = "sourceRoot=$sourceRoot/perl"; }); + passthru.python-bindings = prev.callPackage ./python { }; + meta.platforms = lib.platforms.unix; }); @@ -500,6 +502,10 @@ # Perl bindings for various platforms. perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings); + pythonBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.python-bindings); + # TODO: recurseIntoAttrs or combine multiple tests into a single one + pythonBindingsTests = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.python-bindings.tests.example-buildPythonApplication); + # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of # the installation script. @@ -645,6 +651,7 @@ checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; perlBindings = self.hydraJobs.perlBindings.${system}; + pythonBindings = self.hydraJobs.pythonBindings.${system}; installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { @@ -727,6 +734,7 @@ (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // { default = self.devShells.${system}.native-stdenvPackages; + python = self.packages.${system}.nix.python-bindings.shell; } ); }; diff --git a/python/.gitignore b/python/.gitignore new file mode 100644 index 00000000000..6cf006aad99 --- /dev/null +++ b/python/.gitignore @@ -0,0 +1,3 @@ +build +# For clang-tools +.cache diff --git a/python/Makefile b/python/Makefile new file mode 100644 index 00000000000..75c8f61b2d8 --- /dev/null +++ b/python/Makefile @@ -0,0 +1,49 @@ +# This Makefile is only used for development of the Python bindings, it is not +# used in the Nix build. +# The reason this exists is to make it easier to develop the Python bindings in +# tandem with the main Nix. +# The default `make` (defaults to `make build`) calls the main Nix projects +# `make install` before calling the Python bindings' `meson compile`, therefore +# ensuring that the needed Nix dynamic libraries are up-to-date + +builddir=build + +.PHONY: build +build: nix-install setup-done + meson compile -C $(builddir) + +.PHONY: test +test: nix-install setup-done + meson test -C $(builddir) -v + +.PHONY: clean +clean: + rm -rf $(builddir) + +# We include the main Nix projects Makefile.config file to know the $(libdir) +# variable, which is where Nix is installed in, which we can then use to setup +# the meson build +include ../Makefile.config + +# We need the file to exist though +../Makefile.config: + @# Throw a good error message in case ./configure hasn't been run yet + @[[ -e ../config.status ]] || ( echo "The main Nix project needs to be configured first, see https://nixos.org/manual/nix/stable/contributing/hacking.html" && exit 1 ) + @# If ./configure is done, we can create the file ourselves + $(MAKE) -C .. Makefile.config + +.PHONY: setup +setup: nix-install + @# Make meson be able to find the locally-installed Nix + PKG_CONFIG_PATH=$(libdir)/pkgconfig:$$PKG_CONFIG_PATH meson setup $(builddir) + +.PHONY: setup-done +setup-done: + @# A better error message in case the build directory doesn't exist yet + @[[ -e $(builddir) ]] || ( echo "Run 'make setup' once to configure the project build directory" && exit 1 ) + +.PHONY: nix-install +nix-install: + @# The python bindings don't technically need an _entire_ Nix installation, + @# but it seems non-trivial to pick out only exactly the files it actually needs + $(MAKE) -C .. install diff --git a/python/README.md b/python/README.md new file mode 100644 index 00000000000..886f9b6ebb3 --- /dev/null +++ b/python/README.md @@ -0,0 +1,13 @@ +# Python Bindings + +This directory contains experimental Python bindings to a small subset of Nix's functionality. These bindings are very fast since they link to the necessary dynamic libraries directly, without having to call the Nix CLI for every operation. + +Thanks to [@Mic92](https://github.com/Mic92) who wrote [Pythonix](https://github.com/Mic92/pythonix) which these bindings were originally based on, before they became the official bindings that are part of the Nix project. They were upstreamed to decrease maintenance overhead and make sure they are always up-to-date. + +Note that the Python bindings are new and experimental. The interface is likely to change based on known issues and user feedback. + +## Documentation + +See [index.md](./doc/index.md), which is also rendered in the HTML manual. + +To hack on these bindings, see [hacking.md](./doc/hacking.md), also rendered in the HTML manual. diff --git a/python/default.nix b/python/default.nix new file mode 100644 index 00000000000..9bec351d950 --- /dev/null +++ b/python/default.nix @@ -0,0 +1,97 @@ +{ + system, + lib, + python3, + boost, + gdb, + clang-tools, + pkg-config, + ninja, + meson, + nix, + mkShell, + enableDebugging, + recurseIntoAttrs, + isShell ? false, +}: +let + _python = python3; +in +let + python3 = _python.override { self = enableDebugging _python; }; + # Extracts tests/init.sh + testScripts = nix.overrideAttrs (old: { + name = "nix-test-scripts-${old.version}"; + outputs = [ "out" ]; + separateDebugInfo = false; + buildPhase = '' + make tests/{init.sh,common/vars-and-functions.sh} + ''; + script = '' + pushd ${placeholder "out"}/libexec >/dev/null + source init.sh + popd >/dev/null + ''; + passAsFile = [ "script" ]; + installPhase = '' + rm -rf "$out" + mkdir -p "$out"/{libexec/common,share/bash} + cp tests/init.sh "$out"/libexec + cp tests/common/vars-and-functions.sh "$out"/libexec/common + + cp "$scriptPath" "$out"/share/bash/nix-test.sh + ''; + dontFixup = true; + }); +in +python3.pkgs.buildPythonPackage { + name = "nix"; + format = "other"; + + src = builtins.path { + path = ./.; + filter = path: type: + path == toString ./meson.build + || path == toString ./tests.py + || path == toString ./test.sh + || lib.hasPrefix (toString ./src) path; + }; + + + strictDeps = true; + + nativeBuildInputs = [ + ninja + pkg-config + (meson.override { inherit python3; }) + ] ++ lib.optional (!isShell) nix; + + buildInputs = nix.propagatedBuildInputs ++ [ + boost + ] ++ lib.optional (!isShell) nix; + + mesonBuildType = "release"; + + doInstallCheck = true; + TEST_SCRIPTS = testScripts; + installCheckPhase = "meson test -v"; + + passthru = { + exampleEnv = python3.withPackages (p: [ nix.python-bindings ]); + tests = { + example-buildPythonApplication = import ./examples/buildPythonApplication { + inherit nix system testScripts python3; + }; + }; + shell = mkShell { + packages = [ + clang-tools + gdb + ]; + TEST_SCRIPTS = testScripts; + inputsFrom = [ + (nix.python-bindings.override { isShell = true; }) + ]; + }; + }; +} diff --git a/python/doc/api.md b/python/doc/api.md new file mode 100644 index 00000000000..116ed60dc30 --- /dev/null +++ b/python/doc/api.md @@ -0,0 +1,18 @@ +# Experimental Python Bindings + +## callExprString + +```python +nix.callExprString(expression: str, arg) +``` +Parse a nix expression, then call it as a nix function. + +Note that this function is experimental and subject to change based on known issues and feedback. + +**Parameters:**, + `expression` (str): The string containing a nix expression. + `arg`: the argument to pass to the function + +**Returns:** + `result`: the result of the function invocation, converted to python datatypes. + diff --git a/python/doc/hacking.md b/python/doc/hacking.md new file mode 100644 index 00000000000..dff7e4d5e75 --- /dev/null +++ b/python/doc/hacking.md @@ -0,0 +1,3 @@ +# Python Bindings Hacking + +This is how to hack on the bindings diff --git a/python/doc/index.md b/python/doc/index.md new file mode 100644 index 00000000000..3db5cb42436 --- /dev/null +++ b/python/doc/index.md @@ -0,0 +1,32 @@ +# Experimental Python Bindings + +Nix comes with minimal experimental Python bindings that link directly to the necessary dynamic libraries, making them very fast. + +## Trying it out + +The easiest way to try out the bindings is using the provided example environment: + +``` +$ nix run github:NixOS/nix#nix.python-bindings.exampleEnv +Python 3.10.8 (main, Oct 11 2022, 11:35:05) [GCC 11.3.0] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> import nix +>>> nix.callExprString('"Hello ${name}!"', arg={"name": "Python"})) +'Hello Python!' +``` + +For the available functions and their interfaces, see the API section. + +## Build integration + +In the future these Python bindings will be available from Nixpkgs as `python3Packages.nix`. + +Until then the Python bindings are only available from the Nix derivation via the `python-bindings` [passthru attribute](https://nixos.org/manual/nixpkgs/stable/#var-stdenv-passthru). Without any modifications, this derivation is built for the default Python 3 version from the Nixpkgs version used to build Nix. This Python version might not match the Python version of the project you're trying to use them in. Therefore it is recommended to override the bindings with the correct Python version using + +``` +nix.python-bindings.override { + python = myPythonVersion; +} +``` + +For complete examples, see https://github.com/NixOS/nix/tree/master/python/examples diff --git a/python/examples/buildPythonApplication/default.nix b/python/examples/buildPythonApplication/default.nix new file mode 100644 index 00000000000..9bf26a17b63 --- /dev/null +++ b/python/examples/buildPythonApplication/default.nix @@ -0,0 +1,37 @@ +{ + system ? builtins.currentSystem, + pkgs ? import (fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/545c7a31e5dedea4a6d372712a18e00ce097d462.tar.gz"; + sha256 = "1dbsi2ccq8x0hyl8n0hisigj8q19amvj9irzfbgy4b3szb6x2y6l"; + }) { + config = {}; + overlays = []; + inherit system; + }, + python3 ? pkgs.python3, + nix ? (import ../../..).default, + testScripts, +}: +let + nixBindings = nix.python-bindings.override { inherit python3; }; +in python3.pkgs.buildPythonApplication { + pname = "hello-nix"; + version = "0.1"; + src = builtins.path { + path = ./.; + filter = path: type: + pkgs.lib.hasPrefix (toString ./hello) path + || path == toString ./setup.py; + }; + propagatedBuildInputs = [ nixBindings ]; + doInstallCheck = true; + nativeCheckInputs = [ + nix + ]; + installCheckPhase = '' + ( + source ${testScripts}/share/bash/nix-test.sh + $out/bin/hello-nix + ) + ''; +} diff --git a/python/examples/buildPythonApplication/hello/__init__.py b/python/examples/buildPythonApplication/hello/__init__.py new file mode 100644 index 00000000000..179053c3f7a --- /dev/null +++ b/python/examples/buildPythonApplication/hello/__init__.py @@ -0,0 +1,4 @@ +import nix + +def greet(): + print("Evaluating 1 + 1 in Nix gives: " + str(nix.callExprString("_: 1 + 1", None))) diff --git a/python/examples/buildPythonApplication/setup.py b/python/examples/buildPythonApplication/setup.py new file mode 100644 index 00000000000..c49b416d4e1 --- /dev/null +++ b/python/examples/buildPythonApplication/setup.py @@ -0,0 +1,12 @@ +from setuptools import setup, find_packages + +setup( + name="hello-nix", + version="0.1", + packages=find_packages(), + entry_points={ + 'console_scripts': [ + 'hello-nix = hello:greet', + ], + }, +) diff --git a/python/meson.build b/python/meson.build new file mode 100644 index 00000000000..eb31ffca1ba --- /dev/null +++ b/python/meson.build @@ -0,0 +1,17 @@ +project('python-nix', 'cpp', + version : '0.1.8', + license : 'LGPL-2.0', +) + +python_mod = import('python') +py_installation = python_mod.find_installation() + +nix_expr_dep = dependency('nix-expr', required: true) + +subdir('src') + +fs = import('fs') + +env = environment() +env.prepend('PYTHONPATH', fs.parent(nix_bindings.full_path())) +test('python test', find_program('bash'), args : files('test.sh'), env : env) diff --git a/python/src/eval.cc b/python/src/eval.cc new file mode 100644 index 00000000000..b7ddc4ed454 --- /dev/null +++ b/python/src/eval.cc @@ -0,0 +1,89 @@ +#include "internal/eval.hh" +#include "internal/errors.hh" +#include "internal/nix-to-python.hh" +#include "internal/python-to-nix.hh" +#include + +#include +#include +#include + +namespace nix::python { + +const char * currentExceptionTypeName() +{ + int status; + auto res = abi::__cxa_demangle(abi::__cxa_current_exception_type()->name(), 0, 0, &status); + return res ? res : "(null)"; +} + +static PyObject * _eval(const std::string & expression, PyObject * argument) +{ + nix::Strings storePath; + nix::EvalState state(storePath, nix::openStore()); + + auto nixArgument = pythonToNixValue(state, argument); + if (!nixArgument) { + return nullptr; + } + nix::Value fun; + nix::Value v; + + + // Release the GIL, so that other Python threads can be running in parallel + // while the potentially expensive Nix evaluation happens. This is safe + // because we don't operate on Python objects or call the Python/C API in + // this block + // See https://docs.python.org/3/c-api/init.html#thread-state-and-the-global-interpreter-lock + { + PyThreadState *_save; + _save = PyEval_SaveThread(); + Finally reacquireGIL([&] { + PyEval_RestoreThread(_save); + }); + + // TODO: Should the "." be something else here? + auto e = state.parseExprFromString(expression, "."); + state.eval(e, fun); + // TODO: Add position + state.callFunction(fun, *nixArgument, v, noPos); + state.forceValueDeep(v); + } + + nix::PathSet context; + return nixToPythonObject(state, v, context); +} + +// TODO: Rename this function to callExprString, matching the Python name +PyObject * eval(PyObject * self, PyObject * args, PyObject * keywds) +{ + PyObject * expressionObject; + PyObject * argument = nullptr; + + const char * kwlist[] = {"expression", "arg", nullptr}; + + // See https://docs.python.org/3/c-api/arg.html for the magic string + if (!PyArg_ParseTupleAndKeywords( + args, keywds, "UO", const_cast(kwlist), &expressionObject, &argument)) { + return nullptr; + } + + // This handles null bytes in expressions correctly + Py_ssize_t expressionSize; + auto expressionBase = PyUnicode_AsUTF8AndSize(expressionObject, &expressionSize); + if (!expressionBase) { + return nullptr; + } + std::string expression(expressionBase, expressionSize); + + try { + return _eval(expression, argument); + } catch (nix::ThrownError & e) { + return PyErr_Format(ThrownNixError, "%s", e.message().c_str()); + } catch (nix::Error & e) { + return PyErr_Format(NixError, "%s", e.what()); + } catch (...) { + return PyErr_Format(NixError, "unexpected C++ exception: '%s'", currentExceptionTypeName()); + } +} +} // namespace nix::python diff --git a/python/src/internal/errors.hh b/python/src/internal/errors.hh new file mode 100644 index 00000000000..da5ebef6192 --- /dev/null +++ b/python/src/internal/errors.hh @@ -0,0 +1,9 @@ +#pragma once + +#include + +namespace nix::python { + +extern PyObject * NixError; +extern PyObject * ThrownNixError; +} diff --git a/python/src/internal/eval.hh b/python/src/internal/eval.hh new file mode 100644 index 00000000000..4a4861b7a57 --- /dev/null +++ b/python/src/internal/eval.hh @@ -0,0 +1,8 @@ +#pragma once + +#include + +namespace nix::python { + +PyObject * eval(PyObject * self, PyObject * args, PyObject * kwdict); +} diff --git a/python/src/internal/nix-to-python.hh b/python/src/internal/nix-to-python.hh new file mode 100644 index 00000000000..809ff5f3ad1 --- /dev/null +++ b/python/src/internal/nix-to-python.hh @@ -0,0 +1,11 @@ +#pragma once + +#include + +#include + +namespace nix::python { + +PyObject * nixToPythonObject(nix::EvalState & state, nix::Value & v, nix::PathSet & context); +PyObject * _nixToPythonObject(nix::EvalState & state, nix::Value & v, nix::PathSet & context, std::set seen); +} // namespace nix::python diff --git a/python/src/internal/ptr.hh b/python/src/internal/ptr.hh new file mode 100644 index 00000000000..e723802ec0c --- /dev/null +++ b/python/src/internal/ptr.hh @@ -0,0 +1,17 @@ +#pragma once + +#include +#include + +namespace nix::python { + +struct PyObjectDeleter +{ + void operator()(PyObject * const obj) + { + Py_DECREF(obj); + } +}; + +typedef std::unique_ptr PyObjPtr; +} // namespace nix::python diff --git a/python/src/internal/python-to-nix.hh b/python/src/internal/python-to-nix.hh new file mode 100644 index 00000000000..a4853fb0d8c --- /dev/null +++ b/python/src/internal/python-to-nix.hh @@ -0,0 +1,11 @@ +#pragma once + +#include +#include +#include + +namespace nix::python { + +nix::Value * pythonToNixValue(nix::EvalState & state, PyObject * obj); + +} // namespace nix::python diff --git a/python/src/meson.build b/python/src/meson.build new file mode 100644 index 00000000000..49b94a2b752 --- /dev/null +++ b/python/src/meson.build @@ -0,0 +1,18 @@ +src = [ + 'nix-to-python.cc', + 'python-to-nix.cc', + 'eval.cc', + 'python-module.cc', +] + +nix_bindings = py_installation.extension_module('nix', src, + dependencies : [nix_expr_dep], + install: true, + cpp_args: [ + '-std=c++17', + # -Wnon-virtual-dtor is unnecessarily turned on by Meson + # This is fixed in Meson 1.0.0 with https://github.com/mesonbuild/meson/pull/10339 + '-Wno-non-virtual-dtor', + '-fvisibility=hidden', + '-include', 'config.h' + ]) diff --git a/python/src/nix-to-python.cc b/python/src/nix-to-python.cc new file mode 100644 index 00000000000..fec51bf8820 --- /dev/null +++ b/python/src/nix-to-python.cc @@ -0,0 +1,101 @@ +#include + +#include "internal/errors.hh" +#include "internal/nix-to-python.hh" +#include "internal/ptr.hh" + +namespace nix::python { + +PyObject * nixToPythonObject(nix::EvalState & state, nix::Value & v, nix::PathSet & context) { + std::set seen; + return _nixToPythonObject(state, v, context, seen); +} + +PyObject * _nixToPythonObject(nix::EvalState & state, nix::Value & v, nix::PathSet & context, std::set seen) +{ + switch (v.type()) { + case nix::nInt: + return PyLong_FromLong(v.integer); + + case nix::nBool: + if (v.boolean) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + case nix::nString: + copyContext(v, context); + return PyUnicode_FromString(v.string.s); + + case nix::nPath: { + auto p = state.copyPathToStore(context, v.path).to_string(); + return PyUnicode_FromStringAndSize(p.data(), p.length()); + } + + case nix::nNull: + Py_RETURN_NONE; + + case nix::nAttrs: { + if (!v.attrs->empty() && !seen.insert(v.attrs).second) { + PyErr_Format(NixError, "Infinite recursion in data structure"); + return nullptr; + } + auto i = v.attrs->find(state.sOutPath); + if (i == v.attrs->end()) { + PyObjPtr dict(PyDict_New()); + if (!dict) { + return (PyObject *) nullptr; + } + + for (auto & j : *v.attrs) { + const std::string & name = state.symbols[j.name]; + auto value = _nixToPythonObject(state, *j.value, context, seen); + if (!value) { + return nullptr; + } + PyDict_SetItemString(dict.get(), name.c_str(), value); + } + return dict.release(); + } else { + return _nixToPythonObject(state, *i->value, context, seen); + } + } + + case nix::nList: { + if (v.listSize() && !seen.insert(v.listElems()).second) { + PyErr_Format(NixError, "Infinite recursion in data structure"); + return nullptr; + } + PyObjPtr list(PyList_New(v.listSize())); + if (!list) { + return (PyObject *) nullptr; + } + + for (unsigned int n = 0; n < v.listSize(); ++n) { + auto value = _nixToPythonObject(state, *v.listElems()[n], context, seen); + if (!value) { + return nullptr; + } + PyList_SET_ITEM(list.get(), n, value); + } + return list.release(); + } + + case nix::nExternal: + return PyUnicode_FromString(""); + + case nix::nThunk: + return PyUnicode_FromString(""); + + case nix::nFunction: + return PyUnicode_FromString(""); + + case nix::nFloat: + return PyFloat_FromDouble(v.fpoint); + + default: + PyErr_Format(NixError, "cannot convert nix type '%s' to a python object", showType(v).c_str()); + return nullptr; + } +} +} // namespace nix::python diff --git a/python/src/python-module.cc b/python/src/python-module.cc new file mode 100644 index 00000000000..67549577dc3 --- /dev/null +++ b/python/src/python-module.cc @@ -0,0 +1,76 @@ +#include + +#include "internal/eval.hh" +#include "internal/ptr.hh" + +#include +#include +#include + +namespace nix::python { + +#define _public_ __attribute__((visibility("default"))) + +PyObject * ThrownNixError = nullptr; +PyObject * NixError = nullptr; + +static PyMethodDef NixMethods[] = { + {"callExprString", (PyCFunction) eval, METH_VARARGS | METH_KEYWORDS, "Eval nix expression"}, {NULL, NULL, 0, NULL}}; + +static struct PyModuleDef nixmodule = { + PyModuleDef_HEAD_INIT, "nix", "Nix expression bindings", + -1, /* size of per-interpreter state of the module, + or -1 if the module keeps state in global variables. */ + NixMethods}; + +extern "C" _public_ PyObject * PyInit_nix(void) +{ + // These bindings can trigger builds via IFD. This means we need the build-hook to work. + // By default, Nix sets the build-hook to be "$(readlink /proc/self/exe) __build-remote", expecting the current + // binary to be Nix itself. But when we call the Nix library from Python this isn't the case, the current binary is + // Python then So we need to change this default, pointing it to the Nix binary instead + nix::settings.buildHook = nix::settings.nixBinDir + "/nix __build-remote"; + // And by setting buildHook before calling initNix, we can override the defaults without overriding the + // user-provided options from the config files + nix::initLibStore(); + nix::initGC(); + + PyObjPtr m(PyModule_Create(&nixmodule)); + + if (!m) { + return nullptr; + } + + NixError = PyErr_NewExceptionWithDoc( + "nix.NixError", /* char *name */ + "Base exception class for the nix module.", /* char *doc */ + NULL, /* PyObject *base */ + NULL /* PyObject *dict */ + ); + + if (!NixError) { + return nullptr; + } + + if (PyModule_AddObject(m.get(), "NixError", NixError) == -1) { + return nullptr; + } + + ThrownNixError = PyErr_NewExceptionWithDoc( + "nix.ThrownNixError", /* char *name */ + "Base exception class for the nix module.", /* char *doc */ + NixError, /* PyObject *base */ + NULL /* PyObject *dict */ + ); + + if (!ThrownNixError) { + return nullptr; + } + + if (PyModule_AddObject(m.get(), "ThrownNixError", ThrownNixError) == -1) { + return nullptr; + } + + return m.release(); +} +} // namespace nix::python diff --git a/python/src/python-to-nix.cc b/python/src/python-to-nix.cc new file mode 100644 index 00000000000..eb836fd59cd --- /dev/null +++ b/python/src/python-to-nix.cc @@ -0,0 +1,133 @@ +#include + +#include "internal/errors.hh" +#include "internal/ptr.hh" +#include "internal/python-to-nix.hh" + +#include + +namespace nix::python { + +static const char * checkNullByte(const char * str, const Py_ssize_t size) +{ + for (Py_ssize_t i = 0; i < size; i++) { + if (str[0] == '\0') { + PyErr_Format( + NixError, + "invalid character: nix strings are not allowed " + "to contain null bytes"); + return nullptr; + } + } + return str; +} + +static const char * checkAttrKey(PyObject * obj) +{ + Py_ssize_t size = 0; + + if (!PyUnicode_Check(obj)) { + PyObjPtr typeName(PyObject_Str(PyObject_Type(obj))); + if (!typeName) { + return nullptr; + } + auto utf8 = PyUnicode_AsUTF8AndSize(typeName.get(), &size); + if (!utf8) { + return nullptr; + } + PyErr_Format(NixError, "key of nix attrsets must be strings, got type: %s", utf8); + return nullptr; + } + + auto utf8 = PyUnicode_AsUTF8AndSize(obj, &size); + if (!utf8) { + return nullptr; + } + + return checkNullByte(utf8, size); +} + +static std::optional dictToAttrSet(PyObject * obj, nix::EvalState & state) +{ + PyObject *key = nullptr, *val = nullptr; + Py_ssize_t pos = 0; + + nix::ValueMap attrs; + while (PyDict_Next(obj, &pos, &key, &val)) { + auto name = checkAttrKey(key); + if (!name) { + return {}; + } + + auto attrVal = pythonToNixValue(state, val); + if (!attrVal) { + return {}; + } + attrs[state.symbols.create(name)] = attrVal; + } + + return attrs; +} + +nix::Value * pythonToNixValue(nix::EvalState & state, PyObject * obj) +{ + auto v = state.allocValue(); + + if (obj == Py_True || obj == Py_False) { + v->mkBool(obj == Py_True); + } else if (obj == Py_None) { + v->mkNull(); + } else if (PyBytes_Check(obj)) { + // TODO: Bytes should probably not be coerced to strings + auto str = checkNullByte(PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj)); + if (!str) { + return nullptr; + } + + v->mkString(str); + } else if (PyUnicode_Check(obj)) { + Py_ssize_t size; + const char * utf8 = PyUnicode_AsUTF8AndSize(obj, &size); + auto str = checkNullByte(utf8, size); + if (!str) { + return nullptr; + } + + v->mkString(utf8); + } else if (PyFloat_Check(obj)) { + v->mkFloat(PyFloat_AS_DOUBLE(obj)); + } else if (PyLong_Check(obj)) { + v->mkInt(PyLong_AsLong(obj)); + } else if (PyList_Check(obj)) { + v->mkList(PyList_GET_SIZE(obj)); + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(obj); i++) { + auto val = pythonToNixValue(state, PyList_GET_ITEM(obj, i)); + if (!val) { + return nullptr; + } + v->listElems()[i] = val; + } + } else if (PyTuple_Check(obj)) { + v->mkList(PyTuple_GET_SIZE(obj)); + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(obj); i++) { + auto val = pythonToNixValue(state, PyTuple_GET_ITEM(obj, i)); + if (!val) { + return nullptr; + } + v->listElems()[i] = val; + } + } else if (PyDict_Check(obj)) { + auto attrs = dictToAttrSet(obj, state); + if (!attrs) { + return nullptr; + } + auto attrsValue = attrs.value(); + auto bindings = state.buildBindings(attrsValue.size()); + for (auto & attr : attrsValue) { + bindings.insert(attr.first, attr.second); + } + v->mkAttrs(bindings); + } + return v; +} +} // namespace nix::python diff --git a/python/test.sh b/python/test.sh new file mode 100644 index 00000000000..3a1ad8ae250 --- /dev/null +++ b/python/test.sh @@ -0,0 +1,5 @@ +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +source "$TEST_SCRIPTS"/share/bash/nix-test.sh + +python "$SCRIPT_DIR"/tests.py diff --git a/python/tests.py b/python/tests.py new file mode 100644 index 00000000000..f84e8a0f3dd --- /dev/null +++ b/python/tests.py @@ -0,0 +1,69 @@ +import nix +import unittest + + +class TestPythonNix(unittest.TestCase): + def test_dict(self): + val = dict(a=1) + self.assertEqual(nix.callExprString("{ a }: a", arg=dict(a=val)), val) + + def test_string(self): + self.assertEqual(nix.callExprString("{ a }: a", arg=dict(a="foo")), "foo") + + def test_bool(self): + self.assertEqual(nix.callExprString("{ a }: a", arg=dict(a=True)), True) + + def test_none(self): + self.assertEqual(nix.callExprString("{ a }: a", arg=dict(a=None)), None) + + def test_ifd(self): + expression = """ + {}: + builtins.readFile (derivation { + name = "test"; + args = [ "-c" "printf \\"%s\\" test > $out" ]; + builder = "/bin/sh"; + system = builtins.currentSystem; + }) + """ + self.assertEqual(nix.callExprString(expression, arg={}), "test") + + def test_throw(self): + errorString = "hello hi there\ntest" + with self.assertRaises(nix.ThrownNixError) as cm: + nix.callExprString("{ str }: throw str", arg=dict(str=errorString)) + self.assertEqual(cm.exception.args[0], errorString) + + def test_syntax_error(self): + with self.assertRaises(nix.NixError) as cm: + nix.callExprString("{", arg={}) + + def test_GIL_case(self): + with self.assertRaises(nix.ThrownNixError) as cm: + nix.callExprString("{}: { a = throw \"nope\"; }", arg={}) + self.assertEqual(cm.exception.args[0], "nope") + + def test_infinity(self): + with self.assertRaises(nix.NixError): + nix.callExprString("{}: let x = { inherit x; }; in x", arg={}) + + def test_null_expression(self): + # Null characters should be allowed in expressions, even if they aren't + # very useful really, though at least null's should be supported in + # strings in the future https://github.com/NixOS/nix/issues/1307) + self.assertEqual(nix.callExprString("{}: \"ab\x00cd\"", arg={}), "ab") + + def test_throw_null(self): + with self.assertRaises(nix.ThrownNixError) as cm: + nix.callExprString("{}: throw \"hello\x00there\"", arg={}) + self.assertEqual(cm.exception.args[0], "hello") + + def test_booleans(self): + self.assertIs(nix.callExprString("{ a }: assert a == true; a", arg=dict(a=True)), True) + self.assertIs(nix.callExprString("{ a }: assert a == false; a", arg=dict(a=False)), False) + + def test_null(self): + self.assertIs(nix.callExprString("{ a }: assert a == null; a", arg=dict(a=None)), None) + +if __name__ == '__main__': + unittest.main() diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 6a09230812e..d3aa5eccbe3 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -154,6 +154,10 @@ public: : err(e) { } + std::string message() { + return err.msg.str(); + } + #ifdef EXCEPTION_NEEDS_THROW_SPEC ~BaseError() throw () { }; const char * what() const throw () { return calcWhat().c_str(); } diff --git a/tests/common/vars-and-functions.sh.in b/tests/common/vars-and-functions.sh.in index a9e6c802fe5..2ae12cc6b13 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/common/vars-and-functions.sh.in @@ -4,7 +4,7 @@ if [[ -z "${COMMON_VARS_AND_FUNCTIONS_SH_SOURCED-}" ]]; then COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1 -export PS4='+(${BASH_SOURCE[0]}:$LINENO) ' +export PS4='+(${BASH_SOURCE[0]:-}:$LINENO) ' export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default} export NIX_STORE_DIR @@ -20,7 +20,7 @@ export NIX_CONF_DIR=$TEST_ROOT/etc export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket unset NIX_USER_CONF_FILES export _NIX_TEST_SHARED=$TEST_ROOT/shared -if [[ -n $NIX_STORE ]]; then +if [[ -n ${NIX_STORE:-} ]]; then export _NIX_TEST_NO_SANDBOX=1 fi export _NIX_IN_TEST=$TEST_ROOT/shared