From b2911941befb435d0938b2e06dc867dc7bcae9b6 Mon Sep 17 00:00:00 2001 From: Patrick Decat Date: Fri, 23 Jul 2021 15:35:28 +0200 Subject: [PATCH 1/7] feat: Support installing dependencies with poetry --- examples/build-package/README.md | 2 + examples/build-package/main.tf | 59 +++++- .../python3.8-app-poetry/docker/Dockerfile | 6 + .../python3.8-app-poetry/ignore_please.txt | 1 + .../fixtures/python3.8-app-poetry/index.py | 4 + .../fixtures/python3.8-app-poetry/poetry.lock | 33 ++++ .../python3.8-app-poetry/pyproject.toml | 15 ++ .../fixtures/python3.8-app1/docker/Dockerfile | 4 +- .../fixtures/unittests/pyproject-unknown.toml | 2 + examples/simple/main.tf | 2 +- package.py | 168 +++++++++++++++++- test_package_toml.py | 10 ++ tox.ini | 7 + 13 files changed, 300 insertions(+), 13 deletions(-) create mode 100644 examples/fixtures/python3.8-app-poetry/docker/Dockerfile create mode 100644 examples/fixtures/python3.8-app-poetry/ignore_please.txt create mode 100644 examples/fixtures/python3.8-app-poetry/index.py create mode 100644 examples/fixtures/python3.8-app-poetry/poetry.lock create mode 100644 examples/fixtures/python3.8-app-poetry/pyproject.toml create mode 100644 examples/fixtures/unittests/pyproject-unknown.toml create mode 100644 test_package_toml.py create mode 100644 tox.ini diff --git a/examples/build-package/README.md b/examples/build-package/README.md index 739e8b0a..d7e5a84b 100644 --- a/examples/build-package/README.md +++ b/examples/build-package/README.md @@ -36,8 +36,10 @@ Note that this example may create resources which cost money. Run `terraform des | [lambda\_function\_from\_package](#module\_lambda\_function\_from\_package) | ../../ | n/a | | [lambda\_layer](#module\_lambda\_layer) | ../../ | n/a | | [lambda\_layer\_pip\_requirements](#module\_lambda\_layer\_pip\_requirements) | ../.. | n/a | +| [lambda\_layer\_poetry](#module\_lambda\_layer\_poetry) | ../../ | n/a | | [package\_dir](#module\_package\_dir) | ../../ | n/a | | [package\_dir\_pip\_dir](#module\_package\_dir\_pip\_dir) | ../../ | n/a | +| [package\_dir\_poetry](#module\_package\_dir\_poetry) | ../../ | n/a | | [package\_dir\_with\_npm\_install](#module\_package\_dir\_with\_npm\_install) | ../../ | n/a | | [package\_dir\_without\_npm\_install](#module\_package\_dir\_without\_npm\_install) | ../../ | n/a | | [package\_dir\_without\_pip\_install](#module\_package\_dir\_without\_pip\_install) | ../../ | n/a | diff --git a/examples/build-package/main.tf b/examples/build-package/main.tf index 61dd0102..db349a28 100644 --- a/examples/build-package/main.tf +++ b/examples/build-package/main.tf @@ -17,28 +17,52 @@ resource "random_pet" "this" { # Build packages ################# -# Create zip-archive of a single directory where "pip install" will also be executed (default for python runtime) +# Create zip-archive of a single directory where "pip install" will also be executed (default for python runtime with requirements.txt present) module "package_dir" { source = "../../" create_function = false - runtime = "python3.8" - source_path = "${path.module}/../fixtures/python3.8-app1" + build_in_docker = true + runtime = "python3.8" + source_path = "${path.module}/../fixtures/python3.8-app1" + artifacts_dir = "${path.root}/builds/package_dir/" } -# Create zip-archive of a single directory where "pip install" will also be executed (default for python runtime) and set temporary directory for pip install +# Create zip-archive of a single directory where "pip install" will also be executed (default for python runtime with requirements.txt present) and set temporary directory for pip install module "package_dir_pip_dir" { source = "../../" create_function = false - runtime = "python3.8" + build_in_docker = true + runtime = "python3.8" source_path = [{ path = "${path.module}/../fixtures/python3.8-app1" pip_tmp_dir = "${path.cwd}/../fixtures" pip_requirements = "${path.module}/../fixtures/python3.8-app1/requirements.txt" }] + artifacts_dir = "${path.root}/builds/package_dir_pip_dir/" +} + +# Create zip-archive of a single directory where "poetry install" will also be executed +module "package_dir_poetry" { + source = "../../" + + create_function = false + + build_in_docker = true + runtime = "python3.8" + docker_image = "build-python3.8-poetry" + docker_file = "${path.module}/../fixtures/python3.8-app-poetry/docker/Dockerfile" + + source_path = [ + { + path = "${path.module}/../fixtures/python3.8-app-poetry" + poetry_install = true + } + ] + artifacts_dir = "${path.root}/builds/package_dir_poetry/" } # Create zip-archive of a single directory without running "pip install" (which is default for python runtime) @@ -278,8 +302,31 @@ module "lambda_layer" { build_in_docker = true runtime = "python3.8" - docker_image = "public.ecr.aws/sam/build-python3.8" + docker_image = "build-python3.8" docker_file = "${path.module}/../fixtures/python3.8-app1/docker/Dockerfile" + artifacts_dir = "${path.root}/builds/lambda_layer/" +} + +module "lambda_layer_poetry" { + source = "../../" + + create_layer = true + layer_name = "${random_pet.this.id}-layer-poetry-dockerfile" + compatible_runtimes = ["python3.8"] + + source_path = [ + { + path = "${path.module}/../fixtures/python3.8-app-poetry" + poetry_install = true + } + ] + hash_extra = "extra-hash-to-prevent-conflicts-with-module.package_dir" + + build_in_docker = true + runtime = "python3.8" + docker_image = "build-python3.8-poetry" + docker_file = "${path.module}/../fixtures/python3.8-app-poetry/docker/Dockerfile" + artifacts_dir = "${path.root}/builds/lambda_layer_poetry/" } ####################### diff --git a/examples/fixtures/python3.8-app-poetry/docker/Dockerfile b/examples/fixtures/python3.8-app-poetry/docker/Dockerfile new file mode 100644 index 00000000..3c790987 --- /dev/null +++ b/examples/fixtures/python3.8-app-poetry/docker/Dockerfile @@ -0,0 +1,6 @@ +FROM public.ecr.aws/sam/build-python3.9 + +LABEL maintainer="Betajob AS" \ + description="Patched AWS Lambda build container" + +RUN pip install poetry==1.1.13 diff --git a/examples/fixtures/python3.8-app-poetry/ignore_please.txt b/examples/fixtures/python3.8-app-poetry/ignore_please.txt new file mode 100644 index 00000000..30a2f668 --- /dev/null +++ b/examples/fixtures/python3.8-app-poetry/ignore_please.txt @@ -0,0 +1 @@ +This file should not be included in archive. diff --git a/examples/fixtures/python3.8-app-poetry/index.py b/examples/fixtures/python3.8-app-poetry/index.py new file mode 100644 index 00000000..396c5054 --- /dev/null +++ b/examples/fixtures/python3.8-app-poetry/index.py @@ -0,0 +1,4 @@ +def lambda_handler(event, context): + print("Hello from app1!") + + return event diff --git a/examples/fixtures/python3.8-app-poetry/poetry.lock b/examples/fixtures/python3.8-app-poetry/poetry.lock new file mode 100644 index 00000000..b7e48193 --- /dev/null +++ b/examples/fixtures/python3.8-app-poetry/poetry.lock @@ -0,0 +1,33 @@ +[[package]] +name = "colorama" +version = "0.4.4" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "colorful" +version = "0.5.4" +description = "Terminal string styling done right, in Python." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[metadata] +lock-version = "1.1" +python-versions = "^3.6" +content-hash = "65d9f3b221205b259a18285ee8c78c015794fa2e69c66f9ffc836f1758fd594d" + +[metadata.files] +colorama = [ + {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, +] +colorful = [ + {file = "colorful-0.5.4-py2.py3-none-any.whl", hash = "sha256:8d264b52a39aae4c0ba3e2a46afbaec81b0559a99be0d2cfe2aba4cf94531348"}, + {file = "colorful-0.5.4.tar.gz", hash = "sha256:86848ad4e2eda60cd2519d8698945d22f6f6551e23e95f3f14dfbb60997807ea"}, +] diff --git a/examples/fixtures/python3.8-app-poetry/pyproject.toml b/examples/fixtures/python3.8-app-poetry/pyproject.toml new file mode 100644 index 00000000..fc8ea175 --- /dev/null +++ b/examples/fixtures/python3.8-app-poetry/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "python3.8-app-poetry" +version = "0.1.0" +description = "" +authors = ["Your Name "] + +[tool.poetry.dependencies] +python = "^3.6" +colorful = "^0.5.4" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/examples/fixtures/python3.8-app1/docker/Dockerfile b/examples/fixtures/python3.8-app1/docker/Dockerfile index 2b7d1fde..aeab9fee 100644 --- a/examples/fixtures/python3.8-app1/docker/Dockerfile +++ b/examples/fixtures/python3.8-app1/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM lambci/lambda:build-python3.8 as build +FROM public.ecr.aws/sam/build-python3.8 as build LABEL maintainer="Betajob AS" \ description="Patched AWS Lambda build container" @@ -20,7 +20,7 @@ RUN \ && rpmbuild -ba SPECS/automake.spec --nocheck \ && yum install -y RPMS/noarch/* -FROM lambci/lambda:build-python3.8 +FROM public.ecr.aws/sam/build-python3.8 COPY --from=build /root/rpmbuild/RPMS/noarch/*.rpm . RUN yum install -y *.rpm \ && rm *.rpm diff --git a/examples/fixtures/unittests/pyproject-unknown.toml b/examples/fixtures/unittests/pyproject-unknown.toml new file mode 100644 index 00000000..4f0e31e0 --- /dev/null +++ b/examples/fixtures/unittests/pyproject-unknown.toml @@ -0,0 +1,2 @@ +[build-system] +build-backend = "dummy" diff --git a/examples/simple/main.tf b/examples/simple/main.tf index e64f2310..3bff4e4d 100644 --- a/examples/simple/main.tf +++ b/examples/simple/main.tf @@ -301,7 +301,7 @@ module "lambda_function" { # docker_with_ssh_agent = true # docker_file = "${path.module}/../fixtures/python3.8-app1/docker/Dockerfile" # docker_build_root = "${path.module}/../../docker" - # docker_image = "lambci/lambda:build-python3.8" + # docker_image = "public.ecr.aws/sam/build-python3.8" } #### diff --git a/package.py b/package.py index 79b2ea67..761d662a 100644 --- a/package.py +++ b/package.py @@ -615,6 +615,19 @@ def emit_file(fpath, opath): yield from emit_file(f, o) +def get_build_system_from_pyproject_toml(pyproject_file): + # Implement a basic TOML parser because python stdlib does not provide toml support and we probably do not want to add external dependencies + if os.path.isfile(pyproject_file): + with open(pyproject_file) as f: + bs = False + for line in f.readlines(): + if line.startswith("[build-system]"): + bs = True + continue + if bs and line.startswith("build-backend") and "poetry" in line: + return "poetry" + + class BuildPlanManager: """""" @@ -666,6 +679,20 @@ def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None): step('pip', runtime, requirements, prefix, tmp_dir) hash(requirements) + def poetry_install_step(path, prefix=None, required=False): + pyproject_file = path + if os.path.isdir(path): + pyproject_file = os.path.join(path, "pyproject.toml") + if get_build_system_from_pyproject_toml(pyproject_file) != "poetry": + if required: + raise RuntimeError("poetry configuration not found: {}".format(pyproject_file)) + else: + step("poetry", runtime, path, prefix) + hash(pyproject_file) + poetry_lock_file = os.path.join(path, "poetry.lock") + if os.path.isfile(poetry_lock_file): + hash(poetry_lock_file) + def npm_requirements_step(path, prefix=None, required=False, tmp_dir=None): requirements = path if os.path.isdir(path): @@ -741,6 +768,7 @@ def commands_step(path, commands): if runtime.startswith('python'): pip_requirements_step( os.path.join(path, 'requirements.txt')) + poetry_install_step(path) elif runtime.startswith('nodejs'): npm_requirements_step( os.path.join(path, 'package.json')) @@ -758,6 +786,7 @@ def commands_step(path, commands): else: prefix = claim.get('prefix_in_zip') pip_requirements = claim.get('pip_requirements') + poetry_install = claim.get("poetry_install") npm_requirements = claim.get('npm_package_json') runtime = claim.get('runtime', query.runtime) @@ -768,13 +797,16 @@ def commands_step(path, commands): pip_requirements_step(pip_requirements, prefix, required=True, tmp_dir=claim.get('pip_tmp_dir')) + if poetry_install and runtime.startswith("python"): + if path: + poetry_install_step(path, prefix, required=True) + if npm_requirements and runtime.startswith('nodejs'): if isinstance(npm_requirements, bool) and path: npm_requirements_step(path, prefix, required=True, tmp_dir=claim.get('npm_tmp_dir')) else: npm_requirements_step(npm_requirements, prefix, required=True, tmp_dir=claim.get('npm_tmp_dir')) - if path: step('zip', path, prefix) if patterns: @@ -823,8 +855,16 @@ def execute(self, build_plan, zip_stream, query): with install_pip_requirements(query, pip_requirements, tmp_dir) as rd: if rd: if pf: - self._zip_write_with_filter(zs, pf, rd, prefix, - timestamp=0) + self._zip_write_with_filter(zs, pf, rd, prefix, timestamp=0) + else: + # XXX: timestamp=0 - what actually do with it? + zs.write_dirs(rd, prefix=prefix, timestamp=0) + elif cmd == "poetry": + runtime, path, prefix = action[1:] + with install_poetry_dependencies(query, path) as rd: + if rd: + if pf: + self._zip_write_with_filter(zs, pf, rd, prefix, timestamp=0) else: # XXX: timestamp=0 - what actually do with it? zs.write_dirs(rd, prefix=prefix, timestamp=0) @@ -979,6 +1019,121 @@ def install_pip_requirements(query, requirements_file, tmp_dir): yield temp_dir +@contextmanager +def install_poetry_dependencies(query, path): + # TODO: + # 1. Emit files instead of temp_dir + poetry_lock_file = os.path.join(path, "poetry.lock") + pyproject_file = os.path.join(path, "pyproject.toml") + # pyproject.toml is always required by poetry, poetry.lock is optional and can be created from pyproject.toml + if not os.path.exists(pyproject_file): + yield + return + + runtime = query.runtime + artifacts_dir = query.artifacts_dir + docker = query.docker + docker_image_tag_id = None + + if docker: + docker_file = docker.docker_file + docker_image = docker.docker_image + docker_build_root = docker.docker_build_root + + if docker_image: + ok = False + while True: + output = check_output(docker_image_id_command(docker_image)) + if output: + docker_image_tag_id = output.decode().strip() + log.debug( + "DOCKER TAG ID: %s -> %s", docker_image, docker_image_tag_id + ) + ok = True + if ok: + break + docker_cmd = docker_build_command( + build_root=docker_build_root, + docker_file=docker_file, + tag=docker_image, + ) + check_call(docker_cmd) + ok = True + elif docker_file or docker_build_root: + raise ValueError( + "docker_image must be specified " "for a custom image future references" + ) + + working_dir = os.getcwd() + + log.info("Installing python dependencies with poetry: %s", poetry_lock_file) + with tempdir() as temp_dir: + def copy_file_to_target(file, temp_dir): + filename = os.path.basename(file) + target_file = os.path.join(temp_dir, filename) + shutil.copyfile(file, target_file) + return target_file + + poetry_lock_target_file = copy_file_to_target(poetry_lock_file, temp_dir) + pyproject_target_file = copy_file_to_target(pyproject_file, temp_dir) + + poetry_exec = "poetry" + subproc_env = None + + if not docker: + if WINDOWS: + poetry_exec = "poetry.bat" + + # Install dependencies into the temporary directory. + with cd(temp_dir): + # NOTE: poetry must be available, which is the case with lambci/lambda:build-python* docker images + # FIXME: poetry install does not currently allow to specify the target directory so we extract + # installed packages from .venv/lib/python*/site-packages + poetry_commands = [ + shlex_join([ poetry_exec, "config", "--no-interaction", "virtualenvs.create", "true" ]), + shlex_join([ poetry_exec, "config", "--no-interaction", "virtualenvs.in-project", "true" ]), + shlex_join([ poetry_exec, "install", "--no-interaction" ]), + ] + if docker: + with_ssh_agent = docker.with_ssh_agent + poetry_cache_dir = docker.docker_poetry_cache + if poetry_cache_dir: + if isinstance(poetry_cache_dir, str): + poetry_cache_dir = os.path.abspath( + os.path.join(working_dir, poetry_cache_dir) + ) + else: + poetry_cache_dir = os.path.abspath( + os.path.join(working_dir, artifacts_dir, "cache/poetry") + ) + + chown_mask = "{}:{}".format(os.getuid(), os.getgid()) + shell_commands = poetry_commands + [shlex_join(["chown", "-R", chown_mask, "."])] + shell_command = [" && ".join(shell_commands)] + check_call( + docker_run_command( + ".", + shell_command, + runtime, + image=docker_image_tag_id, + shell=True, + ssh_agent=with_ssh_agent, + poetry_cache_dir=poetry_cache_dir, + ) + ) + else: + cmd_log.info(poetry_commands) + log_handler and log_handler.flush() + for poetry_command in poetry_commands: + check_call(poetry_command, env=subproc_env) + + # FIXME: not really needed as only the content of a subdirectory is exposed + os.remove(poetry_lock_target_file) + os.remove(pyproject_target_file) + + yield os.path.join(temp_dir, ".venv/lib/", runtime, "site-packages") + + @contextmanager def install_npm_requirements(query, requirements_file, tmp_dir): # TODO: @@ -1094,7 +1249,7 @@ def docker_build_command(tag=None, docker_file=None, build_root=False): def docker_run_command(build_root, command, runtime, image=None, shell=None, ssh_agent=False, - interactive=False, pip_cache_dir=None): + interactive=False, pip_cache_dir=None, poetry_cache_dir=None): """""" if platform.system() not in ('Linux', 'Darwin'): raise RuntimeError("Unsupported platform for docker building") @@ -1137,6 +1292,11 @@ def docker_run_command(build_root, command, runtime, docker_cmd.extend([ '-v', '{}:/root/.cache/pip:z'.format(pip_cache_dir), ]) + if poetry_cache_dir: + poetry_cache_dir = os.path.abspath(poetry_cache_dir) + docker_cmd.extend([ + '-v', '{}:/root/.cache/pypoetry:z'.format(poetry_cache_dir), + ]) if not image: image = 'public.ecr.aws/sam/build-{}'.format(runtime) diff --git a/test_package_toml.py b/test_package_toml.py new file mode 100644 index 00000000..659b29de --- /dev/null +++ b/test_package_toml.py @@ -0,0 +1,10 @@ +from package import get_build_system_from_pyproject_toml + +def test_get_build_system_from_pyproject_toml_inexistent(): + assert get_build_system_from_pyproject_toml("examples/fixtures/inexistent/pyproject.toml") is None + +def test_get_build_system_from_pyproject_toml_unknown(): + assert get_build_system_from_pyproject_toml("examples/fixtures/unittests/pyproject-unknown.toml") is None + +def test_get_build_system_from_pyproject_toml_poetry(): + assert get_build_system_from_pyproject_toml("examples/fixtures/python3.8-app-poetry/pyproject.toml") == "poetry" diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..2c794d8e --- /dev/null +++ b/tox.ini @@ -0,0 +1,7 @@ +[tox] +envlist = py36,py37,py38,py39,py310 +skipsdist=True + +[testenv] +deps = pytest==6.2.4 +commands = pytest From c5c2c167ab2cc3b94de292216ab81a84b8359fc5 Mon Sep 17 00:00:00 2001 From: Patrick Decat Date: Tue, 17 May 2022 11:01:41 +0200 Subject: [PATCH 2/7] feat: Export poetry dependencies and install them with pip --no-deps --- examples/build-package/main.tf | 2 +- package.py | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/examples/build-package/main.tf b/examples/build-package/main.tf index db349a28..da1a4348 100644 --- a/examples/build-package/main.tf +++ b/examples/build-package/main.tf @@ -45,7 +45,7 @@ module "package_dir_pip_dir" { artifacts_dir = "${path.root}/builds/package_dir_pip_dir/" } -# Create zip-archive of a single directory where "poetry install" will also be executed +# Create zip-archive of a single directory where "poetry export" & "pip install --no-deps" will also be executed module "package_dir_poetry" { source = "../../" diff --git a/package.py b/package.py index 761d662a..439db959 100644 --- a/package.py +++ b/package.py @@ -1066,7 +1066,7 @@ def install_poetry_dependencies(query, path): working_dir = os.getcwd() - log.info("Installing python dependencies with poetry: %s", poetry_lock_file) + log.info("Installing python dependencies with poetry & pip: %s", poetry_lock_file) with tempdir() as temp_dir: def copy_file_to_target(file, temp_dir): filename = os.path.basename(file) @@ -1078,6 +1078,7 @@ def copy_file_to_target(file, temp_dir): pyproject_target_file = copy_file_to_target(pyproject_file, temp_dir) poetry_exec = "poetry" + python_exec = runtime subproc_env = None if not docker: @@ -1087,12 +1088,13 @@ def copy_file_to_target(file, temp_dir): # Install dependencies into the temporary directory. with cd(temp_dir): # NOTE: poetry must be available, which is the case with lambci/lambda:build-python* docker images - # FIXME: poetry install does not currently allow to specify the target directory so we extract - # installed packages from .venv/lib/python*/site-packages + # FIXME: poetry install does not currently allow to specify the target directory so we export the + # requirements then install them with pip --no-deps to avoid using pip dependency resolver poetry_commands = [ shlex_join([ poetry_exec, "config", "--no-interaction", "virtualenvs.create", "true" ]), shlex_join([ poetry_exec, "config", "--no-interaction", "virtualenvs.in-project", "true" ]), - shlex_join([ poetry_exec, "install", "--no-interaction" ]), + shlex_join([ poetry_exec, "export", "-f", "requirements.txt", "--output", "requirements.txt" ]), + shlex_join([ python_exec, '-m', 'pip', 'install', '--no-compile', '--no-deps', '--prefix=', '--target=.','--requirement=requirements.txt']), ] if docker: with_ssh_agent = docker.with_ssh_agent @@ -1127,11 +1129,10 @@ def copy_file_to_target(file, temp_dir): for poetry_command in poetry_commands: check_call(poetry_command, env=subproc_env) - # FIXME: not really needed as only the content of a subdirectory is exposed os.remove(poetry_lock_target_file) os.remove(pyproject_target_file) - yield os.path.join(temp_dir, ".venv/lib/", runtime, "site-packages") + yield temp_dir @contextmanager From 9dfd7ea69b86f8a342e713a7a97c20f888a26c6e Mon Sep 17 00:00:00 2001 From: Patrick Decat Date: Thu, 29 Sep 2022 10:10:35 +0200 Subject: [PATCH 3/7] Use python 3.9 for poetry example everywhere --- examples/build-package/main.tf | 10 +++++----- .../docker/Dockerfile | 0 .../ignore_please.txt | 0 .../index.py | 0 .../poetry.lock | 0 .../pyproject.toml | 2 +- test_package_toml.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) rename examples/fixtures/{python3.8-app-poetry => python3.9-app-poetry}/docker/Dockerfile (100%) rename examples/fixtures/{python3.8-app-poetry => python3.9-app-poetry}/ignore_please.txt (100%) rename examples/fixtures/{python3.8-app-poetry => python3.9-app-poetry}/index.py (100%) rename examples/fixtures/{python3.8-app-poetry => python3.9-app-poetry}/poetry.lock (100%) rename examples/fixtures/{python3.8-app-poetry => python3.9-app-poetry}/pyproject.toml (90%) diff --git a/examples/build-package/main.tf b/examples/build-package/main.tf index da1a4348..9685c232 100644 --- a/examples/build-package/main.tf +++ b/examples/build-package/main.tf @@ -53,12 +53,12 @@ module "package_dir_poetry" { build_in_docker = true runtime = "python3.8" - docker_image = "build-python3.8-poetry" - docker_file = "${path.module}/../fixtures/python3.8-app-poetry/docker/Dockerfile" + docker_image = "build-python3.9-poetry" + docker_file = "${path.module}/../fixtures/python3.9-app-poetry/docker/Dockerfile" source_path = [ { - path = "${path.module}/../fixtures/python3.8-app-poetry" + path = "${path.module}/../fixtures/python3.9-app-poetry" poetry_install = true } ] @@ -316,7 +316,7 @@ module "lambda_layer_poetry" { source_path = [ { - path = "${path.module}/../fixtures/python3.8-app-poetry" + path = "${path.module}/../fixtures/python3.9-app-poetry" poetry_install = true } ] @@ -325,7 +325,7 @@ module "lambda_layer_poetry" { build_in_docker = true runtime = "python3.8" docker_image = "build-python3.8-poetry" - docker_file = "${path.module}/../fixtures/python3.8-app-poetry/docker/Dockerfile" + docker_file = "${path.module}/../fixtures/python3.9-app-poetry/docker/Dockerfile" artifacts_dir = "${path.root}/builds/lambda_layer_poetry/" } diff --git a/examples/fixtures/python3.8-app-poetry/docker/Dockerfile b/examples/fixtures/python3.9-app-poetry/docker/Dockerfile similarity index 100% rename from examples/fixtures/python3.8-app-poetry/docker/Dockerfile rename to examples/fixtures/python3.9-app-poetry/docker/Dockerfile diff --git a/examples/fixtures/python3.8-app-poetry/ignore_please.txt b/examples/fixtures/python3.9-app-poetry/ignore_please.txt similarity index 100% rename from examples/fixtures/python3.8-app-poetry/ignore_please.txt rename to examples/fixtures/python3.9-app-poetry/ignore_please.txt diff --git a/examples/fixtures/python3.8-app-poetry/index.py b/examples/fixtures/python3.9-app-poetry/index.py similarity index 100% rename from examples/fixtures/python3.8-app-poetry/index.py rename to examples/fixtures/python3.9-app-poetry/index.py diff --git a/examples/fixtures/python3.8-app-poetry/poetry.lock b/examples/fixtures/python3.9-app-poetry/poetry.lock similarity index 100% rename from examples/fixtures/python3.8-app-poetry/poetry.lock rename to examples/fixtures/python3.9-app-poetry/poetry.lock diff --git a/examples/fixtures/python3.8-app-poetry/pyproject.toml b/examples/fixtures/python3.9-app-poetry/pyproject.toml similarity index 90% rename from examples/fixtures/python3.8-app-poetry/pyproject.toml rename to examples/fixtures/python3.9-app-poetry/pyproject.toml index fc8ea175..2b41ff3b 100644 --- a/examples/fixtures/python3.8-app-poetry/pyproject.toml +++ b/examples/fixtures/python3.9-app-poetry/pyproject.toml @@ -1,5 +1,5 @@ [tool.poetry] -name = "python3.8-app-poetry" +name = "python3.9-app-poetry" version = "0.1.0" description = "" authors = ["Your Name "] diff --git a/test_package_toml.py b/test_package_toml.py index 659b29de..07cb9b2b 100644 --- a/test_package_toml.py +++ b/test_package_toml.py @@ -7,4 +7,4 @@ def test_get_build_system_from_pyproject_toml_unknown(): assert get_build_system_from_pyproject_toml("examples/fixtures/unittests/pyproject-unknown.toml") is None def test_get_build_system_from_pyproject_toml_poetry(): - assert get_build_system_from_pyproject_toml("examples/fixtures/python3.8-app-poetry/pyproject.toml") == "poetry" + assert get_build_system_from_pyproject_toml("examples/fixtures/python3.9-app-poetry/pyproject.toml") == "poetry" From dc01989802bb2664ee9bd1b38e73668f65519b9a Mon Sep 17 00:00:00 2001 From: Patrick Decat Date: Thu, 29 Sep 2022 11:57:09 +0200 Subject: [PATCH 4/7] Take poetry.toml into account if it exists --- package.py | 81 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 70 insertions(+), 11 deletions(-) diff --git a/package.py b/package.py index 439db959..f98914a5 100644 --- a/package.py +++ b/package.py @@ -692,6 +692,9 @@ def poetry_install_step(path, prefix=None, required=False): poetry_lock_file = os.path.join(path, "poetry.lock") if os.path.isfile(poetry_lock_file): hash(poetry_lock_file) + poetry_toml_file = os.path.join(path, "poetry.toml") + if os.path.isfile(poetry_toml_file): + hash(poetry_toml_file) def npm_requirements_step(path, prefix=None, required=False, tmp_dir=None): requirements = path @@ -1023,13 +1026,17 @@ def install_pip_requirements(query, requirements_file, tmp_dir): def install_poetry_dependencies(query, path): # TODO: # 1. Emit files instead of temp_dir - poetry_lock_file = os.path.join(path, "poetry.lock") + + # pyproject.toml is always required by poetry pyproject_file = os.path.join(path, "pyproject.toml") - # pyproject.toml is always required by poetry, poetry.lock is optional and can be created from pyproject.toml if not os.path.exists(pyproject_file): yield return + # poetry.lock & poetry.toml are optional + poetry_lock_file = os.path.join(path, "poetry.lock") + poetry_toml_file = os.path.join(path, "poetry.toml") + runtime = query.runtime artifacts_dir = query.artifacts_dir docker = query.docker @@ -1061,7 +1068,7 @@ def install_poetry_dependencies(query, path): ok = True elif docker_file or docker_build_root: raise ValueError( - "docker_image must be specified " "for a custom image future references" + "docker_image must be specified for a custom image future references" ) working_dir = os.getcwd() @@ -1074,9 +1081,20 @@ def copy_file_to_target(file, temp_dir): shutil.copyfile(file, target_file) return target_file - poetry_lock_target_file = copy_file_to_target(poetry_lock_file, temp_dir) pyproject_target_file = copy_file_to_target(pyproject_file, temp_dir) + if os.path.isfile(poetry_lock_file): + log.info("Using poetry lock file: %s", poetry_lock_file) + poetry_lock_target_file = copy_file_to_target(poetry_lock_file, temp_dir) + else: + poetry_lock_target_file = None + + if os.path.isfile(poetry_toml_file): + log.info("Using poetry configuration file: %s", poetry_lock_file) + poetry_toml_target_file = copy_file_to_target(poetry_toml_file, temp_dir) + else: + poetry_toml_target_file = None + poetry_exec = "poetry" python_exec = runtime subproc_env = None @@ -1087,14 +1105,52 @@ def copy_file_to_target(file, temp_dir): # Install dependencies into the temporary directory. with cd(temp_dir): - # NOTE: poetry must be available, which is the case with lambci/lambda:build-python* docker images + # NOTE: poetry must be available in the build environment, which is the case with lambci/lambda:build-python* docker images but not public.ecr.aws/sam/build-python* docker images # FIXME: poetry install does not currently allow to specify the target directory so we export the - # requirements then install them with pip --no-deps to avoid using pip dependency resolver + # requirements then install them with "pip --no-deps" to avoid using pip dependency resolver poetry_commands = [ - shlex_join([ poetry_exec, "config", "--no-interaction", "virtualenvs.create", "true" ]), - shlex_join([ poetry_exec, "config", "--no-interaction", "virtualenvs.in-project", "true" ]), - shlex_join([ poetry_exec, "export", "-f", "requirements.txt", "--output", "requirements.txt" ]), - shlex_join([ python_exec, '-m', 'pip', 'install', '--no-compile', '--no-deps', '--prefix=', '--target=.','--requirement=requirements.txt']), + shlex_join( + [ + poetry_exec, + "config", + "--no-interaction", + "virtualenvs.create", + "true", + ] + ), + shlex_join( + [ + poetry_exec, + "config", + "--no-interaction", + "virtualenvs.in-project", + "true", + ] + ), + shlex_join( + [ + poetry_exec, + "export", + "--format", + "requirements.txt", + "--output", + "requirements.txt", + "--with-credentials", + ] + ), + shlex_join( + [ + python_exec, + "-m", + "pip", + "install", + "--no-compile", + "--no-deps", + "--prefix=", + "--target=.", + "--requirement=requirements.txt", + ] + ), ] if docker: with_ssh_agent = docker.with_ssh_agent @@ -1129,8 +1185,11 @@ def copy_file_to_target(file, temp_dir): for poetry_command in poetry_commands: check_call(poetry_command, env=subproc_env) - os.remove(poetry_lock_target_file) os.remove(pyproject_target_file) + if poetry_lock_target_file: + os.remove(poetry_lock_target_file) + if poetry_toml_target_file: + os.remove(poetry_toml_target_file) yield temp_dir From 5183b4ca9bd61b8a0f29bf9a3afedc43ac31ad05 Mon Sep 17 00:00:00 2001 From: Patrick Decat Date: Thu, 29 Sep 2022 14:15:50 +0200 Subject: [PATCH 5/7] Use poetry 1.2.1 in examples --- examples/fixtures/python3.9-app-poetry/docker/Dockerfile | 2 +- examples/fixtures/python3.9-app-poetry/poetry.toml | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 examples/fixtures/python3.9-app-poetry/poetry.toml diff --git a/examples/fixtures/python3.9-app-poetry/docker/Dockerfile b/examples/fixtures/python3.9-app-poetry/docker/Dockerfile index 3c790987..167b9413 100644 --- a/examples/fixtures/python3.9-app-poetry/docker/Dockerfile +++ b/examples/fixtures/python3.9-app-poetry/docker/Dockerfile @@ -3,4 +3,4 @@ FROM public.ecr.aws/sam/build-python3.9 LABEL maintainer="Betajob AS" \ description="Patched AWS Lambda build container" -RUN pip install poetry==1.1.13 +RUN pip install poetry==1.2.1 diff --git a/examples/fixtures/python3.9-app-poetry/poetry.toml b/examples/fixtures/python3.9-app-poetry/poetry.toml new file mode 100644 index 00000000..e69de29b From c85fddf97f637bd4006f612bc649f9e780e5e66f Mon Sep 17 00:00:00 2001 From: Patrick Decat Date: Sat, 22 Oct 2022 19:10:57 +0200 Subject: [PATCH 6/7] Move python unit tests to tests/ folder, add Github Actions to run theses tests, and add instructions to README --- .github/workflows/test.yml | 37 +++++++++++++++++++ .gitignore | 3 ++ README.md | 23 ++++++++++++ test_package_toml.py | 10 ----- .../fixtures}/pyproject-unknown.toml | 0 tests/test_package_toml.py | 23 ++++++++++++ tox.ini | 7 ++-- 7 files changed, 90 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/test.yml delete mode 100644 test_package_toml.py rename {examples/fixtures/unittests => tests/fixtures}/pyproject-unknown.toml (100%) create mode 100644 tests/test_package_toml.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..139774bf --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,37 @@ +name: Tests + +env: + PYTEST_VERSION: 7.1.3 + +on: + push: + branches: [master] + tags: ["*"] + pull_request: + branches: [master] + +jobs: + tests: + name: Test with Python ${{ matrix.python_version }} + runs-on: ubuntu-latest + strategy: + matrix: + python_version: ["3.7", "3.8", "3.9", "3.10"] + fail-fast: false + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python_version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python_version }} + + - name: Install poetry and tox + shell: bash + run: | + pip install pytest==${PYTEST_VERSION} + + - name: Run tox + shell: bash + run: | + python -m pytest -vvv tests/ diff --git a/.gitignore b/.gitignore index 0da622ae..d5763d01 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ terraform.rc # Lambda directories builds/ __pycache__/ + +# Test directories +.tox diff --git a/README.md b/README.md index fcedb1c7..f58e7384 100644 --- a/README.md +++ b/README.md @@ -819,6 +819,29 @@ No modules. | [s3\_object](#output\_s3\_object) | The map with S3 object data of zip archive deployed (if deployment was from S3) | +## Development + +### Python + +During development involving modifying python files, use tox to run unit tests: + +``` +tox +``` + +This will try to run unit tests which each supported python version, reporting errors for python versions which are not installed locally. + +If you only want to test against your main python version: + +``` +tox -e py +``` + +You can also pass additional positional arguments to pytest which is used to run test, e.g. to make it verbose: +``` +tox -e py -- -vvv +``` + ## Authors Module managed by [Anton Babenko](https://github.com/antonbabenko). Check out [serverless.tf](https://serverless.tf) to learn more about doing serverless with Terraform. diff --git a/test_package_toml.py b/test_package_toml.py deleted file mode 100644 index 07cb9b2b..00000000 --- a/test_package_toml.py +++ /dev/null @@ -1,10 +0,0 @@ -from package import get_build_system_from_pyproject_toml - -def test_get_build_system_from_pyproject_toml_inexistent(): - assert get_build_system_from_pyproject_toml("examples/fixtures/inexistent/pyproject.toml") is None - -def test_get_build_system_from_pyproject_toml_unknown(): - assert get_build_system_from_pyproject_toml("examples/fixtures/unittests/pyproject-unknown.toml") is None - -def test_get_build_system_from_pyproject_toml_poetry(): - assert get_build_system_from_pyproject_toml("examples/fixtures/python3.9-app-poetry/pyproject.toml") == "poetry" diff --git a/examples/fixtures/unittests/pyproject-unknown.toml b/tests/fixtures/pyproject-unknown.toml similarity index 100% rename from examples/fixtures/unittests/pyproject-unknown.toml rename to tests/fixtures/pyproject-unknown.toml diff --git a/tests/test_package_toml.py b/tests/test_package_toml.py new file mode 100644 index 00000000..129ac588 --- /dev/null +++ b/tests/test_package_toml.py @@ -0,0 +1,23 @@ +from package import get_build_system_from_pyproject_toml + + +def test_get_build_system_from_pyproject_toml_inexistent(): + assert ( + get_build_system_from_pyproject_toml("fixtures/inexistent/pyproject.toml") + is None + ) + + +def test_get_build_system_from_pyproject_toml_unknown(): + assert ( + get_build_system_from_pyproject_toml("fixtures/pyproject-unknown.toml") is None + ) + + +def test_get_build_system_from_pyproject_toml_poetry(): + assert ( + get_build_system_from_pyproject_toml( + "examples/fixtures/python3.9-app-poetry/pyproject.toml" + ) + == "poetry" + ) diff --git a/tox.ini b/tox.ini index 2c794d8e..f0297d3f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,8 @@ [tox] -envlist = py36,py37,py38,py39,py310 skipsdist=True [testenv] -deps = pytest==6.2.4 -commands = pytest +deps = + pytest==7.1.3 +commands = + python -m pytest {posargs} tests/ From 876ffaabc4bd5d3ff7ba462bd77508bed5f4259a Mon Sep 17 00:00:00 2001 From: Patrick Decat Date: Sat, 22 Oct 2022 19:15:08 +0200 Subject: [PATCH 7/7] Update versions used in poetry example --- examples/build-package/main.tf | 8 ++++---- .../fixtures/python3.9-app-poetry/docker/Dockerfile | 2 +- examples/fixtures/python3.9-app-poetry/poetry.lock | 10 +++++----- examples/fixtures/python3.9-app-poetry/pyproject.toml | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/build-package/main.tf b/examples/build-package/main.tf index 9685c232..c16ef568 100644 --- a/examples/build-package/main.tf +++ b/examples/build-package/main.tf @@ -52,7 +52,7 @@ module "package_dir_poetry" { create_function = false build_in_docker = true - runtime = "python3.8" + runtime = "python3.9" docker_image = "build-python3.9-poetry" docker_file = "${path.module}/../fixtures/python3.9-app-poetry/docker/Dockerfile" @@ -312,7 +312,7 @@ module "lambda_layer_poetry" { create_layer = true layer_name = "${random_pet.this.id}-layer-poetry-dockerfile" - compatible_runtimes = ["python3.8"] + compatible_runtimes = ["python3.9"] source_path = [ { @@ -323,8 +323,8 @@ module "lambda_layer_poetry" { hash_extra = "extra-hash-to-prevent-conflicts-with-module.package_dir" build_in_docker = true - runtime = "python3.8" - docker_image = "build-python3.8-poetry" + runtime = "python3.9" + docker_image = "build-python3.9-poetry" docker_file = "${path.module}/../fixtures/python3.9-app-poetry/docker/Dockerfile" artifacts_dir = "${path.root}/builds/lambda_layer_poetry/" } diff --git a/examples/fixtures/python3.9-app-poetry/docker/Dockerfile b/examples/fixtures/python3.9-app-poetry/docker/Dockerfile index 167b9413..9d19957b 100644 --- a/examples/fixtures/python3.9-app-poetry/docker/Dockerfile +++ b/examples/fixtures/python3.9-app-poetry/docker/Dockerfile @@ -3,4 +3,4 @@ FROM public.ecr.aws/sam/build-python3.9 LABEL maintainer="Betajob AS" \ description="Patched AWS Lambda build container" -RUN pip install poetry==1.2.1 +RUN pip install poetry==1.2.2 diff --git a/examples/fixtures/python3.9-app-poetry/poetry.lock b/examples/fixtures/python3.9-app-poetry/poetry.lock index b7e48193..a8da85ae 100644 --- a/examples/fixtures/python3.9-app-poetry/poetry.lock +++ b/examples/fixtures/python3.9-app-poetry/poetry.lock @@ -1,6 +1,6 @@ [[package]] name = "colorama" -version = "0.4.4" +version = "0.4.5" description = "Cross-platform colored terminal text." category = "main" optional = false @@ -19,13 +19,13 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [metadata] lock-version = "1.1" -python-versions = "^3.6" -content-hash = "65d9f3b221205b259a18285ee8c78c015794fa2e69c66f9ffc836f1758fd594d" +python-versions = "^3.7" +content-hash = "31bbdf3fc3c5e491c372a8ac467cee0ca3dc43d344b42059cab09342e5c715c1" [metadata.files] colorama = [ - {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, - {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, + {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, + {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] colorful = [ {file = "colorful-0.5.4-py2.py3-none-any.whl", hash = "sha256:8d264b52a39aae4c0ba3e2a46afbaec81b0559a99be0d2cfe2aba4cf94531348"}, diff --git a/examples/fixtures/python3.9-app-poetry/pyproject.toml b/examples/fixtures/python3.9-app-poetry/pyproject.toml index 2b41ff3b..c09d58aa 100644 --- a/examples/fixtures/python3.9-app-poetry/pyproject.toml +++ b/examples/fixtures/python3.9-app-poetry/pyproject.toml @@ -5,7 +5,7 @@ description = "" authors = ["Your Name "] [tool.poetry.dependencies] -python = "^3.6" +python = "^3.7" colorful = "^0.5.4" [tool.poetry.dev-dependencies]