Inconsistent Indentation in Lock File

[Kurt-von-Laven]

• I am on the latest Poetry version.

• I have searched the issues of this repo and believe that this is not a duplicate.

• If an exception occurs when executing a command, I executed it again in debug mode (-vvv option).

• OS version and name: Ubuntu 20.04 LTS

• Poetry version: 1.1.14

• Link to the contents of your pyproject.toml file: pyproject.toml

Issue

poetry lock --no-update produces inconsistent output on the same pyproject.toml. When run locally on Ubuntu 22.04 LTS, poetry lock --no-update continues to consistently produce this poetry.lock. This is consistent with past runs on Ubuntu 22.04 LTS in CI and hence what I perceive to be the correct output. Here is a CI workflow on Ubuntu 20.04 LTS showing the diff (reproduced below) between the old (correct) and new (incorrect) output. The new output appears consistent across CI runs. Note that the new order of lines within [package.extras] is reverse alphabetical order, and the new order of dependencies within each line is reversed with respect to the original order. Both of those sorting issues were the focus of the closely related issue #6153 and have already been addressed (but not yet released) in #6169. The focus of this issue is the two spaces added before metadata.python-versions.

diff --git a/poetry.lock b/poetry.lock
index def2008..4e50b11 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -7,7 +7,7 @@ optional = false
 python-versions = "*"
 
 [package.extras]
-test = ["coverage", "flake8", "pexpect", "wheel"]
+test = ["wheel", "pexpect", "flake8", "coverage"]
 
 [[package]]
 name = "astroid"
@@ -31,10 +31,10 @@ optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 
 [package.extras]
-dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"]
-docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"]
-tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"]
-tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"]
+tests_no_zope = ["cloudpickle", "pytest-mypy-plugins", "mypy", "six", "pytest (>=4.3.0)", "pympler", "hypothesis", "coverage[toml] (>=5.0.2)"]
+tests = ["cloudpickle", "zope.interface", "pytest-mypy-plugins", "mypy", "six", "pytest (>=4.3.0)", "pympler", "hypothesis", "coverage[toml] (>=5.0.2)"]
+docs = ["sphinx-notfound-page", "zope.interface", "sphinx", "furo"]
+dev = ["cloudpickle", "pre-commit", "sphinx-notfound-page", "sphinx", "furo", "zope.interface", "pytest-mypy-plugins", "mypy", "six", "pytest (>=4.3.0)", "pympler", "hypothesis", "coverage[toml] (>=5.0.2)"]
 
 [[package]]
 name = "bandit"
@@ -51,9 +51,9 @@ PyYAML = ">=5.3.1"
 stevedore = ">=1.20.0"
 
 [package.extras]
-test = ["coverage (>=4.5.4)", "fixtures (>=3.0.0)", "flake8 (>=4.0.0)", "stestr (>=2.5.0)", "testscenarios (>=0.5.0)", "testtools (>=2.3.0)", "toml", "beautifulsoup4 (>=4.8.0)", "pylint (==1.9.4)"]
-toml = ["toml"]
 yaml = ["pyyaml"]
+toml = ["toml"]
+test = ["pylint (==1.9.4)", "beautifulsoup4 (>=4.8.0)", "toml", "testtools (>=2.3.0)", "testscenarios (>=0.5.0)", "stestr (>=2.5.0)", "flake8 (>=4.0.0)", "fixtures (>=3.0.0)", "coverage (>=4.5.4)"]
 
 [[package]]
 name = "black"
@@ -72,10 +72,10 @@ tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""}
 [package.extras]
-dmypy = ["psutil (>=4.0)"]
-python2 = ["typed-ast (>=1.4.0,<2)"]
 reports = ["lxml"]
+python2 = ["typed-ast (>=1.4.0,<2)"]
+dmypy = ["psutil (>=4.0)"]
 
 [[package]]
 name = "mypy-extensions"
@@ -386,8 +386,8 @@ optional = false
 python-versions = ">=3.7"
 
 [package.extras]
-docs = ["furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)", "sphinx (>=4)"]
-test = ["appdirs (==1.4.4)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)", "pytest (>=6)"]
+test = ["pytest (>=6)", "pytest-mock (>=3.6)", "pytest-cov (>=2.7)", "appdirs (==1.4.4)"]
+docs = ["sphinx (>=4)", "sphinx-autodoc-typehints (>=1.12)", "proselint (>=0.10.2)", "furo (>=2021.7.5b38)"]
 
 [[package]]
 name = "pre-commit"
@@ -466,8 +466,8 @@ tomlkit = ">=0.10.1"
 typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""}
 
 [package.extras]
-spelling = ["pyenchant (>=3.2,<4.0)"]
 testutils = ["gitpython (>3)"]
+spelling = ["pyenchant (>=3.2,<4.0)"]
 
 [[package]]
 name = "pyparsing"
@@ -478,7 +478,7 @@ optional = false
 python-versions = ">=3.6.8"
 
 [package.extras]
-diagrams = ["railroad-diagrams", "jinja2"]
+diagrams = ["jinja2", "railroad-diagrams"]
 
 [[package]]
 name = "pyyaml"
@@ -500,7 +500,7 @@ python-versions = ">=3.6,<4.0"
 prompt_toolkit = ">=2.0,<4.0"
 
 [package.extras]
-docs = ["Sphinx (>=3.3,<4.0)", "sphinx-rtd-theme (>=0.5.0,<0.6.0)", "sphinx-autobuild (>=2020.9.1,<2021.0.0)", "sphinx-copybutton (>=0.3.1,<0.4.0)", "sphinx-autodoc-typehints (>=1.11.1,<2.0.0)"]
+docs = ["sphinx-autodoc-typehints (>=1.11.1,<2.0.0)", "sphinx-copybutton (>=0.3.1,<0.4.0)", "sphinx-autobuild (>=2020.9.1,<2021.0.0)", "sphinx-rtd-theme (>=0.5.0,<0.6.0)", "Sphinx (>=3.3,<4.0)"]
 
 [[package]]
 name = "six"
@@ -592,8 +592,8 @@ platformdirs = ">=2,<3"
 six = ">=1.9.0,<2"
 
 [package.extras]
-docs = ["proselint (>=0.10.2)", "sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=21.3)"]
-testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)", "packaging (>=20.0)"]
+testing = ["packaging (>=20.0)", "pytest-timeout (>=1)", "pytest-randomly (>=1)", "pytest-mock (>=2)", "pytest-freezegun (>=0.4.1)", "pytest-env (>=0.6.2)", "pytest (>=4)", "flaky (>=3)", "coverage-enable-subprocess (>=1)", "coverage (>=4)"]
+docs = ["towncrier (>=21.3)", "sphinx-rtd-theme (>=0.4.3)", "sphinx-argparse (>=0.2.5)", "sphinx (>=3)", "proselint (>=0.10.2)"]
 
 [[package]]
 name = "wcwidth"
@@ -613,7 +613,7 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7"
 
 [metadata]
 lock-version = "1.1"
-python-versions = "^3.9.13"
+  python-versions = "^3.9.13"
 content-hash = "1e949d67cf02f40a9e634c3f4af3bab8afae8ed429f7ef7e05be51afc0e202e6"
 
 [metadata.files]

[dimbleby]

you probably need to take this one up with tomlkit if you can reproduce it (which I can't)

[mkniewallner]

I am able to reproduce on master with the following minimal pyproject.toml:

[tool.poetry]
name = "foo"
version = "0.0.1"
description = ""
authors = []

[tool.poetry.dependencies]
  python = "^3.9"

Which generates the following poetry.lock:

package = []

[metadata]
lock-version = "1.1"
  python-versions = "^3.9"
content-hash = "c595a0588c25d58f3e3834ad7169126836d262b925fe6ca9b5d540dcf301d254"

[metadata.files]

The issue lies under the fact than here in poetry-core, when we load the config from pyproject.toml, we get a mix of attributes that are still tomlkit instances (tomlkit.items.String for instance) and attributes that have been converted to bare Python objects (str for instance).

python_versions being one of those attributes, here in poetry, when we dump the lock file, we end up retrieving a tomlkit.items.String instance, with the same indentation that is in pyproject.toml for python attribute.

I wonder if we should not make sure to convert the attributes in poetry-core to bare Python objects, because we get a mix of tomlkit attributes and Python bare objects anyway, depending on how we construct the config items. The other commands that do need to preserve the indentation when writing back to pyproject.toml don't seem to be based on local_config, since they read the file from scratch again.

[dimbleby]

nice find.

In the unlikely event of finding (i) time and (ii) consensus, I'd happily abandon tomlkit. (With the possible of exception of the poetry add workflow, where it is desirable to be able to update pyproject.toml and preserve comments). It seems to bring quite a lot of trouble...

Certainly for poetry.lock, which is intended only to be machine generated and machine readable: if starting from scratch I'd probably go tomli (coming soon to the standard library) and then use pydantic or similar to parse it all into a nicely typed python object.

However I don't imagine that either (i) or (ii) is going to happen.

[mkniewallner]

nice find.

In the unlikely event of finding (i) time and (ii) consensus, I'd happily abandon tomlkit. (With the possible of exception of the poetry add workflow, where it is desirable to be able to update pyproject.toml and preserve comments). It seems to bring quite a lot of trouble...

Certainly for poetry.lock, which is intended only to be machine generated and machine readable: if starting from scratch I'd probably go tomli (coming soon to the standard library) and then use pydantic or similar to parse it all into a nicely typed python object.

However I don't imagine that either (i) or (ii) is going to happen.

If we confirm that local_config that is populated by poetry-core is indeed never used in situations where we need to write back to pyproject.toml, I think it could be reasonable to drop tomlkit when loading the config, as you suggest, since using tomlkit should only be useful when the indentation matters.

Obviously, since this is an important change and it requires a consensus, this would not come before 1.2 is released. In the meantime, python-poetry/poetry-core#445 should resolve this specific issue, though it also requires to confirm that this doesn't have unexpected side effects (so it's WIP for now, also because some tests would need to be added).

[Kurt-von-Laven]

Thank you for jumping on this issue so quickly. To summarize the workaround for the benefit of those following along, remove all indentation from pyproject.toml (or at least all indentation from the lines appearing with excess indentation in your lock file) if you are experiencing this issue. I am confused as to why this issue arises inconsistently though. At face value, the described root cause seems like it should consistently produce excess indentation in the lock file, which is not the observed behavior. python-poetry/tomlkit#106 may be related and was first fixed in tomlkit 0.7.1, but Poetry 1.1.14 is already on tomlkit 0.7.2.

[mkniewallner]

I'm not sure why it doesn't consistently produce the extra indentation, but the tomlkit issue you linked is unrelated. The issue you reported here is not a bug in tomlkit, this is clearly Poetry misbehaving.

[Secrus]

Closing, since the bug was caused by the use of tomlkit in poetry-core, which was dropped in favor of tomli

[dimbleby]

I don't think that's correct reasoning, tomlkit is still used to write the poetry.lock file.

(I haven't checked whether this bug is still present or not)

[Kurt-von-Laven]

@dimbleby, I just verified that the bug is fixed in Poetry 1.5.0 and not Poetry 1.4.2 as expected.

[github-actions]

This issue has been automatically locked since there has not been any recent activity after it was closed. Please open a new issue for related bugs.