Move Black into a plugin to enable turning it on and off via `backend…

…_plugins` (#8683)

### Problem

We need some mechanism to turn off autoformatters and linters in V2 other than the current `--fmt-{isort,scalafmt,etc}-skip`, which doesn't work because that uses a task-specific option system.

Ideally, the option will be tool-specific, so you can say something like:

```ini
[black]
enable: True

[mypy]
enable: False
```

However, the solution should also be hygienic. If, for example, someone doesn't want to use Black, then we don't want Black's rules to pollute the rule graph. Nor do we want to make rule authors provide a no-op implementation.


### Solution
We decided in #8346 to turn each linter into its own plugin, meaning it may be enabled via the `backend_packages` option like this:

```ini
[GLOBAL]
backend_packages: +[
    "pants.backend.python.lint.black",
    "pants.backend.python.lint.isort",
    "pants.backend.scala.lint.scalafmt",
  ]
```

When specified, that plugin will be enabled, meaning that its options may be configured and its rules will be automatically registered against `fmt` and `lint`.

### Result

We have a scalable and explicit mechanism for users to enable formatters and linters in a way that also makes it effortless for rule authors to have a way to opt-out of their tool.

This only implements Black. In a followup, we will implement isort under the plugin `pants.backend.python.lint.isort`.

For now, V1 tasks will keep the same mechanism as before. We could move them towards this new plugin system, but save that for a followup if we choose to make the deprecations necessary to move V1 users towards the new system.

pants.ini

@@ -46,10 +46,6 @@ pythonpath: [
 
 backend_packages: +[
     "pants.backend.docgen",
-    "internal_backend.rules_for_testing",
-    "internal_backend.repositories",
-    "internal_backend.sitegen",
-    "internal_backend.utilities",
     "pants.contrib.avro",
     "pants.contrib.awslambda.python",
     "pants.contrib.buildrefactor",
@@ -66,6 +62,10 @@ backend_packages: +[
     "pants.contrib.python.checks",
     "pants.contrib.scrooge",
     "pants.contrib.thrifty",
+    "internal_backend.rules_for_testing",
+    "internal_backend.repositories",
+    "internal_backend.sitegen",
+    "internal_backend.utilities",
   ]
 
 # The pants script in this repo consumes these files to run pants
@@ -82,8 +82,6 @@ pants_ignore: +[
     '/build-support/bin/native/src',
   ]
 
-[black]
-config: pyproject.toml
 
 [cache]
 # Caching is on globally by default, but we disable it here for development purposes.

src/python/pants/backend/python/lint/black/BUILD

@@ -0,0 +1,46 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_library(
+  dependencies=[
+    '3rdparty/python:dataclasses',
+    'src/python/pants/backend/python/subsystems',
+    'src/python/pants/backend/python/rules',
+    'src/python/pants/backend/python/targets',
+    'src/python/pants/engine:fs',
+    'src/python/pants/engine:isolated_process',
+    'src/python/pants/engine:rules',
+    'src/python/pants/engine:selectors',
+    'src/python/pants/option',
+    'src/python/pants/rules/core',
+  ],
+)
+
+python_tests(
+  name='tests',
+  sources=globs('*_test.py', exclude=[globs('*_integration_test.py')]),
+  dependencies=[
+    ':black',
+  ],
+)
+
+
+python_tests(
+  name='integration',
+  sources=globs('*_integration_test.py'),
+  dependencies=[
+    ':black',
+    'src/python/pants/backend/python/subsystems',
+    'src/python/pants/backend/python/targets',
+    'src/python/pants/engine:fs',
+    'src/python/pants/engine:rules',
+    'src/python/pants/engine:selectors',
+    'src/python/pants/engine/legacy:structs',
+    'src/python/pants/rules/core',
+    'src/python/pants/source',
+    'src/python/pants/testutil:test_base',
+    'src/python/pants/testutil/subsystem',
+  ],
+  tags = {'integration'},
+  timeout = 90,
+)

src/python/pants/backend/python/lint/black/register.py

@@ -0,0 +1,13 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+
+from pants.backend.python.lint.black import rules as black_rules
+from pants.backend.python.targets import formattable_python_target
+
+
+def rules():
+  return (
+    *black_rules.rules(),
+    *formattable_python_target.rules(),
+  )

src/python/pants/backend/python/rules/black.py → src/python/pants/backend/python/lint/black/rules.py

@@ -6,13 +6,13 @@
 from pathlib import Path
 from typing import Optional, Tuple
 
+from pants.backend.python.lint.black.subsystem import Black
 from pants.backend.python.rules.pex import (
   CreatePex,
   Pex,
   PexInterpreterConstraints,
   PexRequirements,
 )
-from pants.backend.python.subsystems.black import Black
 from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.subsystems.subprocess_environment import SubprocessEncodingEnvironment
 from pants.backend.python.targets.formattable_python_target import FormattablePythonTarget

src/python/pants/backend/python/rules/black_integration_test.py → src/python/pants/backend/python/lint/black/rules_integration_test.py

@@ -3,10 +3,10 @@
 
 from typing import List, Optional, Tuple
 
-from pants.backend.python.rules.black import BlackSetup, fmt, lint, setup_black
+from pants.backend.python.lint.black.rules import BlackSetup, fmt, lint, setup_black
+from pants.backend.python.lint.black.subsystem import Black
 from pants.backend.python.rules.download_pex_bin import download_pex_bin
 from pants.backend.python.rules.pex import CreatePex, create_pex
-from pants.backend.python.subsystems.black import Black
 from pants.backend.python.subsystems.python_native_code import (
   PythonNativeCode,
   create_pex_native_build_environment,

src/python/pants/backend/python/register.py

@@ -6,15 +6,13 @@
 from pants.backend.python.python_requirement import PythonRequirement
 from pants.backend.python.python_requirements import PythonRequirements
 from pants.backend.python.rules import (
-  black,
   download_pex_bin,
   inject_init,
   pex,
   python_create_binary,
   python_test_runner,
 )
 from pants.backend.python.subsystems import python_native_code, subprocess_environment
-from pants.backend.python.targets import formattable_python_target
 from pants.backend.python.targets.python_app import PythonApp
 from pants.backend.python.targets.python_binary import PythonBinary
 from pants.backend.python.targets.python_distribution import PythonDistribution
@@ -94,9 +92,7 @@ def register_goals():
 
 def rules():
   return (
-    *black.rules(),
     *download_pex_bin.rules(),
-    *formattable_python_target.rules(),
     *inject_init.rules(),
     *pex.rules(),
     *python_test_runner.rules(),

src/python/pants/backend/python/rules/BUILD

@@ -43,20 +43,9 @@ python_tests(
   name='integration',
   sources=globs('*_integration_test.py'),
   dependencies=[
-    ':rules',
-    'src/python/pants/backend/python/subsystems',
-    'src/python/pants/backend/python/targets',
-    'src/python/pants/engine:fs',
-    'src/python/pants/engine:rules',
-    'src/python/pants/engine:selectors',
-    'src/python/pants/engine/legacy:structs',
-    'src/python/pants/rules/core',
-    'src/python/pants/source',
-    'src/python/pants/testutil:test_base',
-    'src/python/pants/testutil/subsystem',
     'src/python/pants/testutil:int-test',
     'testprojects/tests/python/pants:dummies_directory',
   ],
   tags = {'integration'},
-  timeout = 160,
+  timeout = 90,
 )