Update to python 3.10 (#684)

* Upgrade black to remove need for typed-ast

* Simplify workflow and install wheel

* Upgrade to python 3.10

* Update documentation

.github/actions/setup/action.yml

@@ -67,6 +67,10 @@ runs:
                 sudo chown root:root shellcheck
                 sudo mv shellcheck "$install_dir"
             fi
+        - name: Setup Python
+          uses: actions/setup-python@v4
+          with:
+            python-version: '3.10'
         - name: Cache Python environment
           uses: actions/cache@v3
           id: cache-python
@@ -80,5 +84,6 @@ runs:
           run: |
             if [[ "$CACHE_HIT" != 'true' ]]; then
                 python -m pip install --upgrade pip
+                pip install wheel
                 pip install -r requirements.txt
             fi

.github/workflows/pr.yml

@@ -8,10 +8,6 @@ jobs:
         steps:
             - name: Checkout the Git repository
               uses: actions/checkout@v3
-            - name: Setup Python
-              uses: actions/setup-python@v4
-              with:
-                python-version: '3.8'
             - name: Setup Toltec dependencies
               uses: ./.github/actions/setup
             - name: Check formatting
@@ -25,10 +21,6 @@ jobs:
         steps:
             - name: Checkout the Git repository
               uses: actions/checkout@v3
-            - name: Setup Python
-              uses: actions/setup-python@v4
-              with:
-                python-version: '3.8'
             - name: Setup Toltec dependencies
               uses: ./.github/actions/setup
             - name: Build packages

.github/workflows/stable.yml

@@ -10,10 +10,6 @@ jobs:
         steps:
             - name: Checkout the Git repository
               uses: actions/checkout@v3
-            - name: Setup Python
-              uses: actions/setup-python@v4
-              with:
-                python-version: '3.8'
             - name: Setup Toltec dependencies
               uses: ./.github/actions/setup
             - name: Build packages

.github/workflows/testing.yml

@@ -10,10 +10,6 @@ jobs:
         steps:
             - name: Checkout the Git repository
               uses: actions/checkout@v3
-            - name: Setup Python
-              uses: actions/setup-python@v4
-              with:
-                python-version: '3.8'
             - name: Setup Toltec dependencies
               uses: ./.github/actions/setup
             - name: Build packages

docs/building.md

@@ -18,7 +18,7 @@ Before running the build, make sure you have all the required dependencies:
 
 * Docker
 * bsdtar
-* Python ⩾ 3.8
+* Python 3.10
 
 You’ll also need all the Python modules listed in [requirements.txt](../requirements.txt) (install them by running `pip install --user -r requirements.txt` or using a [virtual environment](https://docs.python.org/3/tutorial/venv.html)).
 

requirements.txt

@@ -1,9 +1,9 @@
 appdirs==1.4.4
-astroid==2.4.2
-black==20.8b1
+astroid==2.15.4
+black==23.3.0
 certifi==2020.12.5
 chardet==4.0.0
-click==7.1.2
+click==8.1.3
 docker==4.4.1
 idna==2.10
 isort==5.7.0
@@ -13,16 +13,15 @@ MarkupSafe==1.1.1
 mccabe==0.6.1
 mypy==0.790
 mypy-extensions==0.4.3
-pathspec==0.8.1
+pathspec==0.11.1
 pyelftools==0.27
-pylint==2.6.0
+pylint==2.17.4
 python-dateutil==2.8.1
 regex==2020.11.13
 requests==2.25.1
 six==1.15.0
 toml==0.10.2
-typed-ast==1.4.2
-typing-extensions==3.7.4.3
+typing-extensions==4.5.0
 urllib3==1.26.2
 websocket-client==0.57.0
 wrapt==1.12.1

scripts/toltec/bash.py

@@ -334,6 +334,7 @@ def run_script(variables: Variables, script: str) -> LogGenerator:
     :returns: generator yielding output lines from the script
     :raises ScriptError: if the script exits with a non-zero code
     """
+    # pylint: disable-next=consider-using-with
     process = subprocess.Popen(
         ["/usr/bin/env", "bash"],
         stdin=subprocess.PIPE,

scripts/toltec/builder.py

@@ -96,6 +96,7 @@ def __init__(self, work_dir: str, repo_dir: str) -> None:
         self.context: Dict[str, str] = {}
         self.adapter = BuildContextAdapter(logger, self.context)
 
+        # pylint: disable-next=unspecified-encoding
         with open(install_lib_path, "r") as file:
             for line in file:
                 if not line.strip().startswith("#"):
@@ -201,7 +202,7 @@ def _fetch_sources(
                 )
             else:
                 # Fetch source file from the network
-                req = requests.get(source.url)
+                req = requests.get(source.url, timeout=5)
 
                 if req.status_code != 200:
                     raise BuildError(
@@ -215,7 +216,7 @@ def _fetch_sources(
 
             # Verify checksum
             file_sha = util.file_sha256(local_path)
-            if source.checksum != "SKIP" and file_sha != source.checksum:
+            if source.checksum not in ("SKIP", source.checksum):
                 raise BuildError(
                     f"Invalid checksum for source file {source.url}:\n"
                     f"  expected {source.checksum}\n"
@@ -381,6 +382,7 @@ def _postprocessing(self, recipe: Recipe, src_dir: str) -> None:
         script = []
         mount_src = "/src"
 
+        # pylint: disable-next=unnecessary-lambda-assignment
         docker_file_path = lambda file_path: shlex.quote(
             os.path.join(mount_src, os.path.relpath(file_path, src_dir))
         )

scripts/toltec/graphlib.py

@@ -225,6 +225,7 @@ def _find_cycle(self) -> Optional[List[_T]]:
         seen = set()
         node2stacki: MutableMapping[_T, int] = {}
 
+        # pylint: disable-next=consider-using-dict-items
         for node in n2i:
             if node in seen:
                 continue

scripts/toltec/ipk.py

@@ -22,6 +22,7 @@ def _targz_open(fileobj: IO[bytes], epoch: int) -> tarfile.TarFile:
     )
 
     try:
+        # pylint: disable-next=consider-using-with
         archive = tarfile.TarFile(
             mode="w",
             fileobj=gzipobj,  # type:ignore

scripts/toltec/recipe.py

@@ -41,6 +41,7 @@ def from_file(path: str) -> "GenericRecipe":
         :returns: loaded recipe
         """
         name = os.path.basename(path)
+        # pylint: disable-next=unspecified-encoding
         with open(os.path.join(path, "package"), "r") as recipe:
             return GenericRecipe(name, path, recipe.read())
 

scripts/toltec/repo.py

@@ -24,6 +24,8 @@
 class PackageStatus(Enum):
     """Possible existence statuses of a built package."""
 
+    # pylint: disable=invalid-name
+
     # The package already existed in the local filesystem before the build
     AlreadyExists = auto()
 
@@ -33,6 +35,8 @@ class PackageStatus(Enum):
     # The package is missing both from the local filesystem and the remote repo
     Missing = auto()
 
+    # pylint: enable=invalid-name
+
 
 GroupedPackages = Dict[PackageStatus, Dict[str, Dict[str, List[Package]]]]
 
@@ -132,7 +136,7 @@ def fetch_package(
 
         remote_path = os.path.join(remote, filename)
 
-        req = requests.get(remote_path)
+        req = requests.get(remote_path, timeout=5)
 
         if req.status_code != 200:
             return PackageStatus.Missing
@@ -208,6 +212,7 @@ def make_index(self) -> None:
             index_path = os.path.join(arch_dir, "Packages")
             index_gzip_path = os.path.join(arch_dir, "Packages.gz")
 
+            # pylint: disable-next=unspecified-encoding
             with open(index_path, "w") as index_file:
                 with gzip.open(index_gzip_path, "wt") as index_gzip_file:
                     for generic_recipe in self.generic_recipes.values():
@@ -258,5 +263,6 @@ def make_listing(self) -> None:
         listing_path = os.path.join(self.repo_dir, "index.html")
         template = templating.env.get_template("listing.html")
 
+        # pylint: disable-next=unspecified-encoding
         with open(listing_path, "w") as listing_file:
             listing_file.write(template.render(sections=sections))

scripts/toltec/version.py

@@ -16,12 +16,16 @@
 class VersionComparator(Enum):
     """Operators used to compare two version numbers."""
 
+    # pylint: disable=invalid-name
+
     LowerThan = "<<"
     LowerThanOrEqual = "<="
     Equal = "="
     GreaterThanOrEqual = ">="
     GreaterThan = ">>"
 
+    # pylint: enable=invalid-name
+
 
 class InvalidVersionError(Exception):
     """Raised when parsing of an invalid version is attempted."""
@@ -100,13 +104,17 @@ def __repr__(self) -> str:
 class DependencyKind(Enum):
     """Kinds of dependencies that may be requested by a package."""
 
+    # pylint: disable=invalid-name
+
     # Dependency installed in the system used to build a package
     # (e.g., a Debian package)
     Build = "build"
     # Dependency installed alongside a package
     # (e.g., another Entware or Toltec package)
     Host = "host"
 
+    # pylint: enable=invalid-name
+
 
 class InvalidDependencyError(Exception):
     """Raised when parsing an invalid dependency specification."""