Use non-legacy urlopen() instead of urlretrieve() in "message-ix dl"

.travis.yml

@@ -3,8 +3,9 @@ language: minimal
 env:
   global:
   - PYENV=py37
-  # Encrypted values for MESSAGE_IX_CI_USER, MESSAGE_IX_CI_PW
-  - secure: "YlqCqO3t4XZbSjP7FseXGU0PWKxCTD+1kBwgRsA829yphaK0UEgDoCHVn0JvE1VDtb4E58eO0Ulnpc0MWhUDGe9dX7o5lpSufKGD+GLl/X00hYpahu17krxphAnqIyATQnY1RzQaqPGThkcx5SVJal1LAqNy5TzsLkYoCq7p1H0G4QlK9WeItlF/FLhrN27Zbi6/d1XYxgEataSHLfm7fb/uQJPj35r/DRwdyhmoDEOmtvbOL4+OQdLfdB56d1YLs5AIK2mMt/DhAoZvrwqWiway+87vxUuY/YIRVtgeukxxEhlp982YC6IFZdz149TS5107fAJpiTEUX2mdjNuzOcBGh6MTMwGVRsBSBfFPsmt9QF5Smwq6KVf9uxm0Sgwz5ih828Tg18YH5iYJ84Y1jF8l7kJw7N1nFKBYoTVB5un9z0iY3Dnw6LQDzJCvNGmdd4Ct38/vTj73Ltcr5hSzcDeE9VQs99z8gM1FCb45MbSYjg00YgfCzypuG5LibTmay89YcTRMIQ9OliQRCJyOB8l19Z1HYMn33nX8BN6IocWKSkeJexV1iNSPt0GZEnBRAccb7iJd/KD/tUeQvVtFbwJ/qSiQOduUb/GMEvcldXQeqsAQRSMvCdadlU2aQGjQF2XQJGlPhQf744RnevrTPm4HeIpvqLTeVvgJiF3h7Pw="
+  # Encrypted values for MESSAGE_IX_CI_USER, MESSAGE_IX_CI_PW,
+  # MESSAGE_IX_GH_USER, MESSAGE_IX_GH_PW
+  - secure: "IiNG8HMAFIrkeXnVMocSbAtXKgJ7SrZ9VduuAtdftImyBovbs2whtII5rL7qBg47mibREvY0ylUglmc5eLweEbZJO1zOVvNb8urk+cN4aljw9OubAlkpines9R0alhE1M2cOx94JYrNLPJyvMC8s40dVgCJzOM2kyP+cuf8Sr0vCgCAmbtsHSp7f3pkbvR+U7nSrPSHa/LHGtbQ2XJnhd8vvGeAn0oiP2c/Y6yCRhwI+EZYvh0uFSTf4XH6v/vs1zS+2qmWDMe5CUxXgXPtkPQqBhTmQDlbZXcnyoBIWtrpNAP8lXYv6piddizCg+HGpGkmef2pAnFwekQ6tCuQGyyamLlj6nFkljXHn1lbMoFPbRuH4N1vgh8h95r3HUo/Nfxdsza/PEs5HCOA03dU7rE6+ZrV5o0NRGf6cw0lR2evAmPBvkR66soWf3798T5JuUZib3fJ7XlBnhYisKw4Kr+IA6CJTlLVGZ1yd8YcW0bgUt881/kzC5lLjZEN3Is6PL5pkQO9H8tBLKuT0E+X5k6rJlxmuCtQawrpz+0LkoRjXWczH3SY4X7QU1N1Z7SGybNgsJCtwxYinq6QV1vHy2yhJjZtfn3iXtdU7mVyfGtOa3ZVkTO0MDWeDS2FiELz1hMV3zUC5rTZTkV4wLA88Cdijx9fiaW6tirVKhvMSwnI="
 
 os:
 - linux

message_ix/cli.py

@@ -1,7 +1,9 @@
+from base64 import b64encode
 import json
+import os
 from pathlib import Path
 from shutil import copyfile
-from urllib.request import urlretrieve
+from urllib.request import Request, urlopen
 import tempfile
 import zipfile
 
@@ -84,17 +86,39 @@ def copy_model(path, overwrite, set_default):
 def dl(branch, tag, path):
     if tag and branch:
         raise click.BadOptionUsage('Can only provide one of `tag` or `branch`')
-
-    if tag or branch is None:
+    elif branch:
+        # Construct URL and filename from branch
+        zipname = f"{branch}.zip"
+        url = f"https://github.com/iiasa/message_ix/archive/{zipname}"
+    else:
         # Get tag information using GitHub API
-        url = "https://api.github.com/repos/iiasa/message_ix/tags"
-        with open(urlretrieve(url)[0]) as f:
-            tags_info = json.load(f)
+        args = dict(
+            url="https://api.github.com/repos/iiasa/message_ix/tags",
+            headers=dict(),
+        )
+        try:
+            # Only for Travis/macOS: GitHub rate limits unathenticated API
+            # requests by IP address. Because the build worker shares an IP,
+            # the limit is exceeded and the request fails. Use HTTP Basic Auth
+            # with an encrypted username and  password from .travis.yml for a
+            # higher rate limit.
+            auth_bytes = b64encode(
+                "{MESSAGE_IX_GH_USER}:{MESSAGE_IX_GH_PW}"
+                .format(**os.environ)
+                .encode()
+            )
+            args["headers"]["Authorization"] = f"Basic {auth_bytes.decode()}"
+        except KeyError:
+            pass
+
+        with urlopen(Request(**args)) as response:
+            tags_info = json.load(response)
 
         if tag is None:
             tag = tags_info[0]["name"]
             print(f"Default: latest release {tag}")
 
+        # Get the zipball URL for the matching tag
         url = None
         for info in tags_info:
             if info["name"] == tag:
@@ -105,31 +129,28 @@ def dl(branch, tag, path):
             raise ValueError(f"tag {repr(tag)} does not exist")
 
         zipname = f"{tag}.zip"
-    else:
-        # Construct URL and filename from branch
-        zipname = f"{branch}.zip"
-        url = f"https://github.com/iiasa/message_ix/archive/{zipname}"
-
-    path = Path(path)
 
+    # Context manager to remove the TemporaryDirectory when complete
     with tempfile.TemporaryDirectory() as td:
-        print(f"Retrieving {url}")
+        # Path for zip file
         zippath = Path(td) / zipname
-        urlretrieve(url, zippath)
 
-        archive = zipfile.ZipFile(zippath)
+        print(f"Retrieving {url}")
+        with urlopen(url) as response:  # Unauthenticated request
+            zippath.write_bytes(response.read())
 
+        # Context manager to close the ZipFile when done, so it can be removed
         print(f"Unzipping {zippath} to {path}")
-        path.mkdir(parents=True, exist_ok=True)
-
-        # Extract only tutorial files
-        archive.extractall(
-            path,
-            members=filter(lambda n: "/tutorial/" in n, archive.namelist())
-        )
-
-        # Close *zipfile* so it can be deleted with *td*
-        archive.close()
+        with zipfile.ZipFile(zippath) as archive:
+            # Zip archive successfully opened; create the output path
+            path = Path(path)
+            path.mkdir(parents=True, exist_ok=True)
+
+            # Extract only tutorial files
+            archive.extractall(
+                path,
+                members=filter(lambda n: "/tutorial/" in n, archive.namelist())
+            )
 
 
 # Add subcommands

message_ix/tests/test_cli.py

@@ -42,6 +42,7 @@ def test_dl(message_ix_cli, opts, tmp_path):
         # Debugging information
         print(r.exception, r.output)
         assert False
+    print(r.exception, r.output)
 
     if opts == "":
         assert "Default: latest release v2.0.0" in r.output