From 93156b20804befeedc742435db60826c4cce78da Mon Sep 17 00:00:00 2001 From: Evan Rusackas Date: Thu, 14 Dec 2023 09:48:46 -0700 Subject: [PATCH 1/4] feat(releasing): adding a SHA512 and --- RELEASING/README.md | 14 ++++++ RELEASING/verify_release.py | 91 +++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 RELEASING/verify_release.py diff --git a/RELEASING/README.md b/RELEASING/README.md index b007a891700b9..0f2668ddf1e6a 100644 --- a/RELEASING/README.md +++ b/RELEASING/README.md @@ -388,8 +388,22 @@ The script will generate the email text that should be sent to dev@superset.apac ## Validating a release +Official instructions: https://www.apache.org/info/verification.html +We now have a handy script for anyone validating a release to use. It's in this very folder, `verify_release.py` + +Just make sure you have all three release files in the directory (`{some version}.tar.gz`, `{some version}.tar.gz.asc` and `{some version}tar.gz.sha512`). Then run the script with the path to the `.tar.gz` file like so: + +`python verify_release.py ~/path/tp/apache-superset-{version/candidate}-source.tar.gz` + +If all goes well, you will see this result in your terminal: + +```bash +SHA-512 verified +RSA key verified +``` + ## Publishing a successful release Upon a successful vote, you'll have to copy the folder into the non-"dev/" folder. diff --git a/RELEASING/verify_release.py b/RELEASING/verify_release.py new file mode 100644 index 0000000000000..c49f8230baad6 --- /dev/null +++ b/RELEASING/verify_release.py @@ -0,0 +1,91 @@ +import subprocess +import sys +import re +import requests + +# Part 1: Verify SHA512 hash - this is the same as running `shasum -a 512 {release}` and comparing it against `{release}.sha512` + +def get_sha512_hash(filename): + """Run the shasum command on the file and return the SHA512 hash.""" + result = subprocess.run(['shasum', '-a', '512', filename], stdout=subprocess.PIPE) + sha512_hash = result.stdout.decode().split()[0] + return sha512_hash + +def read_sha512_file(filename): + """Read the corresponding .sha512 file and process its contents.""" + sha_filename = filename + '.sha512' + with open(sha_filename, 'r') as file: + lines = file.readlines() + processed_sha = ''.join(lines[1:]).replace(' ', '').replace('\n', '').lower() + return processed_sha + +def verify_sha512(filename): + """Verify if the SHA512 hash of the file matches with the hash in the .sha512 file.""" + sha512_hash = get_sha512_hash(filename) + sha512_file_content = read_sha512_file(filename) + + if sha512_hash == sha512_file_content: + return "SHA verified" + else: + return "SHA failed" + +# Part 2: Verify RSA key - this is the same as running `gpg --verify {release}.asc {release}` and comparing the RSA key and email address against the KEYS file + +def get_gpg_info(filename): + """Run the GPG verify command and extract RSA key and email address.""" + asc_filename = filename + '.asc' + result = subprocess.run(['gpg', '--verify', asc_filename, filename], stderr=subprocess.PIPE, stdout=subprocess.PIPE) + output = result.stderr.decode() + + rsa_key = re.search(r'RSA key ([0-9A-F]+)', output) + email = re.search(r'issuer "([^"]+)"', output) + + rsa_key_result = rsa_key.group(1) if rsa_key else None + email_result = email.group(1) if email else None + + # Debugging: print warnings if rsa_key or email is not found + if rsa_key_result is None: + print("Warning: No RSA key found in GPG verification output.") + if email_result is None: + print("Warning: No email address found in GPG verification output.") + + return rsa_key_result, email_result + + +def verify_rsa_key(rsa_key, email): + """Fetch the KEYS file and verify if the RSA key and email match.""" + url = 'https://downloads.apache.org/superset/KEYS' + response = requests.get(url) + if response.status_code == 200: + if rsa_key not in response.text: + return "RSA key not found on KEYS page" + + # Check if email is None or not in response.text + if email and email in response.text: + return "RSA key and email verified against Apache KEYS file" + elif email: + return "RSA key verified, but Email not found on KEYS page" + else: + return "RSA key verified, but Email not available for verification" + else: + return "Failed to fetch KEYS file" + + +def verify_sha512_and_rsa(filename): + """Verify SHA512 hash and RSA key.""" + sha_result = verify_sha512(filename) + print(sha_result) + + rsa_key, email = get_gpg_info(filename) + if rsa_key: + rsa_result = verify_rsa_key(rsa_key, email) + print(rsa_result) + else: + print("GPG verification failed: RSA key or email not found") + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: python script.py ") + else: + filename = sys.argv[1] + verify_sha512_and_rsa(filename) From bf2f6c9b6e23fdcab7a8fadb04f4b67a9b475122 Mon Sep 17 00:00:00 2001 From: Evan Rusackas Date: Thu, 14 Dec 2023 11:48:14 -0700 Subject: [PATCH 2/4] linting --- RELEASING/verify_release.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/RELEASING/verify_release.py b/RELEASING/verify_release.py index c49f8230baad6..2be55cf54467a 100644 --- a/RELEASING/verify_release.py +++ b/RELEASING/verify_release.py @@ -1,24 +1,28 @@ +import re import subprocess import sys -import re + import requests # Part 1: Verify SHA512 hash - this is the same as running `shasum -a 512 {release}` and comparing it against `{release}.sha512` + def get_sha512_hash(filename): """Run the shasum command on the file and return the SHA512 hash.""" - result = subprocess.run(['shasum', '-a', '512', filename], stdout=subprocess.PIPE) + result = subprocess.run(["shasum", "-a", "512", filename], stdout=subprocess.PIPE) sha512_hash = result.stdout.decode().split()[0] return sha512_hash + def read_sha512_file(filename): """Read the corresponding .sha512 file and process its contents.""" - sha_filename = filename + '.sha512' - with open(sha_filename, 'r') as file: + sha_filename = filename + ".sha512" + with open(sha_filename) as file: lines = file.readlines() - processed_sha = ''.join(lines[1:]).replace(' ', '').replace('\n', '').lower() + processed_sha = "".join(lines[1:]).replace(" ", "").replace("\n", "").lower() return processed_sha + def verify_sha512(filename): """Verify if the SHA512 hash of the file matches with the hash in the .sha512 file.""" sha512_hash = get_sha512_hash(filename) @@ -29,15 +33,19 @@ def verify_sha512(filename): else: return "SHA failed" + # Part 2: Verify RSA key - this is the same as running `gpg --verify {release}.asc {release}` and comparing the RSA key and email address against the KEYS file + def get_gpg_info(filename): """Run the GPG verify command and extract RSA key and email address.""" - asc_filename = filename + '.asc' - result = subprocess.run(['gpg', '--verify', asc_filename, filename], stderr=subprocess.PIPE, stdout=subprocess.PIPE) + asc_filename = filename + ".asc" + result = subprocess.run( + ["gpg", "--verify", asc_filename, filename], capture_output=True + ) output = result.stderr.decode() - rsa_key = re.search(r'RSA key ([0-9A-F]+)', output) + rsa_key = re.search(r"RSA key ([0-9A-F]+)", output) email = re.search(r'issuer "([^"]+)"', output) rsa_key_result = rsa_key.group(1) if rsa_key else None @@ -54,7 +62,7 @@ def get_gpg_info(filename): def verify_rsa_key(rsa_key, email): """Fetch the KEYS file and verify if the RSA key and email match.""" - url = 'https://downloads.apache.org/superset/KEYS' + url = "https://downloads.apache.org/superset/KEYS" response = requests.get(url) if response.status_code == 200: if rsa_key not in response.text: @@ -83,6 +91,7 @@ def verify_sha512_and_rsa(filename): else: print("GPG verification failed: RSA key or email not found") + if __name__ == "__main__": if len(sys.argv) != 2: print("Usage: python script.py ") From 8d3d64253a322fea7ffa2bc57675fbf32af66066 Mon Sep 17 00:00:00 2001 From: Evan Rusackas Date: Thu, 14 Dec 2023 12:25:38 -0700 Subject: [PATCH 3/4] improved typing --- RELEASING/verify_release.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/RELEASING/verify_release.py b/RELEASING/verify_release.py index 2be55cf54467a..067154e1901f7 100644 --- a/RELEASING/verify_release.py +++ b/RELEASING/verify_release.py @@ -1,20 +1,21 @@ import re import subprocess import sys +from typing import Optional import requests # Part 1: Verify SHA512 hash - this is the same as running `shasum -a 512 {release}` and comparing it against `{release}.sha512` -def get_sha512_hash(filename): +def get_sha512_hash(filename: str) -> str: """Run the shasum command on the file and return the SHA512 hash.""" result = subprocess.run(["shasum", "-a", "512", filename], stdout=subprocess.PIPE) sha512_hash = result.stdout.decode().split()[0] return sha512_hash -def read_sha512_file(filename): +def read_sha512_file(filename: str) -> str: """Read the corresponding .sha512 file and process its contents.""" sha_filename = filename + ".sha512" with open(sha_filename) as file: @@ -23,7 +24,7 @@ def read_sha512_file(filename): return processed_sha -def verify_sha512(filename): +def verify_sha512(filename: str) -> str: """Verify if the SHA512 hash of the file matches with the hash in the .sha512 file.""" sha512_hash = get_sha512_hash(filename) sha512_file_content = read_sha512_file(filename) @@ -37,7 +38,7 @@ def verify_sha512(filename): # Part 2: Verify RSA key - this is the same as running `gpg --verify {release}.asc {release}` and comparing the RSA key and email address against the KEYS file -def get_gpg_info(filename): +def get_gpg_info(filename: str) -> tuple[Optional[str], Optional[str]]: """Run the GPG verify command and extract RSA key and email address.""" asc_filename = filename + ".asc" result = subprocess.run( @@ -60,7 +61,7 @@ def get_gpg_info(filename): return rsa_key_result, email_result -def verify_rsa_key(rsa_key, email): +def verify_rsa_key(rsa_key: str, email: Optional[str]) -> str: """Fetch the KEYS file and verify if the RSA key and email match.""" url = "https://downloads.apache.org/superset/KEYS" response = requests.get(url) @@ -79,7 +80,7 @@ def verify_rsa_key(rsa_key, email): return "Failed to fetch KEYS file" -def verify_sha512_and_rsa(filename): +def verify_sha512_and_rsa(filename: str) -> None: """Verify SHA512 hash and RSA key.""" sha_result = verify_sha512(filename) print(sha_result) From 396d064f58c7480f28988e49315d5e57f36403ff Mon Sep 17 00:00:00 2001 From: Evan Rusackas Date: Fri, 15 Dec 2023 15:43:53 -0700 Subject: [PATCH 4/4] more supporting scripts --- RELEASING/README.md | 11 +++--- RELEASING/validate_this_release.sh | 54 ++++++++++++++++++++++++++++++ RELEASING/verify_release.py | 0 superset-frontend/package.json | 3 +- 4 files changed, 62 insertions(+), 6 deletions(-) create mode 100755 RELEASING/validate_this_release.sh mode change 100644 => 100755 RELEASING/verify_release.py diff --git a/RELEASING/README.md b/RELEASING/README.md index 0f2668ddf1e6a..b0cad547caa82 100644 --- a/RELEASING/README.md +++ b/RELEASING/README.md @@ -391,19 +391,20 @@ The script will generate the email text that should be sent to dev@superset.apac Official instructions: https://www.apache.org/info/verification.html -We now have a handy script for anyone validating a release to use. It's in this very folder, `verify_release.py` - -Just make sure you have all three release files in the directory (`{some version}.tar.gz`, `{some version}.tar.gz.asc` and `{some version}tar.gz.sha512`). Then run the script with the path to the `.tar.gz` file like so: - +We now have a handy script for anyone validating a release to use. The core of it is in this very folder, `verify_release.py`. Just make sure you have all three release files in the same directory (`{some version}.tar.gz`, `{some version}.tar.gz.asc` and `{some version}tar.gz.sha512`). Then you can pass this script the path to the `.gz` file like so: `python verify_release.py ~/path/tp/apache-superset-{version/candidate}-source.tar.gz` -If all goes well, you will see this result in your terminal: +If all goes well, you will see this result in your terminal: ```bash SHA-512 verified RSA key verified ``` +There are also additional support scripts leveraging this to make it easy for those downloading a release to test it in-situ. You can do either of the following to validate these release assets: +* `cd` into `superset-frontend` and run `npm run validate-release` +* `cd` into `RELEASES` and run `./validate_this_release.sh` + ## Publishing a successful release Upon a successful vote, you'll have to copy the folder into the non-"dev/" folder. diff --git a/RELEASING/validate_this_release.sh b/RELEASING/validate_this_release.sh new file mode 100755 index 0000000000000..6d2b6fca5f4dd --- /dev/null +++ b/RELEASING/validate_this_release.sh @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#!/bin/bash + +# Function to determine Python command +get_python_command() { + if command -v python3 &>/dev/null; then + echo "python3" + else + echo "python" + fi +} + +# Function to determine Pip command +get_pip_command() { + if command -v pip3 &>/dev/null; then + echo "pip3" + else + echo "pip" + fi +} + +PYTHON=$(get_python_command) +PIP=$(get_pip_command) + +# Get the release directory's path. If you unzip an Apache release and just run the npm script to validate the release, this will be a file name like `apache-superset-x.x.xrcx-source.tar.gz` +RELEASE_DIR_NAME="../../$(basename "$(dirname "$(pwd)")").tar.gz" + +# Install dependencies from requirements.txt if the file exists +if [ -f "path/to/requirements.txt" ]; then + echo "Installing Python dependencies..." + $PYTHON -m $PIP install -r path/to/requirements.txt +fi + +# echo $PYTHON +# echo $RELEASE_DIR_NAME + +# Run the Python script with the parent directory name as an argument +$PYTHON ../RELEASING/verify_release.py "$RELEASE_DIR_NAME" diff --git a/RELEASING/verify_release.py b/RELEASING/verify_release.py old mode 100644 new mode 100755 diff --git a/superset-frontend/package.json b/superset-frontend/package.json index 083b83c69e990..c60a3bf95f832 100644 --- a/superset-frontend/package.json +++ b/superset-frontend/package.json @@ -70,7 +70,8 @@ "storybook": "cross-env NODE_ENV=development BABEL_ENV=development start-storybook -p 6006", "tdd": "cross-env NODE_ENV=test jest --watch", "test": "cross-env NODE_ENV=test jest", - "type": "tsc --noEmit" + "type": "tsc --noEmit", + "validate-release": "../RELEASING/validate_this_release.sh" }, "browserslist": [ "last 3 chrome versions",