forked from NVIDIA/spark-rapids
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Split DB scripts to make them common for the build and IT pipeline (N…
…VIDIA#1933) * Split DB scripts to make them common for the build and IT pipeline Signed-off-by: Tim Liu <timl@nvidia.com> * update getopt for jar_path * Add 'cluster_type' parameter to make create.py common for aws and azure Databricks Signed-off-by: Tim Liu <timl@nvidia.com> * Change to use a more readable var name change the var name from cluster_type to cloud_provider. Signed-off-by: Tim Liu <timl@nvidia.com>
- Loading branch information
Showing
7 changed files
with
188 additions
and
109 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| # Copyright (c) 2021, NVIDIA CORPORATION. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| import sys | ||
| import getopt | ||
|
|
||
| workspace = 'https://dbc-9ff9942e-a9c4.cloud.databricks.com' | ||
| token = '' | ||
| private_key_file = "~/.ssh/id_rsa" | ||
| local_script = 'build.sh' | ||
| script_dest = '/home/ubuntu/build.sh' | ||
| source_tgz = 'spark-rapids-ci.tgz' | ||
| tgz_dest = '/home/ubuntu/spark-rapids-ci.tgz' | ||
| base_spark_pom_version = '3.0.0' | ||
| clusterid = '' | ||
| build_profiles = 'databricks,!snapshot-shims' | ||
| jar_path = '' | ||
|
|
||
| try: | ||
| opts, args = getopt.getopt(sys.argv[1:], 'hw:t:c:p:l:d:z:m:v:b:j:', | ||
| ['workspace=', 'token=', 'clusterid=', 'private=', 'localscript=', 'dest=', 'sparktgz=', 'basesparkpomversion=', 'buildprofiles=', 'jarpath']) | ||
| except getopt.GetoptError: | ||
| print( | ||
| 'run-tests.py -s <workspace> -t <token> -c <clusterid> -p <privatekeyfile> -l <localscript> -d <scriptdestinatino> -z <sparktgz> -v <basesparkpomversion> -b <buildprofiles> -j <jarpath>') | ||
| sys.exit(2) | ||
|
|
||
| for opt, arg in opts: | ||
| if opt == '-h': | ||
| print( | ||
| 'run-tests.py -s <workspace> -t <token> -c <clusterid> -p <privatekeyfile> -n <skipstartingcluster> -l <localscript> -d <scriptdestinatino>, -z <sparktgz> -v <basesparkpomversion> -b <buildprofiles>') | ||
| sys.exit() | ||
| elif opt in ('-w', '--workspace'): | ||
| workspace = arg | ||
| elif opt in ('-t', '--token'): | ||
| token = arg | ||
| elif opt in ('-c', '--clusterid'): | ||
| clusterid = arg | ||
| elif opt in ('-p', '--private'): | ||
| private_key_file = arg | ||
| elif opt in ('-l', '--localscript'): | ||
| local_script = arg | ||
| elif opt in ('-d', '--dest'): | ||
| script_dest = arg | ||
| elif opt in ('-z', '--sparktgz'): | ||
| source_tgz = arg | ||
| elif opt in ('-v', '--basesparkpomversion'): | ||
| base_spark_pom_version = arg | ||
| elif opt in ('-b', '--bulidprofiles'): | ||
| build_profiles = arg | ||
| elif opt in ('-j', '--jarpath'): | ||
| jar_path = arg | ||
|
|
||
| print('-w is ' + workspace) | ||
| print('-c is ' + clusterid) | ||
| print('-p is ' + private_key_file) | ||
| print('-l is ' + local_script) | ||
| print('-d is ' + script_dest) | ||
| print('-z is ' + source_tgz) | ||
| print('-v is ' + base_spark_pom_version) | ||
| print('-j is ' + jar_path) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| # Copyright (c) 2021, NVIDIA CORPORATION. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| import json | ||
| import requests | ||
| import sys | ||
| import getopt | ||
| import time | ||
| import os | ||
| import subprocess | ||
| from clusterutils import ClusterUtils | ||
| import params | ||
|
|
||
| def main(): | ||
| master_addr = ClusterUtils.cluster_get_master_addr(params.workspace, params.clusterid, params.token) | ||
| if master_addr is None: | ||
| print("Error, didn't get master address") | ||
| sys.exit(1) | ||
| print("Master node address is: %s" % master_addr) | ||
|
|
||
| print("Copying script") | ||
| rsync_command = "rsync -I -Pave \"ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2200 -i %s\" %s ubuntu@%s:%s" % (params.private_key_file, params.local_script, master_addr, params.script_dest) | ||
| print("rsync command: %s" % rsync_command) | ||
| subprocess.check_call(rsync_command, shell = True) | ||
|
|
||
| print("Copying source") | ||
| rsync_command = "rsync -I -Pave \"ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2200 -i %s\" %s ubuntu@%s:%s" % (params.private_key_file, params.source_tgz, master_addr, params.tgz_dest) | ||
| print("rsync command: %s" % rsync_command) | ||
| subprocess.check_call(rsync_command, shell = True) | ||
|
|
||
| ssh_command = "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@%s -p 2200 -i %s %s %s %s %s 2>&1 | tee buildout; if [ `echo ${PIPESTATUS[0]}` -ne 0 ]; then false; else true; fi" % (master_addr, params.private_key_file, params.script_dest, params.tgz_dest, params.base_spark_pom_version, params.build_profiles) | ||
| print("ssh command: %s" % ssh_command) | ||
| subprocess.check_call(ssh_command, shell = True) | ||
|
|
||
| print("Copying built tarball back") | ||
| rsync_command = "rsync -I -Pave \"ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2200 -i %s\" ubuntu@%s:/home/ubuntu/spark-rapids-built.tgz ./" % (params.private_key_file, master_addr) | ||
| print("rsync command to get built tarball: %s" % rsync_command) | ||
| subprocess.check_call(rsync_command, shell = True) | ||
|
|
||
| if __name__ == '__main__': | ||
| main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| #!/bin/bash | ||
| # | ||
| # Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # | ||
|
|
||
| set -e | ||
|
|
||
| LOCAL_JAR_PATH=$1 | ||
|
|
||
| # tests | ||
| export PATH=/databricks/conda/envs/databricks-ml-gpu/bin:/databricks/conda/condabin:$PATH | ||
| sudo /databricks/conda/envs/databricks-ml-gpu/bin/pip install pytest sre_yield requests pandas \ | ||
| pyarrow findspark pytest-xdist | ||
|
|
||
| export SPARK_HOME=/databricks/spark | ||
| # change to not point at databricks confs so we don't conflict with their settings | ||
| export SPARK_CONF_DIR=$PWD | ||
| export PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/pyspark/:$SPARK_HOME/python/lib/py4j-0.10.9-src.zip | ||
| sudo ln -s /databricks/jars/ $SPARK_HOME/jars || true | ||
| sudo chmod 777 /databricks/data/logs/ | ||
| sudo chmod 777 /databricks/data/logs/* | ||
| echo { \"port\":\"15002\" } > ~/.databricks-connect | ||
|
|
||
| if [ -d "$LOCAL_JAR_PATH" ]; then | ||
| ## Run tests with jars in the LOCAL_JAR_PATH dir downloading from the denpedency repo | ||
| LOCAL_JAR_PATH=$LOCAL_JAR_PATH bash $LOCAL_JAR_PATH/integration_tests/run_pyspark_from_build.sh --runtime_env="databricks" | ||
| else | ||
| ## Run tests with jars building from the spark-rapids source code | ||
| bash /home/ubuntu/spark-rapids/integration_tests/run_pyspark_from_build.sh --runtime_env="databricks" | ||
| fi |