Branch 0.5 doc remove numpartitions (#1951)

* Remove --conf spark.sql.shuffle.partitions=10 from docs Signed-off-by: Sameer Raheja <sraheja@nvidia.com> * Remove spark.sql.shuffle.partitions=40 from get-started/yarn-gpu.md Signed-off-by: Sameer Raheja <sraheja@nvidia.com> * Remove spark.sql.shuffle.partitions from AWS EMR getting started gui Signed-off-by: Sameer Raheja <sraheja@nvidia.com>
NVIDIA · Mar 19, 2021 · a50ddfb · a50ddfb
1 parent 706c4ef
commit a50ddfb
Show file tree

Hide file tree

Showing 3 changed files with 0 additions and 8 deletions.
diff --git a/docs/get-started/getting-started-aws-emr.md b/docs/get-started/getting-started-aws-emr.md
@@ -136,7 +136,6 @@ default settings:
         "spark.rapids.memory.pinnedPool.size":"2G",
         "spark.executor.memoryOverhead":"2G",
         "spark.locality.wait":"0s",
-        "spark.sql.shuffle.partitions":"200",
         "spark.sql.files.maxPartitionBytes":"256m",
         "spark.sql.adaptive.enabled":"false"
         }

diff --git a/docs/get-started/getting-started-on-prem.md b/docs/get-started/getting-started-on-prem.md
@@ -98,7 +98,6 @@ $SPARK_HOME/bin/spark-shell \
        --conf spark.rapids.memory.pinnedPool.size=2G \
        --conf spark.locality.wait=0s \
        --conf spark.sql.files.maxPartitionBytes=512m \
-       --conf spark.sql.shuffle.partitions=10 \
        --conf spark.plugins=com.nvidia.spark.SQLPlugin \
        --jars ${SPARK_CUDF_JAR},${SPARK_RAPIDS_PLUGIN_JAR}
 ```
@@ -173,7 +172,6 @@ $SPARK_HOME/bin/spark-shell \
        --conf spark.rapids.memory.pinnedPool.size=2G \
        --conf spark.locality.wait=0s \
        --conf spark.sql.files.maxPartitionBytes=512m \
-       --conf spark.sql.shuffle.partitions=10 \
        --conf spark.plugins=com.nvidia.spark.SQLPlugin
 ```
 
@@ -226,7 +224,6 @@ $SPARK_HOME/bin/spark-shell \
        --conf spark.rapids.memory.pinnedPool.size=2G \
        --conf spark.locality.wait=0s \
        --conf spark.sql.files.maxPartitionBytes=512m \
-       --conf spark.sql.shuffle.partitions=10 \
        --conf spark.plugins=com.nvidia.spark.SQLPlugin \
        --conf spark.executor.resource.gpu.discoveryScript=./getGpusResources.sh \
        --files ${SPARK_RAPIDS_DIR}/getGpusResources.sh \
@@ -253,7 +250,6 @@ $SPARK_HOME/bin/spark-shell \
        --conf spark.rapids.memory.pinnedPool.size=2G \
        --conf spark.locality.wait=0s \
        --conf spark.sql.files.maxPartitionBytes=512m \
-       --conf spark.sql.shuffle.partitions=10 \
        --conf spark.plugins=com.nvidia.spark.SQLPlugin \
        --conf spark.executor.resource.gpu.amount=1 \
        --conf spark.executor.resource.gpu.discoveryScript=./getGpusResources.sh \
@@ -291,7 +287,6 @@ $SPARK_HOME/bin/spark-shell \
        --conf spark.rapids.memory.pinnedPool.size=2G \
        --conf spark.locality.wait=0s \
        --conf spark.sql.files.maxPartitionBytes=512m \
-       --conf spark.sql.shuffle.partitions=10 \
        --conf spark.plugins=com.nvidia.spark.SQLPlugin \
        --conf spark.resources.discoveryPlugin=com.nvidia.spark.ExclusiveModeGpuDiscoveryPlugin \
        --conf spark.executor.resource.gpu.amount=1 \
@@ -339,7 +334,6 @@ $SPARK_HOME/bin/spark-shell \
        --conf spark.rapids.memory.pinnedPool.size=2G \
        --conf spark.locality.wait=0s \
        --conf spark.sql.files.maxPartitionBytes=512m \
-       --conf spark.sql.shuffle.partitions=10 \
        --conf spark.plugins=com.nvidia.spark.SQLPlugin \
        --conf spark.executor.resource.gpu.amount=1 \
        --conf spark.executor.resource.gpu.discoveryScript=/opt/sparkRapidsPlugin/getGpusResources.sh \

diff --git a/docs/get-started/yarn-gpu.md b/docs/get-started/yarn-gpu.md
@@ -29,7 +29,6 @@ spark.plugins=com.nvidia.spark.SQLPlugin
 spark.executor.extraJavaOptions='-Dai.rapids.cudf.prefer-pinned=true'
 spark.locality.wait=0s
 spark.executor.resource.gpu.discoveryScript=/usr/lib/spark/scripts/gpu/getGpusResources.sh # this match the location of discovery script
-spark.sql.shuffle.partitions=40
 spark.sql.files.maxPartitionBytes=512m
 ```