diff --git a/docs/get-started/getting-started-aws-emr.md b/docs/get-started/getting-started-aws-emr.md index bbb0bcf240f..7209f583892 100644 --- a/docs/get-started/getting-started-aws-emr.md +++ b/docs/get-started/getting-started-aws-emr.md @@ -136,7 +136,6 @@ default settings: "spark.rapids.memory.pinnedPool.size":"2G", "spark.executor.memoryOverhead":"2G", "spark.locality.wait":"0s", - "spark.sql.shuffle.partitions":"200", "spark.sql.files.maxPartitionBytes":"256m", "spark.sql.adaptive.enabled":"false" } diff --git a/docs/get-started/getting-started-on-prem.md b/docs/get-started/getting-started-on-prem.md index 356814f3bce..54575011b87 100644 --- a/docs/get-started/getting-started-on-prem.md +++ b/docs/get-started/getting-started-on-prem.md @@ -98,7 +98,6 @@ $SPARK_HOME/bin/spark-shell \ --conf spark.rapids.memory.pinnedPool.size=2G \ --conf spark.locality.wait=0s \ --conf spark.sql.files.maxPartitionBytes=512m \ - --conf spark.sql.shuffle.partitions=10 \ --conf spark.plugins=com.nvidia.spark.SQLPlugin \ --jars ${SPARK_CUDF_JAR},${SPARK_RAPIDS_PLUGIN_JAR} ``` @@ -173,7 +172,6 @@ $SPARK_HOME/bin/spark-shell \ --conf spark.rapids.memory.pinnedPool.size=2G \ --conf spark.locality.wait=0s \ --conf spark.sql.files.maxPartitionBytes=512m \ - --conf spark.sql.shuffle.partitions=10 \ --conf spark.plugins=com.nvidia.spark.SQLPlugin ``` @@ -226,7 +224,6 @@ $SPARK_HOME/bin/spark-shell \ --conf spark.rapids.memory.pinnedPool.size=2G \ --conf spark.locality.wait=0s \ --conf spark.sql.files.maxPartitionBytes=512m \ - --conf spark.sql.shuffle.partitions=10 \ --conf spark.plugins=com.nvidia.spark.SQLPlugin \ --conf spark.executor.resource.gpu.discoveryScript=./getGpusResources.sh \ --files ${SPARK_RAPIDS_DIR}/getGpusResources.sh \ @@ -253,7 +250,6 @@ $SPARK_HOME/bin/spark-shell \ --conf spark.rapids.memory.pinnedPool.size=2G \ --conf spark.locality.wait=0s \ --conf spark.sql.files.maxPartitionBytes=512m \ - --conf spark.sql.shuffle.partitions=10 \ --conf spark.plugins=com.nvidia.spark.SQLPlugin \ --conf spark.executor.resource.gpu.amount=1 \ --conf spark.executor.resource.gpu.discoveryScript=./getGpusResources.sh \ @@ -291,7 +287,6 @@ $SPARK_HOME/bin/spark-shell \ --conf spark.rapids.memory.pinnedPool.size=2G \ --conf spark.locality.wait=0s \ --conf spark.sql.files.maxPartitionBytes=512m \ - --conf spark.sql.shuffle.partitions=10 \ --conf spark.plugins=com.nvidia.spark.SQLPlugin \ --conf spark.resources.discoveryPlugin=com.nvidia.spark.ExclusiveModeGpuDiscoveryPlugin \ --conf spark.executor.resource.gpu.amount=1 \ @@ -339,7 +334,6 @@ $SPARK_HOME/bin/spark-shell \ --conf spark.rapids.memory.pinnedPool.size=2G \ --conf spark.locality.wait=0s \ --conf spark.sql.files.maxPartitionBytes=512m \ - --conf spark.sql.shuffle.partitions=10 \ --conf spark.plugins=com.nvidia.spark.SQLPlugin \ --conf spark.executor.resource.gpu.amount=1 \ --conf spark.executor.resource.gpu.discoveryScript=/opt/sparkRapidsPlugin/getGpusResources.sh \ diff --git a/docs/get-started/yarn-gpu.md b/docs/get-started/yarn-gpu.md index 462b9dc593b..42b8f64293e 100644 --- a/docs/get-started/yarn-gpu.md +++ b/docs/get-started/yarn-gpu.md @@ -29,7 +29,6 @@ spark.plugins=com.nvidia.spark.SQLPlugin spark.executor.extraJavaOptions='-Dai.rapids.cudf.prefer-pinned=true' spark.locality.wait=0s spark.executor.resource.gpu.discoveryScript=/usr/lib/spark/scripts/gpu/getGpusResources.sh # this match the location of discovery script -spark.sql.shuffle.partitions=40 spark.sql.files.maxPartitionBytes=512m ```