From b57c8bb448c462813b1fd498372b1a1f22cd2a44 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Tue, 30 Sep 2014 13:28:18 +0200 Subject: [PATCH] Rename sparkConsumerParallelism to sparkProcessingParallelism to clarify meaning --- .../com/miguno/kafkastorm/spark/KafkaSparkStreamingSpec.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/scala/com/miguno/kafkastorm/spark/KafkaSparkStreamingSpec.scala b/src/test/scala/com/miguno/kafkastorm/spark/KafkaSparkStreamingSpec.scala index b8b7b9d..8f3143b 100644 --- a/src/test/scala/com/miguno/kafkastorm/spark/KafkaSparkStreamingSpec.scala +++ b/src/test/scala/com/miguno/kafkastorm/spark/KafkaSparkStreamingSpec.scala @@ -181,11 +181,11 @@ class KafkaSparkStreamingSpec extends FeatureSpec with Matchers with BeforeAndAf ).map(_._2) } val unifiedStream = ssc.union(streams) // Merge the "per-partition" DStreams - val sparkConsumerParallelism = 1 // You'd probably pick a higher value than 1 in production. + val sparkProcessingParallelism = 1 // You'd probably pick a higher value than 1 in production. // Repartition distributes the received batches of data across specified number of machines in the cluster // before further processing. Essentially, what we are doing here is to decouple processing parallelism from // reading parallelism (limited by #partitions). - unifiedStream.repartition(sparkConsumerParallelism) + unifiedStream.repartition(sparkProcessingParallelism) } // We use accumulators to track the number of consumed and produced messages across all tasks. Named accumulators