From 7c544abf4c30474d504b74f2d4c252dc55f1ba4e Mon Sep 17 00:00:00 2001 From: Cheng Su Date: Sun, 2 Dec 2018 18:00:59 -0800 Subject: [PATCH] Fix the unit test in BasicWriteTaskStatsTrackerSuite --- .../BasicWriteTaskStatsTrackerSuite.scala | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/BasicWriteTaskStatsTrackerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/BasicWriteTaskStatsTrackerSuite.scala index 32941d8d2cd11..36a143fad7a78 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/BasicWriteTaskStatsTrackerSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/BasicWriteTaskStatsTrackerSuite.scala @@ -124,6 +124,7 @@ class BasicWriteTaskStatsTrackerSuite extends SparkFunSuite { val tracker = new BasicWriteTaskStatsTracker(conf) tracker.newFile(file.toString) val stream = localfs.create(file, true) + try { assertStats(tracker, 1, 0) stream.write(data1) @@ -148,7 +149,7 @@ class BasicWriteTaskStatsTrackerSuite extends SparkFunSuite { test("Three files, last one empty") { val file1 = new Path(tempDirPath, "f-3-1") val file2 = new Path(tempDirPath, "f-3-2") - val file3 = new Path(tempDirPath, "f-3-2") + val file3 = new Path(tempDirPath, "f-3-3") val tracker = new BasicWriteTaskStatsTracker(conf) tracker.newFile(file1.toString) write1(file1) @@ -180,6 +181,21 @@ class BasicWriteTaskStatsTrackerSuite extends SparkFunSuite { assertStats(tracker, 2, len1 + len2) } + test("Three files, one duplicated") { + val file1 = new Path(tempDirPath, "f-3-1") + val file2 = new Path(tempDirPath, "f-3-2") + val tracker = new BasicWriteTaskStatsTracker(conf) + tracker.newFile(file1.toString) + write1(file1) + tracker.newFile(file2.toString) + write2(file2) + // file 2 is noted again + tracker.newFile(file2.toString) + // the expected size is file1 + file2; only two files are reported + // as found + assertStats(tracker, 2, len1 + len2) + } + /** * Write a 0-byte file. * @param file file path