header in multiple file chunks bugfix

Tagar · Nov 5, 2017 · 5ffe084 · 5ffe084
1 parent c6c3cb2
commit 5ffe084
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/abalon/spark/sparkutils.py b/abalon/spark/sparkutils.py
@@ -186,7 +186,8 @@ def dataframeToHDFSfile (dataframe, dst_file, overwrite=False,
 
     (dataframe
         .write
-        .option('header', header)
+        .option('header', False)    # always save without header as if there are multiple partitions,
+                                    # each datafile will have a header - not good.
         .option('delimiter', delimiter)
         .option('quoteMode', quoteMode)
         .mode('overwrite')     # temp directory will always be overwritten