used get_spark() in pivoter

Tagar · Feb 18, 2018 · c586fc8 · c586fc8
1 parent 1ee5856
commit c586fc8
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 1 deletion.
diff --git a/abalon/spark/pivoter.py b/abalon/spark/pivoter.py
@@ -46,6 +46,7 @@
 
 ###########################################################################################################
 
+from abalon.spark.sparkutils import get_spark
 from pyspark.sql.types import *
 
 
@@ -85,6 +86,8 @@ def map_dict_to_denseArray (self, idx, d):
 
     def pivot_df (self, df, idx_col, all_vars):
 
+        spark = get_spark()
+
         if not all_vars:
             # get list of variables from the dataset:
             all_vars = sorted([row[0] for row in df.rdd.map(lambda (idx, k, v): k).distinct().collect()])

diff --git a/abalon/spark/sparkutils.py b/abalon/spark/sparkutils.py
@@ -77,6 +77,12 @@ def sparkutils_init (i_spark=None, i_debug=False):
     sparkutils_init_complete = True
 
 
+def get_spark ():
+
+    sparkutils_init()
+    return spark
+
+
 ###########################################################################################################
 
 

diff --git a/abalon/version.py b/abalon/version.py
@@ -12,6 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-version = '2.1.4'
+version = '2.1.5'