From e3a08166d49f49b92616fe2dfbed5138c253398c Mon Sep 17 00:00:00 2001
From: Joseph Balsamo <joseph.balsamo@stonybrook.edu>
Date: Tue, 23 Jul 2019 15:57:34 -0400
Subject: [PATCH 01/16] Fixed path issues

---
 convert_heatmaps.js | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/convert_heatmaps.js b/convert_heatmaps.js
index 22fbeb9..09b0a43 100644
--- a/convert_heatmaps.js
+++ b/convert_heatmaps.js
@@ -60,7 +60,7 @@ manifest.splice(0,1);
 manifest.forEach((line)=>{
   if(line != '') {
     let fileInfo = line.split(',');
-    mfData.push({ file: fileInfo[0],study_id:fileInfo[1],subject_id:fileInfo[2],image_id:fileInfo[3] });
+    mfData.push({ file: path.resolve(inputFolder + '/' +fileInfo[0]),study_id:fileInfo[1],subject_id:fileInfo[2],image_id:fileInfo[3] });
   }
 });
 
@@ -86,7 +86,7 @@ function convert(filename,metadata){
 
   // read file
   const myInterface = readline.createInterface({
-    input: fs.createReadStream(`${inputFolder}/${filename}`)
+    input: fs.createReadStream(filename)
   });
 
   remainder++;
@@ -127,10 +127,11 @@ function convert(filename,metadata){
       });
       res.on('end', function() {
         let result = JSON.parse(body);
+        let basename = path.basename(filename);
         slide_id = result[0].nid[0].value;
         const content = generateDoc(data,filename,metadata);
         if(!fs.existsSync(outputFolder)) fs.mkdirSync(outputFolder);
-        fs.writeFile(`${outputFolder}/NEW_${filename}`, content, function(err) {
+        fs.writeFile(`${outputFolder}/NEW_${basename}`, content, function(err) {
           if (err) throw err;
           remainder--;
           console.log(`${filename} completed`);

From 2822a01bd6b4808f66bbf8b44eb5f18de15a951c Mon Sep 17 00:00:00 2001
From: Joseph Balsamo <joseph.balsamo@stonybrook.edu>
Date: Fri, 26 Jul 2019 16:26:33 -0400
Subject: [PATCH 02/16] Errorchecking in Bash Script

---
 uploadHeatmaps.sh | 191 ++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 159 insertions(+), 32 deletions(-)

diff --git a/uploadHeatmaps.sh b/uploadHeatmaps.sh
index a87ab20..5d7967d 100755
--- a/uploadHeatmaps.sh
+++ b/uploadHeatmaps.sh
@@ -1,29 +1,60 @@
 #!/bin/bash
 # bash uploadHeatmaps.sh <options>
-# Authors: Alina Jasniewski, Joseph Balsamo
+# Authors: Joseph Balsamo
 
+#-----------------------------------------------------------------------
 # Functions
-# function: usage(brief)
+#-----------------------------------------------------------------------
+
+#-----------------------------------------------------------------------
+# Function: chk_opt($2)
+# Description: Takes an argument and determines if it is a flag instead of an parameter value
+#-----------------------------------------------------------------------
+function chk_opt() {
+  # Check the first char of the option passed 
+  checkc="$(echo $1 | head -c 1)"
+
+  if [ $checkc == "-" ]
+  then
+    echo 'true'
+  else
+    echo 'false'
+  fi
+}
+# End chk_opt
+#-----------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------
+# Function: usage(brief)
+# Description: Display help message.  When true is passed it will show a brief version showing the bare minimum flags needed.
+#-----------------------------------------------------------------------
 function usage() {
-    echo "Usage: $ ./uploadHeatmaps.sh [options] -c <pathDB_collection> -u <username> -p <password>"
+    echo "Usage: $ uploadHeatmaps.sh [options] -c <pathDB_collection>"
     if [ $1 == false ]
     then
       echo "  Options:"
-      echo "    -c <pathDB_collection>: PathDB Collection for heatmaps being loaded (this parameter required)"
-      echo "    -u <username>: PathDB username (this parameter required)"
-      echo "    -p <password>: PathDB password (this parameter required)"
-      echo "    -i <input_folder>: Folder where heatmaps are loaded from (default: /mnt/data/xfer/input)"
-      echo "    -o <output_folder>: Folder where converted heatmaps are imported from (default: /mnt/data/xfer/output)"
+      echo "    -c, --collection <pathDB_collection>: PathDB Collection for heatmaps (*this parameter required)"
+      echo "    -i, --input <input_folder>: Folder where heatmaps are loaded from (default: /mnt/data/xfer/input)"
+      echo "    -o, --output <output_folder>: Folder where converted heatmaps are imported from (default: /mnt/data/xfer/output)"
       echo "    -q <host>: ip or hostname of PathDB Server (default: quip-pathdb)"
       echo "    -h <host>: ip or hostname of database (default: ca-mongo)"
-      echo "    -d <database name> (default: camic)"
-      echo "    -P <database port> (default: 27017)"
+      echo "    -d, --database <database name> (default: camic)"
+      echo "    -p, --port <database port> (default: 27017)"
       echo ""
       echo "    --help Display full help usage."
       echo "  Notes: requires mongoDB client tools installed on running server"
     fi
 }
+#-----------------------------------------------------------------------
+
+#-----------------------------------------------------------------------
 # end functions
+#-----------------------------------------------------------------------
+
+#-----------------------------------------------------------------------
+# Main Script
+#-----------------------------------------------------------------------
 
 # Set Default variables.
 database="camic"
@@ -32,42 +63,118 @@ HOST="ca-mongo"
 errcode=0
 brief=true
 
+# Loop through all command-line arguments.
 while [ -n "$1" ]
 # while loop starts
 do
   # Process command-line flags
+  # Start Case
   case "$1" in
-    -q) qhost="$2"
-        shift;;
-    -h) host="$2"
-        shift;;
-    -P) port="$2"
-        shift ;;
-    -p) passw="$2"
-        shift ;;
-    -c) collection="$2"
-        shift ;;
-    -m) manifest="$2"
-        shift ;;
-    -u) uname="$2"
-        shift ;;
-    -i) in="$2"
-        shift ;;
-    -o) out=${2}
-        shift;;
-    -d) database=${2}
-        shift;;
+    -q | --quip-host) 
+        opterr="$(chk_opt ${2})"
+        if [ $opterr ==  'true' ]
+        then
+          echo "Error: Missing parameter."
+          exit 5
+        else
+          qhost="$2"
+          shift
+        fi;;
+    -h | --data-host) 
+        opterr="$(chk_opt ${2})"
+        if [ $opterr ==  'true' ]
+        then
+          echo "Error: Missing parameter."
+          exit 5
+        else
+          host="$2"
+          shift
+        fi;;
+    -p | --port) 
+        opterr="$(chk_opt ${2})"
+        if [ $opterr ==  'true' ]
+        then
+          echo "Error: Missing parameter."
+          exit 5
+        else
+          port="$2"
+          shift
+        fi;;
+    -c | --collection) 
+        opterr="$(chk_opt ${2})"
+        if [ $opterr ==  'true' ]
+        then
+          echo "Error: Missing parameter."
+          exit 5
+        else
+          collection="$2"
+          shift
+        fi;;
+    -m | --manifest) 
+        opterr="$(chk_opt ${2})"
+        if [ $opterr ==  'true' ]
+        then
+          echo "Error: Missing parameter."
+          exit 5
+        else
+          manifest="$2"
+          shift
+        fi;;
+    -i | --input) 
+        opterr="$(chk_opt ${2})"
+        if [ $opterr ==  'true' ]
+        then
+          echo "Error: Missing parameter."
+          exit 5
+        else
+          in="$2"
+          shift
+        fi;;
+    -o | --output) 
+        opterr="$(chk_opt ${2})"
+        if [ $opterr ==  'true' ]
+        then
+          echo "Error: Missing parameter."
+          exit 5
+        else
+          out=${2}
+          shift
+        fi;;
+    -d | --database) 
+        opterr="$(chk_opt ${2})"
+        if [ $opterr ==  'true' ]
+        then
+          echo "Error: Missing parameter."
+          exit 5
+        else
+          database=${2}
+          shift
+        fi;;
     --help)  
         usage false
         exit 0;;
     *) usage true;; 
   esac
+  # End of Case
   shift
 done
+# End of While
+
+# Request username and password for upload.
+if [ -z "${uname}" ] 
+then
+  echo -n "Username: "
+  read uname
+fi
+if [ -z "${passw}" ] 
+then
+  echo -n "Password: "
+  read -s passw
+fi
 
-if [ -z "${collection}" ] || [ -z "${passw}" ] || [ -z "${uname}" ] 
+if [ -z "${collection}" ]  
 then
-  echo "Missing required parameters"
+  echo "Missing required parameter"
   usage true
   exit 1
 fi
@@ -102,6 +209,26 @@ then
   database="camic"
 fi
 
+# Check that input and output folders exist.  
+if [ ! -f $in ]
+then
+  echo "Error: Input folder does not exist."
+  exit 2
+fi
+if [ ! -f $out ]
+then
+  echo "Error: Output folder does not exist."
+  exit 3
+fi
+
+# Verify that PathDB Server is reachable using current username/password combo
+ret_code="$(curl -Is -u ${uname}:${passw} http://${qhost}/ | head -1  | awk '{ print $2 }')"
+if [ ! $ret_code -lt 400 ]
+then
+  echo "Error: PathDB Server is unreachable."
+  exit 100
+fi
+
 # Convert heatmap data in the 'in' folder into uploadable json in the 'out' folder.
 node --max_old_space_size=16384 /usr/local/bin/convert_heatmaps.js -h ${qhost} -c ${collection} -m ${manifest} -i ${in} -o ${out} -u ${uname} -p ${passw}
 

From 7ab32a53bc7d94fc38144225a76a348f6c7782b3 Mon Sep 17 00:00:00 2001
From: Joseph Balsamo <joseph.balsamo@stonybrook.edu>
Date: Sat, 27 Jul 2019 11:47:19 -0400
Subject: [PATCH 03/16] Setup helper  scripts

---
 helpers.sh        | 30 ++++++++++++++++++++
 readpass.sh       | 64 +++++++++++++++++++++++++++++++++++++++++
 uploadHeatmaps.sh | 72 ++++++++++++++++++++---------------------------
 3 files changed, 125 insertions(+), 41 deletions(-)
 create mode 100644 helpers.sh
 create mode 100644 readpass.sh

diff --git a/helpers.sh b/helpers.sh
new file mode 100644
index 0000000..e0b5a9e
--- /dev/null
+++ b/helpers.sh
@@ -0,0 +1,30 @@
+#---------------------------------------------------
+# File: Helper Function Library
+#---------------------------------------------------
+
+#---------------------------------------------------
+# Function: chk_opt 
+# Description: Takes an argument and determines if it 
+#              is a flag instead of an parameter value.
+# parameters: cl argument
+# usage: opterr="$(chk_opt ${2})"
+#---------------------------------------------------
+function chk_opt() {
+  # Check the first char of the option passed 
+  checkc="$(echo $1 | head -c 1)"
+
+  if [ $checkc == "-" ]
+  then
+    echo 'true'
+  else
+    echo 'false'
+  fi
+}
+#-----------------------------------------------------------------------
+# End chk_opt
+#-----------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------
+# end functions
+#-----------------------------------------------------------------------
diff --git a/readpass.sh b/readpass.sh
new file mode 100644
index 0000000..737d374
--- /dev/null
+++ b/readpass.sh
@@ -0,0 +1,64 @@
+#---------------------------------------------------
+# File: ReadPass Function Library
+#---------------------------------------------------
+
+#---------------------------------------------------
+# Function: getPrompt
+# Description: Read a line of input hile echoing on
+#              the screen. 
+# parameters: prompt
+# usage: myvar="$(getPrompt 'prompt')"
+#---------------------------------------------------
+function getPrompt() {
+    prompt="$1 "
+    read -p "$prompt" input
+    echo "$input"
+}
+#---------------------------------------------------
+# End getPrompt
+#---------------------------------------------------
+
+#---------------------------------------------------
+# Function: getPass
+# Description: Read a line of input while showing '*'
+#              on the screen. 
+# parameters: prompt
+# usage: myPW="$(getPass 'prompt')"
+#---------------------------------------------------
+function getPass() {
+    prompt="$1 "
+    # Loop to get all characters of the password
+    while IFS= read -p "$prompt" -r -s -n 1 char 
+    do
+        # Check if the return key was pressed
+        if [[ $char == $'\0' ]];     then
+            break
+        fi
+        # Check to see if backspace was pressed
+        if [[ $char == $'\177' ]];  then
+            l=${#password} # length of password
+            # if l is > 0 then delete last char
+            # else do not remove any character
+            if [[ l -gt 0 ]]
+            then
+                prompt=$'\b \b'
+                password="${password%?}"
+            else
+                prompt=$''
+            fi
+        else
+            # Add new char to password
+            prompt='*'
+            password+="$char"
+        fi
+    done
+    # Echo out password to be able to capture
+    echo "$password"
+}
+#---------------------------------------------------
+# End getPass
+#---------------------------------------------------
+
+#---------------------------------------------------
+# End Function Library
+#---------------------------------------------------
diff --git a/uploadHeatmaps.sh b/uploadHeatmaps.sh
index 5d7967d..4221278 100755
--- a/uploadHeatmaps.sh
+++ b/uploadHeatmaps.sh
@@ -1,34 +1,30 @@
 #!/bin/bash
+#---------------------------------------------------
 # bash uploadHeatmaps.sh <options>
-# Authors: Joseph Balsamo
-
-#-----------------------------------------------------------------------
-# Functions
-#-----------------------------------------------------------------------
-
-#-----------------------------------------------------------------------
-# Function: chk_opt($2)
-# Description: Takes an argument and determines if it is a flag instead of an parameter value
-#-----------------------------------------------------------------------
-function chk_opt() {
-  # Check the first char of the option passed 
-  checkc="$(echo $1 | head -c 1)"
-
-  if [ $checkc == "-" ]
-  then
-    echo 'true'
-  else
-    echo 'false'
-  fi
-}
-# End chk_opt
-#-----------------------------------------------------------------------
-
-
-#-----------------------------------------------------------------------
+# Description: This script manages the conversion and
+#              upload of heatmaps to a containerized
+#              pathdb instance of quip.
+# Author(s): Joseph Balsamo
+#---------------------------------------------------
+
+#---------------------------------------------------
+# Source Function Libraries
+#---------------------------------------------------
+source helpers.sh
+source readpass.sh
+
+#---------------------------------------------------
+# Function Definitions
+#---------------------------------------------------
+
+#---------------------------------------------------
 # Function: usage(brief)
-# Description: Display help message.  When true is passed it will show a brief version showing the bare minimum flags needed.
-#-----------------------------------------------------------------------
+# Description: Display help message.  When true is passed 
+#              it will show a brief version showing the 
+#              bare minimum flags needed.
+# parameters: brief
+# usage: usage [true | false]
+#---------------------------------------------------
 function usage() {
     echo "Usage: $ uploadHeatmaps.sh [options] -c <pathDB_collection>"
     if [ $1 == false ]
@@ -47,11 +43,13 @@ function usage() {
     fi
 }
 #-----------------------------------------------------------------------
-
-#-----------------------------------------------------------------------
-# end functions
+# End usage
 #-----------------------------------------------------------------------
 
+#---------------------------------------------------
+# End Function Definitions
+#---------------------------------------------------
+
 #-----------------------------------------------------------------------
 # Main Script
 #-----------------------------------------------------------------------
@@ -161,16 +159,8 @@ done
 # End of While
 
 # Request username and password for upload.
-if [ -z "${uname}" ] 
-then
-  echo -n "Username: "
-  read uname
-fi
-if [ -z "${passw}" ] 
-then
-  echo -n "Password: "
-  read -s passw
-fi
+uname="$(getPrompt 'Username:')"
+passw="$(getPass 'Password:')"
 
 if [ -z "${collection}" ]  
 then

From 7ddf6f4e3f281966016ac292e47f2aaa3a01907c Mon Sep 17 00:00:00 2001
From: Joseph Balsamo <joseph.balsamo@stonybrook.edu>
Date: Sat, 27 Jul 2019 21:24:18 -0400
Subject: [PATCH 04/16] node error checks

---
 Dockerfile          | 2 ++
 convert_heatmaps.js | 5 +++++
 uploadHeatmaps.sh   | 4 ++--
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 241ae9e..1a1ea15 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -36,6 +36,8 @@ ADD root/.scripts /root/.scripts
 
 COPY convert_heatmaps.js /usr/local/bin/
 COPY uploadHeatmaps.sh /usr/local/bin/
+COPY helpers.sh /usr/local/bin/
+COPY readpass.sh /usr/local/bin/
 RUN chmod 0775 /usr/local/bin/uploadHeatmaps.sh
 RUN mkdir /mnt/data
 
diff --git a/convert_heatmaps.js b/convert_heatmaps.js
index 09b0a43..d907be0 100644
--- a/convert_heatmaps.js
+++ b/convert_heatmaps.js
@@ -56,6 +56,11 @@ const fileTemps = {};
 var manifest = [];
 const mfData = []
 manifest = fs.readFileSync(inputFolder + '/' + clio.manifest).toString().split('\n');
+// Exit with error if manifest file is empty
+if (manifest.length <= 1) {
+  console.error("Error: Empty manifest file");
+  process.exit(1001);
+}
 manifest.splice(0,1);
 manifest.forEach((line)=>{
   if(line != '') {
diff --git a/uploadHeatmaps.sh b/uploadHeatmaps.sh
index 4221278..7342983 100755
--- a/uploadHeatmaps.sh
+++ b/uploadHeatmaps.sh
@@ -33,8 +33,8 @@ function usage() {
       echo "    -c, --collection <pathDB_collection>: PathDB Collection for heatmaps (*this parameter required)"
       echo "    -i, --input <input_folder>: Folder where heatmaps are loaded from (default: /mnt/data/xfer/input)"
       echo "    -o, --output <output_folder>: Folder where converted heatmaps are imported from (default: /mnt/data/xfer/output)"
-      echo "    -q <host>: ip or hostname of PathDB Server (default: quip-pathdb)"
-      echo "    -h <host>: ip or hostname of database (default: ca-mongo)"
+      echo "    -q, --quip-host <host>: ip or hostname of PathDB Server (default: quip-pathdb)"
+      echo "    -h, --data-host <host>: ip or hostname of database (default: ca-mongo)"
       echo "    -d, --database <database name> (default: camic)"
       echo "    -p, --port <database port> (default: 27017)"
       echo ""

From b363436dae0af0c1950593b59618b70f81d3ea87 Mon Sep 17 00:00:00 2001
From: Joseph Balsamo <joseph.balsamo@stonybrook.edu>
Date: Mon, 29 Jul 2019 09:53:37 -0400
Subject: [PATCH 05/16] Updated settings

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index c6f9a44..3fc6a99 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
 .vscode/settings.json
+quip.code-workspace

From 7ce69c2df4ad9ab5467647aa1b1536cb7d0259e6 Mon Sep 17 00:00:00 2001
From: Joseph Balsamo <joseph.balsamo@stonybrook.edu>
Date: Mon, 29 Jul 2019 16:31:47 -0400
Subject: [PATCH 06/16] Add filecheck to heatmap data

---
 convert_heatmaps.js | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/convert_heatmaps.js b/convert_heatmaps.js
index d907be0..efcb202 100644
--- a/convert_heatmaps.js
+++ b/convert_heatmaps.js
@@ -89,6 +89,12 @@ function convert(filename,metadata){
   let fields = [];
   let ranges = [0,1];
 
+  try {
+    fs.accessSync(filename,fs.F_OK);
+  } catch(e) {
+    console.log(e.message);
+    process.exit(30);
+  }
   // read file
   const myInterface = readline.createInterface({
     input: fs.createReadStream(filename)

From d46569f981d06c4882af3837c9d7822c2961958a Mon Sep 17 00:00:00 2001
From: Joseph Balsamo <joseph.balsamo@stonybrook.edu>
Date: Mon, 29 Jul 2019 22:31:31 -0400
Subject: [PATCH 07/16] error check added

---
 convert_heatmaps.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/convert_heatmaps.js b/convert_heatmaps.js
index efcb202..7791df6 100644
--- a/convert_heatmaps.js
+++ b/convert_heatmaps.js
@@ -59,7 +59,7 @@ manifest = fs.readFileSync(inputFolder + '/' + clio.manifest).toString().split('
 // Exit with error if manifest file is empty
 if (manifest.length <= 1) {
   console.error("Error: Empty manifest file");
-  process.exit(1001);
+  process.exit(50);
 }
 manifest.splice(0,1);
 manifest.forEach((line)=>{
@@ -93,7 +93,7 @@ function convert(filename,metadata){
     fs.accessSync(filename,fs.F_OK);
   } catch(e) {
     console.log(e.message);
-    process.exit(30);
+    process.exit(51);
   }
   // read file
   const myInterface = readline.createInterface({

From 16869765f5a85850ab9812f387a0113625cee301 Mon Sep 17 00:00:00 2001
From: Joseph Balsamo <joseph.balsamo@stonybrook.edu>
Date: Tue, 30 Jul 2019 12:44:16 -0400
Subject: [PATCH 08/16] Final error checks and comments

---
 Dockerfile          |  1 +
 convert_heatmaps.js | 68 ++++++++++++++++++++++++++++++++++++++++-----
 uploadHeatmaps.sh   | 18 ++++++++----
 3 files changed, 75 insertions(+), 12 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 1a1ea15..ef53903 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -40,6 +40,7 @@ COPY helpers.sh /usr/local/bin/
 COPY readpass.sh /usr/local/bin/
 RUN chmod 0775 /usr/local/bin/uploadHeatmaps.sh
 RUN mkdir /mnt/data
+RUN mkdir /heatmaps
 
 # Set environment variables.
 ENV HOME /root
diff --git a/convert_heatmaps.js b/convert_heatmaps.js
index 7791df6..710a6b8 100644
--- a/convert_heatmaps.js
+++ b/convert_heatmaps.js
@@ -1,5 +1,17 @@
-//convert.js
-//
+//--------------------------------------------------------------
+// Name: convert_heatmaps.js
+// Description: Given the parameters this script converts the
+//              old heatmap format for loading into a 3.x or >
+//              pathdb quip instance. Parameters have been error
+//              checked by this point and sill be correct.
+// Author(s):  Ryan Birmingham, Joseph Balsamo
+//--------------------------------------------------------------
+
+
+//--------------------------------------------------------------
+// Constants and Variables 
+//--------------------------------------------------------------
+
 const http = require('http');
 const readline = require('readline');
 const path = require('path')
@@ -12,6 +24,12 @@ var clio = {};
 var slide_id = "";
 var dummy = argv.shift();
 var dummy = argv.shift();
+var inputFolder = '/mnt/data/xfer/input';
+var outputFolder = '/mnt/data/xfer/output';
+
+//--------------------------------------------------------------
+// Main Program
+//--------------------------------------------------------------
 
 // Parse command-line args
 while(argv.length > 0) {
@@ -44,8 +62,6 @@ while(argv.length > 0) {
   }
 }
 
-var inputFolder = '/mnt/data/xfer/input';
-var outputFolder = '/mnt/data/xfer/output';
 inputFolder = !clio.input ? '/mnt/data/xfer/input':clio.input;
 outputFolder = !clio.output ? '/mnt/data/xfer/output':clio.output;
 
@@ -56,6 +72,7 @@ const fileTemps = {};
 var manifest = [];
 const mfData = []
 manifest = fs.readFileSync(inputFolder + '/' + clio.manifest).toString().split('\n');
+
 // Exit with error if manifest file is empty
 if (manifest.length <= 1) {
   console.error("Error: Empty manifest file");
@@ -79,9 +96,26 @@ mfData.forEach(mfItem => {
   convert(mfItem.file,mfItem);
 });
 
-
+// Exit with a normal completion.
+process.exit(0);
+
+//--------------------------------------------------------------
+// End of Main Program
+//--------------------------------------------------------------
+
+//--------------------------------------------------------------
+// Function Declarations
+//--------------------------------------------------------------
+
+//--------------------------------------------------------------
+// Function: convert
+// Description: This reads in the data from all heatmap files of
+//              a prediction run and creates a 3.x style import
+//              file to be loaded by the calling script.
+// Parameters: filename:string, metadata: object
+// Returns: Undefined
+//--------------------------------------------------------------
 function convert(filename,metadata){
-
   let lineno = 0;
   let data = [];
   let size = {};
@@ -89,6 +123,7 @@ function convert(filename,metadata){
   let fields = [];
   let ranges = [0,1];
 
+  // Check to verify the file exists
   try {
     fs.accessSync(filename,fs.F_OK);
   } catch(e) {
@@ -130,7 +165,8 @@ function convert(filename,metadata){
         Authorization: 'Basic ' + Buffer.from(clio.username + ':' + clio.passw).toString('base64')
       }   
     };
-    //this is the call
+
+    // this is the call to retrieve the slides unique identifier from pathDB
     request = http.get(options, function(res){
       var body = "";
       res.on('data', function(data) {
@@ -139,7 +175,14 @@ function convert(filename,metadata){
       res.on('end', function() {
         let result = JSON.parse(body);
         let basename = path.basename(filename);
+        // Check if no results are returned.
+        if(result == []) {
+          console.log('Error: No data for ' + image_id);
+          process.exit(50);
+        }
+        // Set slide_id for the given parameters above.
         slide_id = result[0].nid[0].value;
+        // Make the new heatmap JSON Doc
         const content = generateDoc(data,filename,metadata);
         if(!fs.existsSync(outputFolder)) fs.mkdirSync(outputFolder);
         fs.writeFile(`${outputFolder}/NEW_${basename}`, content, function(err) {
@@ -159,6 +202,13 @@ function convert(filename,metadata){
   });
 }
 
+//--------------------------------------------------------------
+// Function: generateDoc
+// Description: This takes the data from convert and produces a
+//              json document to write out to the import file.
+// Parameters: pdata:array of objects,filename:string, metadata: object
+// Returns: JSON Document for heatmap
+//--------------------------------------------------------------
 function generateDoc(pdata,filename,metadata){
   const [x,y,x1,y1] = fileTemps[filename].bbox;
   const width = x1 - x;
@@ -199,3 +249,7 @@ function generateDoc(pdata,filename,metadata){
     "data":${JSON.stringify(pdata)}
   }`;
 }
+
+//--------------------------------------------------------------
+// End of Function Declarations
+//--------------------------------------------------------------
diff --git a/uploadHeatmaps.sh b/uploadHeatmaps.sh
index 7342983..25834f6 100755
--- a/uploadHeatmaps.sh
+++ b/uploadHeatmaps.sh
@@ -221,10 +221,18 @@ fi
 
 # Convert heatmap data in the 'in' folder into uploadable json in the 'out' folder.
 node --max_old_space_size=16384 /usr/local/bin/convert_heatmaps.js -h ${qhost} -c ${collection} -m ${manifest} -i ${in} -o ${out} -u ${uname} -p ${passw}
+exitStatus=$?
+# Check to see conversion process succeded.
+if [[ $exitStatus == 0 ]]
+then
+  # Import into the quip database
+  for filename in ${out}/*.json ; do
+    mongoimport --port ${port} --host ${host} -d ${database} -c heatmap ${filename}
+  done
+else
+  rm -f ${out}/*
+  exit $exitStatus
+fi
 
-# Import into the quip database
-for filename in ${out}/*.json ; do
-  mongoimport --port ${port} --host ${host} -d ${database} -c heatmap ${filename}
-done
-
+# exit normally
 exit 0
\ No newline at end of file

From e627152b9f888884cd7b82f44a8c5f355f0ee087 Mon Sep 17 00:00:00 2001
From: Joseph Balsamo <joseph.balsamo@stonybrook.edu>
Date: Tue, 30 Jul 2019 14:04:02 -0400
Subject: [PATCH 09/16] fixed help usage

---
 uploadHeatmaps.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/uploadHeatmaps.sh b/uploadHeatmaps.sh
index 25834f6..554314f 100755
--- a/uploadHeatmaps.sh
+++ b/uploadHeatmaps.sh
@@ -35,6 +35,7 @@ function usage() {
       echo "    -o, --output <output_folder>: Folder where converted heatmaps are imported from (default: /mnt/data/xfer/output)"
       echo "    -q, --quip-host <host>: ip or hostname of PathDB Server (default: quip-pathdb)"
       echo "    -h, --data-host <host>: ip or hostname of database (default: ca-mongo)"
+      echo "    -m, --manifest <manifest name> (default: manifest.csv)"
       echo "    -d, --database <database name> (default: camic)"
       echo "    -p, --port <database port> (default: 27017)"
       echo ""

From bdd9fe6771ebcdf0ba6b51aa99206368d0663f3a Mon Sep 17 00:00:00 2001
From: Joseph Balsamo <joseph.balsamo@stonybrook.edu>
Date: Tue, 30 Jul 2019 14:14:18 -0400
Subject: [PATCH 10/16] added echo after pass

---
 uploadHeatmaps.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/uploadHeatmaps.sh b/uploadHeatmaps.sh
index 554314f..01004cb 100755
--- a/uploadHeatmaps.sh
+++ b/uploadHeatmaps.sh
@@ -162,6 +162,7 @@ done
 # Request username and password for upload.
 uname="$(getPrompt 'Username:')"
 passw="$(getPass 'Password:')"
+echo 
 
 if [ -z "${collection}" ]  
 then

From 15bca460b8a83698369f2c3d73cbd64b2eed49df Mon Sep 17 00:00:00 2001
From: Joseph Balsamo <joseph.balsamo@stonybrook.edu>
Date: Tue, 30 Jul 2019 16:57:42 -0400
Subject: [PATCH 11/16] Working copy

---
 convert_heatmaps.js | 12 +++++++++---
 uploadHeatmaps.sh   | 26 ++++++++++++++++++--------
 2 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/convert_heatmaps.js b/convert_heatmaps.js
index 710a6b8..b1fc73b 100644
--- a/convert_heatmaps.js
+++ b/convert_heatmaps.js
@@ -31,6 +31,8 @@ var outputFolder = '/mnt/data/xfer/output';
 // Main Program
 //--------------------------------------------------------------
 
+console.log('Beginning conversion');
+
 // Parse command-line args
 while(argv.length > 0) {
   element = argv.shift().trim();
@@ -73,6 +75,8 @@ var manifest = [];
 const mfData = []
 manifest = fs.readFileSync(inputFolder + '/' + clio.manifest).toString().split('\n');
 
+console.log('Reading manifest file.');
+
 // Exit with error if manifest file is empty
 if (manifest.length <= 1) {
   console.error("Error: Empty manifest file");
@@ -97,7 +101,7 @@ mfData.forEach(mfItem => {
 });
 
 // Exit with a normal completion.
-process.exit(0);
+// process.exit(0);
 
 //--------------------------------------------------------------
 // End of Main Program
@@ -130,6 +134,9 @@ function convert(filename,metadata){
     console.log(e.message);
     process.exit(51);
   }
+
+  console.log('Reading file');
+
   // read file
   const myInterface = readline.createInterface({
     input: fs.createReadStream(filename)
@@ -196,9 +203,8 @@ function convert(filename,metadata){
       res.on('error', function(e) {
           console.log("Got error: " + e.message);
       });
-      console.log('Get Ended');
     });
-
+    console.log('Get Ended');
   });
 }
 
diff --git a/uploadHeatmaps.sh b/uploadHeatmaps.sh
index 01004cb..0dd0fef 100755
--- a/uploadHeatmaps.sh
+++ b/uploadHeatmaps.sh
@@ -159,11 +159,6 @@ do
 done
 # End of While
 
-# Request username and password for upload.
-uname="$(getPrompt 'Username:')"
-passw="$(getPass 'Password:')"
-echo 
-
 if [ -z "${collection}" ]  
 then
   echo "Missing required parameter"
@@ -202,17 +197,30 @@ then
 fi
 
 # Check that input and output folders exist.  
-if [ ! -f $in ]
+if [ ! -d $in ]
 then
   echo "Error: Input folder does not exist."
   exit 2
 fi
-if [ ! -f $out ]
+if [ ! -d $out ]
 then
   echo "Error: Output folder does not exist."
   exit 3
 fi
 
+# Check that manifest file exists
+if [ ! -f "${in}/${manifest}" ]
+then
+  echo "Error: Manifest does not exist."
+  exit 2
+fi
+
+# Request username and password for upload.
+uname="$(getPrompt 'Username:')"
+passw="$(getPass 'Password:')"
+echo 
+echo
+
 # Verify that PathDB Server is reachable using current username/password combo
 ret_code="$(curl -Is -u ${uname}:${passw} http://${qhost}/ | head -1  | awk '{ print $2 }')"
 if [ ! $ret_code -lt 400 ]
@@ -224,10 +232,12 @@ fi
 # Convert heatmap data in the 'in' folder into uploadable json in the 'out' folder.
 node --max_old_space_size=16384 /usr/local/bin/convert_heatmaps.js -h ${qhost} -c ${collection} -m ${manifest} -i ${in} -o ${out} -u ${uname} -p ${passw}
 exitStatus=$?
+console.log($exitStatus);
 # Check to see conversion process succeded.
-if [[ $exitStatus == 0 ]]
+if [[ $exitStatus -eq 0 ]]
 then
   # Import into the quip database
+  console.log('Importing data')
   for filename in ${out}/*.json ; do
     mongoimport --port ${port} --host ${host} -d ${database} -c heatmap ${filename}
   done

From 995e888d1799097f34a65f388980ff78e96ffe42 Mon Sep 17 00:00:00 2001
From: Joseph Balsamo <joseph.balsamo@stonybrook.edu>
Date: Wed, 31 Jul 2019 14:28:38 -0400
Subject: [PATCH 12/16] Cleaned up Console.logs

---
 uploadHeatmaps.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/uploadHeatmaps.sh b/uploadHeatmaps.sh
index 0dd0fef..29e158c 100755
--- a/uploadHeatmaps.sh
+++ b/uploadHeatmaps.sh
@@ -232,8 +232,7 @@ fi
 # Convert heatmap data in the 'in' folder into uploadable json in the 'out' folder.
 node --max_old_space_size=16384 /usr/local/bin/convert_heatmaps.js -h ${qhost} -c ${collection} -m ${manifest} -i ${in} -o ${out} -u ${uname} -p ${passw}
 exitStatus=$?
-console.log($exitStatus);
-# Check to see conversion process succeded.
+# Check to see conversion process succeeded.
 if [[ $exitStatus -eq 0 ]]
 then
   # Import into the quip database

From 1ec8d3e0e44b54a1cf158a4b03a724877dc82567 Mon Sep 17 00:00:00 2001
From: Joseph Balsamo <joseph.balsamo@stonybrook.edu>
Date: Fri, 2 Aug 2019 12:56:29 -0400
Subject: [PATCH 13/16] Clean logs

---
 convert_heatmaps.js | 4 +---
 uploadHeatmaps.sh   | 3 +--
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/convert_heatmaps.js b/convert_heatmaps.js
index b1fc73b..bcbe19f 100644
--- a/convert_heatmaps.js
+++ b/convert_heatmaps.js
@@ -135,8 +135,6 @@ function convert(filename,metadata){
     process.exit(51);
   }
 
-  console.log('Reading file');
-
   // read file
   const myInterface = readline.createInterface({
     input: fs.createReadStream(filename)
@@ -183,7 +181,7 @@ function convert(filename,metadata){
         let result = JSON.parse(body);
         let basename = path.basename(filename);
         // Check if no results are returned.
-        if(result == []) {
+        if(result == [] || !result) {
           console.log('Error: No data for ' + image_id);
           process.exit(50);
         }
diff --git a/uploadHeatmaps.sh b/uploadHeatmaps.sh
index 29e158c..481445b 100755
--- a/uploadHeatmaps.sh
+++ b/uploadHeatmaps.sh
@@ -222,7 +222,7 @@ echo
 echo
 
 # Verify that PathDB Server is reachable using current username/password combo
-ret_code="$(curl -Is -u ${uname}:${passw} http://${qhost}/ | head -1  | awk '{ print $2 }')"
+ret_code="$(curl -Is http://${qhost}/ | head -1  | awk '{ print $2 }')"
 if [ ! $ret_code -lt 400 ]
 then
   echo "Error: PathDB Server is unreachable."
@@ -236,7 +236,6 @@ exitStatus=$?
 if [[ $exitStatus -eq 0 ]]
 then
   # Import into the quip database
-  console.log('Importing data')
   for filename in ${out}/*.json ; do
     mongoimport --port ${port} --host ${host} -d ${database} -c heatmap ${filename}
   done

From b7c68ac9274ab5b1085bafda0dabe5faa331c8e9 Mon Sep 17 00:00:00 2001
From: Joseph Balsamo <joseph.balsamo@stonybrook.edu>
Date: Fri, 2 Aug 2019 16:51:47 -0400
Subject: [PATCH 14/16] Cleaned error messages and fixed i/o folders

---
 Dockerfile          |  1 -
 convert_heatmaps.js | 16 ++++++++--------
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index ef53903..1a1ea15 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -40,7 +40,6 @@ COPY helpers.sh /usr/local/bin/
 COPY readpass.sh /usr/local/bin/
 RUN chmod 0775 /usr/local/bin/uploadHeatmaps.sh
 RUN mkdir /mnt/data
-RUN mkdir /heatmaps
 
 # Set environment variables.
 ENV HOME /root
diff --git a/convert_heatmaps.js b/convert_heatmaps.js
index bcbe19f..770697f 100644
--- a/convert_heatmaps.js
+++ b/convert_heatmaps.js
@@ -24,8 +24,8 @@ var clio = {};
 var slide_id = "";
 var dummy = argv.shift();
 var dummy = argv.shift();
-var inputFolder = '/mnt/data/xfer/input';
-var outputFolder = '/mnt/data/xfer/output';
+var inputFolder = '/mnt/data/heatmaps/input';
+var outputFolder = '/mnt/data/heatmaps/output';
 
 //--------------------------------------------------------------
 // Main Program
@@ -64,8 +64,8 @@ while(argv.length > 0) {
   }
 }
 
-inputFolder = !clio.input ? '/mnt/data/xfer/input':clio.input;
-outputFolder = !clio.output ? '/mnt/data/xfer/output':clio.output;
+inputFolder = !clio.input ? '/mnt/data/heatmaps/input':'/mnt/data/heatmaps/' + clio.input;
+outputFolder = !clio.output ? '/mnt/data/heatmaps/output':'/mnt/data/heatmaps/' + clio.output;
 
 url = clio.host;
 
@@ -131,7 +131,7 @@ function convert(filename,metadata){
   try {
     fs.accessSync(filename,fs.F_OK);
   } catch(e) {
-    console.log(e.message);
+    console.error(e.message);
     process.exit(51);
   }
 
@@ -182,7 +182,7 @@ function convert(filename,metadata){
         let basename = path.basename(filename);
         // Check if no results are returned.
         if(result == [] || !result) {
-          console.log('Error: No data for ' + image_id);
+          console.error('Error: No data for ' + image_id);
           process.exit(50);
         }
         // Set slide_id for the given parameters above.
@@ -199,10 +199,10 @@ function convert(filename,metadata){
         });
       });
       res.on('error', function(e) {
-          console.log("Got error: " + e.message);
+          console.error("Error: " + e.message);
       });
     });
-    console.log('Get Ended');
+    console.log('Conversion Completed.');
   });
 }
 

From e0a75b394448bfa712ca24b6bc10762746cf92c8 Mon Sep 17 00:00:00 2001
From: Joseph Balsamo <joseph.balsamo@stonybrook.edu>
Date: Thu, 8 Aug 2019 13:38:03 -0400
Subject: [PATCH 15/16] fixed import locations and updated docs

---
 README.md         | 22 ++++++++++------------
 uploadHeatmaps.sh | 10 +++++-----
 2 files changed, 15 insertions(+), 17 deletions(-)

diff --git a/README.md b/README.md
index d763557..f0fa2bb 100644
--- a/README.md
+++ b/README.md
@@ -11,24 +11,23 @@ Under the input directory "_import folder_", with the heatmap files, you need to
 To load the current input files with a manifest named "manifest.csv":
 
 ```bash
-[ root@b93a23bb7fa1:~ ]$ uploadHeatmaps.sh -c <PathDB Collection> -u <user_name> -p <password>
+[ root@b93a23bb7fa1:~ ]$ uploadHeatmaps.sh -c <PathDB Collection> [option]
 ```
 
 For the full usage info type:
 
 ```bash
 bash-3.2$ ~/uploadHeatmaps/uploadHeatmaps.sh --help
-Usage: $ ./uploadHeatmaps.sh [options] -c <pathDB_collection> -u <username> -p <password>
+Usage: $ uploadHeatmaps.sh [options] -c <pathDB_collection>
   Options:
-    -c <pathDB_collection>: PathDB Collection for heatmaps being loaded (this parameter required)
-    -u <username>: PathDB username (this parameter required)
-    -p <password>: PathDB password (this parameter required)
-    -i <input_folder>: Folder where heatmaps are loaded from (default: /mnt/data/xfer/input)
-    -o <output_folder>: Folder where converted heatmaps are imported from (default: /mnt/data/xfer/output)
-    -q <host>: ip or hostname of PathDB Server (default: quip-pathdb)
-    -h <host>: ip or hostname of database (default: ca-mongo)
-    -d <database name> (default: camic)
-    -P <database port> (default: 27017)
+    -c, --collection <pathDB_collection>: PathDB Collection for heatmaps (*this parameter required)
+    -i, --input <input_folder>: Folder where heatmaps are loaded from (default: /mnt/data/xfer/input)
+    -o, --output <output_folder>: Folder where converted heatmaps are imported from (default: /mnt/data/xfer/output)
+    -q, --quip-host <host>: ip or hostname of PathDB Server (default: quip-pathdb)
+    -h, --data-host <host>: ip or hostname of database (default: ca-mongo)
+    -m, --manifest <manifest name> (default: manifest.csv)
+    -d, --database <database name> (default: camic)
+    -p, --port <database port> (default: 27017)
 
     --help Display full help usage.
   Notes: requires mongoDB client tools installed on running server
@@ -36,4 +35,3 @@ Usage: $ ./uploadHeatmaps.sh [options] -c <pathDB_collection> -u <username> -p <
 
 Depending on the number of images and their resolution it may take some time to complete.  Do not stop the process until it completes.
 
-## Container Version
diff --git a/uploadHeatmaps.sh b/uploadHeatmaps.sh
index 481445b..f84aaa3 100755
--- a/uploadHeatmaps.sh
+++ b/uploadHeatmaps.sh
@@ -181,11 +181,11 @@ then
 fi
 if [ -z "${in}" ]
 then
-  in="/mnt/data/xfer/input"
+  in="/mnt/data/heatmaps/input"
 fi
 if [ -z "${out}" ]
 then
-  out="/mnt/data/xfer/output"
+  out="/mnt/data/heatmaps/output"
 fi
 if [ -z "${port}" ]
 then
@@ -197,19 +197,19 @@ then
 fi
 
 # Check that input and output folders exist.  
-if [ ! -d $in ]
+if [ ! -d "/mnt/data/heatmaps/${in}" ]
 then
   echo "Error: Input folder does not exist."
   exit 2
 fi
-if [ ! -d $out ]
+if [ ! -d "/mnt/data/heatmaps/${out}" ]
 then
   echo "Error: Output folder does not exist."
   exit 3
 fi
 
 # Check that manifest file exists
-if [ ! -f "${in}/${manifest}" ]
+if [ ! -f "/mnt/data/heatmaps/${in}/${manifest}" ]
 then
   echo "Error: Manifest does not exist."
   exit 2

From b07068a6dc25d7bbf7bba8df2a854627376e0be4 Mon Sep 17 00:00:00 2001
From: Joseph Balsamo <joseph.balsamo@stonybrook.edu>
Date: Tue, 13 Aug 2019 11:34:38 -0400
Subject: [PATCH 16/16] Updated Docs

---
 README.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index f0fa2bb..aed0faa 100644
--- a/README.md
+++ b/README.md
@@ -2,9 +2,9 @@
 
 ## File Preparation
 
-Start the import process by storing the data in an import folder.  The directory should be: _quip folder_/_data_/_xfer_/_input folder_/ on the quip host server.  You only need to copy the "heatmap_XXXXXXXXXXXXXXXXXX.json" files for import.
+Start the import process by storing the data in an import folder.  The directory should be: _quip folder_/_data_/_heatmaps_/_input folder_/ on the quip host server.  You only need to copy the "heatmap_XXXXXXXXXXXXXXXXXX.json" files for import.  The metadata file is unnecessary because the manifest contains the required data.
 
-Under the input directory "_import folder_", with the heatmap files, you need to also add a manifest file that contains data about the heatmaps you are loading.
+Under the input directory "_import folder_", with the heatmap files, you need to also add a manifest file that contains data about the heatmaps you are loading.  The heatmap files and be distributed under multiple directories in the "_import folder_" as long as the path is defined in manifest with a relative path (i.e. "./study1/heatmap01.json").
 
 ## Usage
 
@@ -21,8 +21,8 @@ bash-3.2$ ~/uploadHeatmaps/uploadHeatmaps.sh --help
 Usage: $ uploadHeatmaps.sh [options] -c <pathDB_collection>
   Options:
     -c, --collection <pathDB_collection>: PathDB Collection for heatmaps (*this parameter required)
-    -i, --input <input_folder>: Folder where heatmaps are loaded from (default: /mnt/data/xfer/input)
-    -o, --output <output_folder>: Folder where converted heatmaps are imported from (default: /mnt/data/xfer/output)
+    -i, --input <input_folder>: Folder where heatmaps are loaded from (default: input)
+    -o, --output <output_folder>: Folder where converted heatmaps are imported from (default: output)
     -q, --quip-host <host>: ip or hostname of PathDB Server (default: quip-pathdb)
     -h, --data-host <host>: ip or hostname of database (default: ca-mongo)
     -m, --manifest <manifest name> (default: manifest.csv)
@@ -33,5 +33,5 @@ Usage: $ uploadHeatmaps.sh [options] -c <pathDB_collection>
   Notes: requires mongoDB client tools installed on running server
 ```
 
-Depending on the number of images and their resolution it may take some time to complete.  Do not stop the process until it completes.
+Depending on the number of images and their resolution it may take some time to complete.  Do not stop the process until it completes.  If an error occurs then no data will be uploaded.