Merge pull request #2 from SBU-BMI/develop

Develop

.gitignore

@@ -1 +1,2 @@
 .vscode/settings.json
+quip.code-workspace

Dockerfile

@@ -36,6 +36,8 @@ ADD root/.scripts /root/.scripts
 
 COPY convert_heatmaps.js /usr/local/bin/
 COPY uploadHeatmaps.sh /usr/local/bin/
+COPY helpers.sh /usr/local/bin/
+COPY readpass.sh /usr/local/bin/
 RUN chmod 0775 /usr/local/bin/uploadHeatmaps.sh
 RUN mkdir /mnt/data
 

README.md

@@ -2,36 +2,35 @@
 
 ## File Preparation
 
-Start the import process by storing the data in an import folder.  The directory should be: _quip folder_/_data_/_xfer_/_input folder_/ on the quip host server.  You only need to copy the "heatmap_XXXXXXXXXXXXXXXXXX.json" files for import.
+Start the import process by storing the data in an import folder.  The directory should be: _quip folder_/_data_/_heatmaps_/_input folder_/ on the quip host server.  You only need to copy the "heatmap_XXXXXXXXXXXXXXXXXX.json" files for import.  The metadata file is unnecessary because the manifest contains the required data.
 
-Under the input directory "_import folder_", with the heatmap files, you need to also add a manifest file that contains data about the heatmaps you are loading.
+Under the input directory "_import folder_", with the heatmap files, you need to also add a manifest file that contains data about the heatmaps you are loading.  The heatmap files and be distributed under multiple directories in the "_import folder_" as long as the path is defined in manifest with a relative path (i.e. "./study1/heatmap01.json").
 
 ## Usage
 
 To load the current input files with a manifest named "manifest.csv":
 
 ```bash
-[ root@b93a23bb7fa1:~ ]$ uploadHeatmaps.sh -c <PathDB Collection> -u <user_name> -p <password>
+[ root@b93a23bb7fa1:~ ]$ uploadHeatmaps.sh -c <PathDB Collection> [option]
 ```
 
 For the full usage info type:
 
 ```bash
 bash-3.2$ ~/uploadHeatmaps/uploadHeatmaps.sh --help
-Usage: $ ./uploadHeatmaps.sh [options] -c <pathDB_collection> -u <username> -p <password>
+Usage: $ uploadHeatmaps.sh [options] -c <pathDB_collection>
   Options:
-    -c <pathDB_collection>: PathDB Collection for heatmaps being loaded (this parameter required)
-    -u <username>: PathDB username (this parameter required)
-    -p <password>: PathDB password (this parameter required)
-    -i <input_folder>: Folder where heatmaps are loaded from (default: /mnt/data/xfer/input)
-    -o <output_folder>: Folder where converted heatmaps are imported from (default: /mnt/data/xfer/output)
-    -q <host>: ip or hostname of PathDB Server (default: quip-pathdb)
-    -h <host>: ip or hostname of database (default: ca-mongo)
-    -d <database name> (default: camic)
-    -P <database port> (default: 27017)
+    -c, --collection <pathDB_collection>: PathDB Collection for heatmaps (*this parameter required)
+    -i, --input <input_folder>: Folder where heatmaps are loaded from (default: input)
+    -o, --output <output_folder>: Folder where converted heatmaps are imported from (default: output)
+    -q, --quip-host <host>: ip or hostname of PathDB Server (default: quip-pathdb)
+    -h, --data-host <host>: ip or hostname of database (default: ca-mongo)
+    -m, --manifest <manifest name> (default: manifest.csv)
+    -d, --database <database name> (default: camic)
+    -p, --port <database port> (default: 27017)
 
     --help Display full help usage.
   Notes: requires mongoDB client tools installed on running server
 ```
 
-Depending on the number of images and their resolution it may take some time to complete.  Do not stop the process until it completes.
+Depending on the number of images and their resolution it may take some time to complete.  Do not stop the process until it completes.  If an error occurs then no data will be uploaded.

convert_heatmaps.js

@@ -1,5 +1,17 @@
-//convert.js
-//
+//--------------------------------------------------------------
+// Name: convert_heatmaps.js
+// Description: Given the parameters this script converts the
+//              old heatmap format for loading into a 3.x or >
+//              pathdb quip instance. Parameters have been error
+//              checked by this point and sill be correct.
+// Author(s):  Ryan Birmingham, Joseph Balsamo
+//--------------------------------------------------------------
+
+
+//--------------------------------------------------------------
+// Constants and Variables 
+//--------------------------------------------------------------
+
 const http = require('http');
 const readline = require('readline');
 const path = require('path')
@@ -12,6 +24,14 @@ var clio = {};
 var slide_id = "";
 var dummy = argv.shift();
 var dummy = argv.shift();
+var inputFolder = '/mnt/data/heatmaps/input';
+var outputFolder = '/mnt/data/heatmaps/output';
+
+//--------------------------------------------------------------
+// Main Program
+//--------------------------------------------------------------
+
+console.log('Beginning conversion');
 
 // Parse command-line args
 while(argv.length > 0) {
@@ -44,10 +64,8 @@ while(argv.length > 0) {
   }
 }
 
-var inputFolder = '/mnt/data/xfer/input';
-var outputFolder = '/mnt/data/xfer/output';
-inputFolder = !clio.input ? '/mnt/data/xfer/input':clio.input;
-outputFolder = !clio.output ? '/mnt/data/xfer/output':clio.output;
+inputFolder = !clio.input ? '/mnt/data/heatmaps/input':'/mnt/data/heatmaps/' + clio.input;
+outputFolder = !clio.output ? '/mnt/data/heatmaps/output':'/mnt/data/heatmaps/' + clio.output;
 
 url = clio.host;
 
@@ -56,11 +74,19 @@ const fileTemps = {};
 var manifest = [];
 const mfData = []
 manifest = fs.readFileSync(inputFolder + '/' + clio.manifest).toString().split('\n');
+
+console.log('Reading manifest file.');
+
+// Exit with error if manifest file is empty
+if (manifest.length <= 1) {
+  console.error("Error: Empty manifest file");
+  process.exit(50);
+}
 manifest.splice(0,1);
 manifest.forEach((line)=>{
   if(line != '') {
     let fileInfo = line.split(',');
-    mfData.push({ file: fileInfo[0],study_id:fileInfo[1],subject_id:fileInfo[2],image_id:fileInfo[3] });
+    mfData.push({ file: path.resolve(inputFolder + '/' +fileInfo[0]),study_id:fileInfo[1],subject_id:fileInfo[2],image_id:fileInfo[3] });
   }
 });
 
@@ -74,19 +100,44 @@ mfData.forEach(mfItem => {
   convert(mfItem.file,mfItem);
 });
 
+// Exit with a normal completion.
+// process.exit(0);
 
-function convert(filename,metadata){
+//--------------------------------------------------------------
+// End of Main Program
+//--------------------------------------------------------------
 
+//--------------------------------------------------------------
+// Function Declarations
+//--------------------------------------------------------------
+
+//--------------------------------------------------------------
+// Function: convert
+// Description: This reads in the data from all heatmap files of
+//              a prediction run and creates a 3.x style import
+//              file to be loaded by the calling script.
+// Parameters: filename:string, metadata: object
+// Returns: Undefined
+//--------------------------------------------------------------
+function convert(filename,metadata){
   let lineno = 0;
   let data = [];
   let size = {};
 
   let fields = [];
   let ranges = [0,1];
 
+  // Check to verify the file exists
+  try {
+    fs.accessSync(filename,fs.F_OK);
+  } catch(e) {
+    console.error(e.message);
+    process.exit(51);
+  }
+
   // read file
   const myInterface = readline.createInterface({
-    input: fs.createReadStream(`${inputFolder}/${filename}`)
+    input: fs.createReadStream(filename)
   });
 
   remainder++;
@@ -119,18 +170,27 @@ function convert(filename,metadata){
         Authorization: 'Basic ' + Buffer.from(clio.username + ':' + clio.passw).toString('base64')
       }   
     };
-    //this is the call
+
+    // this is the call to retrieve the slides unique identifier from pathDB
     request = http.get(options, function(res){
       var body = "";
       res.on('data', function(data) {
           body += data;
       });
       res.on('end', function() {
         let result = JSON.parse(body);
+        let basename = path.basename(filename);
+        // Check if no results are returned.
+        if(result == [] || !result) {
+          console.error('Error: No data for ' + image_id);
+          process.exit(50);
+        }
+        // Set slide_id for the given parameters above.
         slide_id = result[0].nid[0].value;
+        // Make the new heatmap JSON Doc
         const content = generateDoc(data,filename,metadata);
         if(!fs.existsSync(outputFolder)) fs.mkdirSync(outputFolder);
-        fs.writeFile(`${outputFolder}/NEW_${filename}`, content, function(err) {
+        fs.writeFile(`${outputFolder}/NEW_${basename}`, content, function(err) {
           if (err) throw err;
           remainder--;
           console.log(`${filename} completed`);
@@ -139,14 +199,20 @@ function convert(filename,metadata){
         });
       });
       res.on('error', function(e) {
-          console.log("Got error: " + e.message);
+          console.error("Error: " + e.message);
       });
-      console.log('Get Ended');
     });
-
+    console.log('Conversion Completed.');
   });
 }
 
+//--------------------------------------------------------------
+// Function: generateDoc
+// Description: This takes the data from convert and produces a
+//              json document to write out to the import file.
+// Parameters: pdata:array of objects,filename:string, metadata: object
+// Returns: JSON Document for heatmap
+//--------------------------------------------------------------
 function generateDoc(pdata,filename,metadata){
   const [x,y,x1,y1] = fileTemps[filename].bbox;
   const width = x1 - x;
@@ -187,3 +253,7 @@ function generateDoc(pdata,filename,metadata){
     "data":${JSON.stringify(pdata)}
   }`;
 }
+
+//--------------------------------------------------------------
+// End of Function Declarations
+//--------------------------------------------------------------

helpers.sh

@@ -0,0 +1,30 @@
+#---------------------------------------------------
+# File: Helper Function Library
+#---------------------------------------------------
+
+#---------------------------------------------------
+# Function: chk_opt 
+# Description: Takes an argument and determines if it 
+#              is a flag instead of an parameter value.
+# parameters: cl argument
+# usage: opterr="$(chk_opt ${2})"
+#---------------------------------------------------
+function chk_opt() {
+  # Check the first char of the option passed 
+  checkc="$(echo $1 | head -c 1)"
+
+  if [ $checkc == "-" ]
+  then
+    echo 'true'
+  else
+    echo 'false'
+  fi
+}
+#-----------------------------------------------------------------------
+# End chk_opt
+#-----------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------
+# end functions
+#-----------------------------------------------------------------------

readpass.sh

@@ -0,0 +1,64 @@
+#---------------------------------------------------
+# File: ReadPass Function Library
+#---------------------------------------------------
+
+#---------------------------------------------------
+# Function: getPrompt
+# Description: Read a line of input hile echoing on
+#              the screen. 
+# parameters: prompt
+# usage: myvar="$(getPrompt 'prompt')"
+#---------------------------------------------------
+function getPrompt() {
+    prompt="$1 "
+    read -p "$prompt" input
+    echo "$input"
+}
+#---------------------------------------------------
+# End getPrompt
+#---------------------------------------------------
+
+#---------------------------------------------------
+# Function: getPass
+# Description: Read a line of input while showing '*'
+#              on the screen. 
+# parameters: prompt
+# usage: myPW="$(getPass 'prompt')"
+#---------------------------------------------------
+function getPass() {
+    prompt="$1 "
+    # Loop to get all characters of the password
+    while IFS= read -p "$prompt" -r -s -n 1 char 
+    do
+        # Check if the return key was pressed
+        if [[ $char == $'\0' ]];     then
+            break
+        fi
+        # Check to see if backspace was pressed
+        if [[ $char == $'\177' ]];  then
+            l=${#password} # length of password
+            # if l is > 0 then delete last char
+            # else do not remove any character
+            if [[ l -gt 0 ]]
+            then
+                prompt=$'\b \b'
+                password="${password%?}"
+            else
+                prompt=$''
+            fi
+        else
+            # Add new char to password
+            prompt='*'
+            password+="$char"
+        fi
+    done
+    # Echo out password to be able to capture
+    echo "$password"
+}
+#---------------------------------------------------
+# End getPass
+#---------------------------------------------------
+
+#---------------------------------------------------
+# End Function Library
+#---------------------------------------------------