Merge pull request #121 from SumoLogic/hpal_security_fix

Updated dependencies

.github/workflows/arm-template-test.yml

@@ -53,7 +53,7 @@ jobs:
       - uses: bridgecrewio/checkov-action@master
         with:
           file: ${{ matrix.dir }}/${{ matrix.file }}
-          skip_check: CKV_AZURE_16,CKV_AZURE_17,CKV_AZURE_35,CKV_AZURE_78
+          skip_check: CKV_AZURE_16,CKV_AZURE_17,CKV_AZURE_35,CKV_AZURE_78,CKV_AZURE_43,CKV_AZURE_206,CKV_AZURE_225,CKV_AZURE_222,CKV_AZURE_213,CKV_AZURE_212
           quiet: false
           framework: arm
           output_format: cli

AppendBlobReader/README.md

@@ -31,7 +31,7 @@ Make all the code changes in `AppendBlobReader/src` directory, once all the chan
 
 This command copies required files in `AppendBlobReader/target` directory
 
-### Run Unit Test
+### Run Integrations Test
 
 Integrations tests are in `AppendBlobReader/tests` folder and unit tests are in sumo-`function-utils/tests` folder
 
@@ -50,3 +50,23 @@ export SUMO_DEPLOYMENT=`Enter one of the allowed values au, ca, de, eu, fed, in,
 Execute below command under `AppendBlobReader/tests` directory
 
 `python test_appendblobreader.py`
+
+### Run Unit Tests
+
+To run unit tests, first install test dependencies and then run the tests using below commands
+
+ `npm install`
+
+ `npm test`
+
+## Security Fixes
+
+  package-lock.json can be created using below command
+
+     npm install --package-lock
+
+  Fix the security dependencies by running below command
+
+     npm audit fix
+
+

AppendBlobReader/src/consumer.js

@@ -264,7 +264,7 @@ async function appendBlobStreamMessageHandlerv2(context, serviceBusTask) {
  * @returns {string} - The new string, truncated if necessary.
  */
 function checkAndTruncate(data) {
-    const maxLength = 128;
+    const maxLength = 1024;
 
     // Check if the string length exceeds the maximum length
     if (data.length > maxLength) {
@@ -297,9 +297,9 @@ function setSourceCategory(serviceBusTask, options) {
         });
         options.metadata["sourceFields"] = customFieldsArr.join();
     }
-    options.metadata["sourceHost"] = `${serviceBusTask.storageName}/${serviceBusTask.containerName}`
+    options.metadata["sourceHost"] = checkAndTruncate(`${serviceBusTask.storageName}/${serviceBusTask.containerName}`);
     // context.log(serviceBusTask.blobName, serviceBusTask.storageName,serviceBusTask.containerName);
-    // options.metadata["sourceCategory"] = "custom_source_category";
+    // options.metadata["sourceCategory"] = checkAndTruncate("custom_source_category");
     options.metadata["sourceName"] = checkAndTruncate(serviceBusTask.blobName);
 }
 

AppendBlobReader/src/decodeDataChunks.js

@@ -24,7 +24,7 @@ function getBoundaryRegex(serviceBusTask) {
     // https://stackoverflow.com/questions/31969913/why-does-this-regexp-exec-cause-an-infinite-loop
     var file_ext = String(serviceBusTask.blobName).split(".").pop();
     if (file_ext === "json" || file_ext === "blob") {
-        logRegex = '\{\\s*\"';
+        logRegex = '{\\s*\"';
     }
     // uncomment and use the snippet below for overriding boundary regex for your log files
     // if (serviceBusTask.storageName === "<your storageAccountName>" || serviceBusTask.containerName === "<your containerName>" ) {

AppendBlobReader/target/consumer_build/AppendBlobTaskConsumer/decodeDataChunks.js

@@ -24,7 +24,7 @@ function getBoundaryRegex(serviceBusTask) {
     // https://stackoverflow.com/questions/31969913/why-does-this-regexp-exec-cause-an-infinite-loop
     var file_ext = String(serviceBusTask.blobName).split(".").pop();
     if (file_ext === "json" || file_ext === "blob") {
-        logRegex = '\{\\s*\"';
+        logRegex = '{\\s*\"';
     }
     // uncomment and use the snippet below for overriding boundary regex for your log files
     // if (serviceBusTask.storageName === "<your storageAccountName>" || serviceBusTask.containerName === "<your containerName>" ) {

AppendBlobReader/target/consumer_build/AppendBlobTaskConsumer/index.js

@@ -264,7 +264,7 @@ async function appendBlobStreamMessageHandlerv2(context, serviceBusTask) {
  * @returns {string} - The new string, truncated if necessary.
  */
 function checkAndTruncate(data) {
-    const maxLength = 128;
+    const maxLength = 1024;
 
     // Check if the string length exceeds the maximum length
     if (data.length > maxLength) {
@@ -297,9 +297,9 @@ function setSourceCategory(serviceBusTask, options) {
         });
         options.metadata["sourceFields"] = customFieldsArr.join();
     }
-    options.metadata["sourceHost"] = `${serviceBusTask.storageName}/${serviceBusTask.containerName}`
+    options.metadata["sourceHost"] = checkAndTruncate(`${serviceBusTask.storageName}/${serviceBusTask.containerName}`);
     // context.log(serviceBusTask.blobName, serviceBusTask.storageName,serviceBusTask.containerName);
-    // options.metadata["sourceCategory"] = "custom_source_category";
+    // options.metadata["sourceCategory"] = checkAndTruncate("custom_source_category");
     options.metadata["sourceName"] = checkAndTruncate(serviceBusTask.blobName);
 }
 

AppendBlobReader/tests/package.json

@@ -1,5 +1,8 @@
 {
     "scripts": {
         "test": "jest"
+    },
+    "devDependencies": {
+        "jest": "^29.7.0"
     }
-}
+}

AppendBlobReader/tests/test_appendblobreader.py

@@ -83,6 +83,7 @@ def upload_file_of_unknown_extension(self):
         self.block_blob_service.append_blob_from_text(self.test_container_name, self.test_filename_unsupported_extension, chunk, encoding='utf-8')
 
     def test_03_func_logs(self):
+        time.sleep(120)  # waiting for language runtime to be available
         self.logger.info("inserting mock data in BlobStorage")
         self.upload_file_chunks_using_append_blobs()
         self.upload_file_in_another_container()
@@ -182,7 +183,9 @@ def test_03_func_logs(self):
                         f"append blob file's record count: {record_unsupported_extension_count}, logs with unsupported blob extension should not be ingested")
 
         # Verify with a very long append blob filename (1024 characters)
-        if len(self.test_filename) > 128:
+        # _sourceCategory, _sourceHost, _sourceName have this limit
+        maxMetadataLength = 1024
+        if len(self.test_filename) > maxMetadataLength:
             expected_filename = self.test_filename[:60] + "..." + self.test_filename[-60:]
         else:
             expected_filename = self.test_filename

BlockBlobReader/README.md

@@ -33,7 +33,7 @@ This command copies required files in `BlockBlobReader/target` directory
 
 Integrations tests are in `BlockBlobReader/tests` folder and unit tests are in `sumo-function-utils/tests` folder
 
-### Run Unit Test
+### Run Integration Tests
 
 Integrations tests are in `BlockBlobReader/tests` folder and unit tests are in sumo-`function-utils/tests` folder
 
@@ -53,3 +53,15 @@ export SUMO_DEPLOYMENT=`Enter one of the allowed values au, ca, de, eu, fed, in,
 Execute below command under `BlockBlobReader/tests` directory
 
 `python test_blobreader.py`
+
+## Security Fixes
+
+  package-lock.json can be created using below command
+
+     npm install --package-lock
+
+  Fix the security dependencies by running below command
+
+     npm audit fix
+
+

BlockBlobReader/src/consumer.js

@@ -141,7 +141,7 @@ function regexLastIndexOf(string, regex, startpos) {
  */
 function getParseableJsonArray(data, context) {
 
-    let logRegex = '\{\\s*\"time\"\:'; // starting regex for nsg logs
+    let logRegex = '{\\s*\"time\"\:'; // starting regex for nsg logs
     let defaultEncoding = "utf8";
     let orginalDatalength = data.length;
     // If the byte sequence in the buffer data is not valid according to the provided encoding, then it is replaced by the default replacement character i.e. U+FFFD.
@@ -414,7 +414,8 @@ function messageHandler(serviceBusTask, context, sumoClient) {
  * @returns {string} - The new string, truncated if necessary.
  */
 function checkAndTruncate(data) {
-    const maxLength = 128;
+
+    const maxLength = 1024;
 
     // Check if the string length exceeds the maximum length
     if (data.length > maxLength) {
@@ -437,6 +438,7 @@ function checkAndTruncate(data) {
  * metadata.sourceCategory attribute sets the source category
  */
 function setSourceCategory(serviceBusTask, options) {
+
     options.metadata = options.metadata || {};
     // make sure to add custom fileds in HTTP source in sumologic portal: https://help.sumologic.com/docs/manage/fields/#collector-and-source-fields, otherwise these fileds will be dropped.
     let customFields = {}; // { "containername": serviceBusTask.containerName, "storagename": serviceBusTask.storageName };
@@ -447,7 +449,7 @@ function setSourceCategory(serviceBusTask, options) {
         });
         options.metadata["sourceFields"] = customFieldsArr.join();
     }
-    options.metadata["sourceHost"] = `${serviceBusTask.storageName}/${serviceBusTask.containerName}`
+    options.metadata["sourceHost"] = checkAndTruncate(`${serviceBusTask.storageName}/${serviceBusTask.containerName}`);
     // context.log(serviceBusTask.blobName, serviceBusTask.storageName,serviceBusTask.containerName);
     // options.metadata["sourceCategory"] = "custom_source_category";
     options.metadata["sourceName"] = checkAndTruncate(serviceBusTask.blobName);

BlockBlobReader/target/consumer_build/BlobTaskConsumer/index.js

@@ -141,7 +141,7 @@ function regexLastIndexOf(string, regex, startpos) {
  */
 function getParseableJsonArray(data, context) {
 
-    let logRegex = '\{\\s*\"time\"\:'; // starting regex for nsg logs
+    let logRegex = '{\\s*\"time\"\:'; // starting regex for nsg logs
     let defaultEncoding = "utf8";
     let orginalDatalength = data.length;
     // If the byte sequence in the buffer data is not valid according to the provided encoding, then it is replaced by the default replacement character i.e. U+FFFD.
@@ -414,7 +414,8 @@ function messageHandler(serviceBusTask, context, sumoClient) {
  * @returns {string} - The new string, truncated if necessary.
  */
 function checkAndTruncate(data) {
-    const maxLength = 128;
+
+    const maxLength = 1024;
 
     // Check if the string length exceeds the maximum length
     if (data.length > maxLength) {
@@ -437,6 +438,7 @@ function checkAndTruncate(data) {
  * metadata.sourceCategory attribute sets the source category
  */
 function setSourceCategory(serviceBusTask, options) {
+
     options.metadata = options.metadata || {};
     // make sure to add custom fileds in HTTP source in sumologic portal: https://help.sumologic.com/docs/manage/fields/#collector-and-source-fields, otherwise these fileds will be dropped.
     let customFields = {}; // { "containername": serviceBusTask.containerName, "storagename": serviceBusTask.storageName };
@@ -447,7 +449,7 @@ function setSourceCategory(serviceBusTask, options) {
         });
         options.metadata["sourceFields"] = customFieldsArr.join();
     }
-    options.metadata["sourceHost"] = `${serviceBusTask.storageName}/${serviceBusTask.containerName}`
+    options.metadata["sourceHost"] = checkAndTruncate(`${serviceBusTask.storageName}/${serviceBusTask.containerName}`);
     // context.log(serviceBusTask.blobName, serviceBusTask.storageName,serviceBusTask.containerName);
     // options.metadata["sourceCategory"] = "custom_source_category";
     options.metadata["sourceName"] = checkAndTruncate(serviceBusTask.blobName);