Feat/smoke test

src/cli/index.ts

@@ -43,6 +43,7 @@ const debug = Debug('snyk');
 const EXIT_CODES = {
   VULNS_FOUND: 1,
   ERROR: 2,
+  NO_SUPPORTED_MANIFESTS_FOUND: 3,
 };
 
 async function runCommand(args: Args) {
@@ -89,6 +90,13 @@ async function handleError(args, error) {
   spinner.clearAll();
   let command = 'bad-command';
   let exitCode = EXIT_CODES.ERROR;
+  const noSupportedManifestsFound = error.message?.includes(
+    'Could not detect supported target files in',
+  );
+
+  if (noSupportedManifestsFound) {
+    exitCode = EXIT_CODES.NO_SUPPORTED_MANIFESTS_FOUND;
+  }
 
   const vulnsFound = error.code === 'VULNS';
   if (vulnsFound) {

src/lib/snyk-test/index.js

@@ -51,7 +51,9 @@ function executeTest(root, options) {
       return results;
     });
   } catch (error) {
-    return Promise.reject(chalk.red.bold(error));
+    return Promise.reject(
+      chalk.red.bold(error.message ? error.message : error),
+    );
   }
 }
 

test/acceptance/cli-args.test.ts

@@ -1,8 +1,6 @@
 import { test } from 'tap';
 import { exec } from 'child_process';
-import { sep, join } from 'path';
-import { readFileSync, unlinkSync, rmdirSync, mkdirSync, existsSync } from 'fs';
-import { v4 as uuidv4 } from 'uuid';
+import { sep } from 'path';
 
 const osName = require('os-name');
 
@@ -101,7 +99,7 @@ test('snyk test command should fail when iac file is not supported', (t) => {
       }
       t.match(
         stdout.trim(),
-        'CustomError: Illegal infrastructure as code target file',
+        'Illegal infrastructure as code target file',
         'correct error output',
       );
     },
@@ -118,7 +116,7 @@ test('snyk test command should fail when iac file is not supported', (t) => {
       }
       t.match(
         stdout.trim(),
-        'CustomError: Not supported infrastructure as code target files in',
+        'Not supported infrastructure as code target files in',
         'correct error output',
       );
     },
@@ -346,104 +344,3 @@ test('`test --json-file-output no value produces error message`', (t) => {
 
   optionsToTest.forEach(validate);
 });
-
-test('`test --json-file-output can save JSON output to file while sending human readable output to stdout`', (t) => {
-  t.plan(2);
-
-  exec(
-    `node ${main} test --json-file-output=snyk-direct-json-test-output.json`,
-    (err, stdout) => {
-      if (err) {
-        throw err;
-      }
-      t.match(stdout, 'Organization:', 'contains human readable output');
-      const outputFileContents = readFileSync(
-        'snyk-direct-json-test-output.json',
-        'utf-8',
-      );
-      unlinkSync('./snyk-direct-json-test-output.json');
-      const jsonObj = JSON.parse(outputFileContents);
-      const okValue = jsonObj.ok as boolean;
-      t.ok(okValue, 'JSON output ok');
-    },
-  );
-});
-
-test('`test --json-file-output produces same JSON output as normal JSON output to stdout`', (t) => {
-  t.plan(1);
-
-  exec(
-    `node ${main} test --json --json-file-output=snyk-direct-json-test-output.json`,
-    (err, stdout) => {
-      if (err) {
-        throw err;
-      }
-      const stdoutJson = stdout;
-      const outputFileContents = readFileSync(
-        'snyk-direct-json-test-output.json',
-        'utf-8',
-      );
-      unlinkSync('./snyk-direct-json-test-output.json');
-      t.equals(stdoutJson, outputFileContents);
-    },
-  );
-});
-
-test('`test --json-file-output can handle a relative path`', (t) => {
-  t.plan(1);
-
-  // if 'test-output' doesn't exist, created it
-  if (!existsSync('test-output')) {
-    mkdirSync('test-output');
-  }
-
-  const tempFolder = uuidv4();
-  const outputPath = `test-output/${tempFolder}/snyk-direct-json-test-output.json`;
-
-  exec(
-    `node ${main} test --json --json-file-output=${outputPath}`,
-    (err, stdout) => {
-      if (err) {
-        throw err;
-      }
-      const stdoutJson = stdout;
-      const outputFileContents = readFileSync(outputPath, 'utf-8');
-      unlinkSync(outputPath);
-      rmdirSync(`test-output/${tempFolder}`);
-      t.equals(stdoutJson, outputFileContents);
-    },
-  );
-});
-
-test(
-  '`test --json-file-output can handle an absolute path`',
-  { skip: iswindows },
-  (t) => {
-    t.plan(1);
-
-    // if 'test-output' doesn't exist, created it
-    if (!existsSync('test-output')) {
-      mkdirSync('test-output');
-    }
-
-    const tempFolder = uuidv4();
-    const outputPath = join(
-      process.cwd(),
-      `test-output/${tempFolder}/snyk-direct-json-test-output.json`,
-    );
-
-    exec(
-      `node ${main} test --json --json-file-output=${outputPath}`,
-      (err, stdout) => {
-        if (err) {
-          throw err;
-        }
-        const stdoutJson = stdout;
-        const outputFileContents = readFileSync(outputPath, 'utf-8');
-        unlinkSync(outputPath);
-        rmdirSync(`test-output/${tempFolder}`);
-        t.equals(stdoutJson, outputFileContents);
-      },
-    );
-  },
-);

test/smoke/spec/snyk_test_spec.sh

@@ -10,9 +10,28 @@ Describe "Snyk test command"
       snyk test
     }
 
+    run_test_in_empty_subfolder() {
+      cd ../fixtures/empty || return
+      snyk test
+    }
+
+    It "throws error when file does not exist"
+      When run snyk test --file=non-existent/package.json
+      The status should equal 2
+      The output should include "Could not find the specified file"
+      The stderr should equal ""
+    End
+
+    It "throws error when no suppored manifests detected"
+      When run run_test_in_empty_subfolder
+      The status should equal 3
+      The output should include "Could not detect supported target files in"
+      The stderr should equal ""
+    End
+
     It "finds vulns in a project in the same folder"
       When run run_test_in_subfolder
-      The status should be failure # issues found
+      The status should equal 1
       The output should include "https://snyk.io/vuln/npm:minimatch:20160620"
       The stderr should equal ""
     End

test/system/cli-json-file-output.test.ts

@@ -0,0 +1,114 @@
+import { test } from 'tap';
+import { exec } from 'child_process';
+import { sep, join } from 'path';
+import { readFileSync, unlinkSync, rmdirSync, mkdirSync, existsSync } from 'fs';
+import { v4 as uuidv4 } from 'uuid';
+
+const osName = require('os-name');
+
+const main = './dist/cli/index.js'.replace(/\//g, sep);
+const iswindows =
+  osName()
+    .toLowerCase()
+    .indexOf('windows') === 0;
+
+test('`test --json-file-output can save JSON output to file while sending human readable output to stdout`', (t) => {
+  t.plan(2);
+
+  exec(
+    `node ${main} test --json-file-output=snyk-direct-json-test-output.json`,
+    (err, stdout) => {
+      if (err) {
+        throw err;
+      }
+      t.match(stdout, 'Organization:', 'contains human readable output');
+      const outputFileContents = readFileSync(
+        'snyk-direct-json-test-output.json',
+        'utf-8',
+      );
+      unlinkSync('./snyk-direct-json-test-output.json');
+      const jsonObj = JSON.parse(outputFileContents);
+      const okValue = jsonObj.ok as boolean;
+      t.ok(okValue, 'JSON output ok');
+    },
+  );
+});
+
+test('`test --json-file-output produces same JSON output as normal JSON output to stdout`', (t) => {
+  t.plan(1);
+
+  exec(
+    `node ${main} test --json --json-file-output=snyk-direct-json-test-output.json`,
+    (err, stdout) => {
+      if (err) {
+        throw err;
+      }
+      const stdoutJson = stdout;
+      const outputFileContents = readFileSync(
+        'snyk-direct-json-test-output.json',
+        'utf-8',
+      );
+      unlinkSync('./snyk-direct-json-test-output.json');
+      t.equals(stdoutJson, outputFileContents);
+    },
+  );
+});
+
+test('`test --json-file-output can handle a relative path`', (t) => {
+  t.plan(1);
+
+  // if 'test-output' doesn't exist, created it
+  if (!existsSync('test-output')) {
+    mkdirSync('test-output');
+  }
+
+  const tempFolder = uuidv4();
+  const outputPath = `test-output/${tempFolder}/snyk-direct-json-test-output.json`;
+
+  exec(
+    `node ${main} test --json --json-file-output=${outputPath}`,
+    (err, stdout) => {
+      if (err) {
+        throw err;
+      }
+      const stdoutJson = stdout;
+      const outputFileContents = readFileSync(outputPath, 'utf-8');
+      unlinkSync(outputPath);
+      rmdirSync(`test-output/${tempFolder}`);
+      t.equals(stdoutJson, outputFileContents);
+    },
+  );
+});
+
+test(
+  '`test --json-file-output can handle an absolute path`',
+  { skip: iswindows },
+  (t) => {
+    t.plan(1);
+
+    // if 'test-output' doesn't exist, created it
+    if (!existsSync('test-output')) {
+      mkdirSync('test-output');
+    }
+
+    const tempFolder = uuidv4();
+    const outputPath = join(
+      process.cwd(),
+      `test-output/${tempFolder}/snyk-direct-json-test-output.json`,
+    );
+
+    exec(
+      `node ${main} test --json --json-file-output=${outputPath}`,
+      (err, stdout) => {
+        if (err) {
+          throw err;
+        }
+        const stdoutJson = stdout;
+        const outputFileContents = readFileSync(outputPath, 'utf-8');
+        unlinkSync(outputPath);
+        rmdirSync(`test-output/${tempFolder}`);
+        t.equals(stdoutJson, outputFileContents);
+      },
+    );
+  },
+);