Feat:exit code 3 when no projects detected

help/help.txt

@@ -175,6 +175,7 @@ Possible exit statuses and their meaning:
   - 0: success, no vulns found
   - 1: action_needed, vulns found
   - 2: failure, try to re-run command
+  - 3: failure, no supported projects detected
 
 Pro tip: use `snyk test` in your test scripts, if a vulnerability is
 found, the process will exit with a non-zero exit code.

src/cli/index.ts

@@ -43,6 +43,7 @@ const debug = Debug('snyk');
 const EXIT_CODES = {
   VULNS_FOUND: 1,
   ERROR: 2,
+  NO_SUPPORTED_MANIFESTS_FOUND: 3,
 };
 
 async function runCommand(args: Args) {
@@ -89,6 +90,13 @@ async function handleError(args, error) {
   spinner.clearAll();
   let command = 'bad-command';
   let exitCode = EXIT_CODES.ERROR;
+  const noSupportedManifestsFound = error.message?.includes(
+    'Could not detect supported target files in',
+  );
+
+  if (noSupportedManifestsFound) {
+    exitCode = EXIT_CODES.NO_SUPPORTED_MANIFESTS_FOUND;
+  }
 
   const vulnsFound = error.code === 'VULNS';
   if (vulnsFound) {

src/lib/snyk-test/index.js

@@ -51,7 +51,9 @@ function executeTest(root, options) {
       return results;
     });
   } catch (error) {
-    return Promise.reject(chalk.red.bold(error));
+    return Promise.reject(
+      chalk.red.bold(error.message ? error.message : error),
+    );
   }
 }
 

test/acceptance/cli-args.test.ts

@@ -1,8 +1,6 @@
 import { test } from 'tap';
 import { exec } from 'child_process';
-import { sep, join } from 'path';
-import { readFileSync, unlinkSync, rmdirSync, mkdirSync, existsSync } from 'fs';
-import { v4 as uuidv4 } from 'uuid';
+import { sep } from 'path';
 
 const osName = require('os-name');
 
@@ -101,7 +99,7 @@ test('snyk test command should fail when iac file is not supported', (t) => {
       }
       t.match(
         stdout.trim(),
-        'CustomError: Illegal infrastructure as code target file',
+        'Illegal infrastructure as code target file',
         'correct error output',
       );
     },
@@ -118,7 +116,7 @@ test('snyk test command should fail when iac file is not supported', (t) => {
       }
       t.match(
         stdout.trim(),
-        'CustomError: Not supported infrastructure as code target files in',
+        'Not supported infrastructure as code target files in',
         'correct error output',
       );
     },
@@ -346,104 +344,3 @@ test('`test --json-file-output no value produces error message`', (t) => {
 
   optionsToTest.forEach(validate);
 });
-
-test('`test --json-file-output can save JSON output to file while sending human readable output to stdout`', (t) => {
-  t.plan(2);
-
-  exec(
-    `node ${main} test --json-file-output=snyk-direct-json-test-output.json`,
-    (err, stdout) => {
-      if (err) {
-        throw err;
-      }
-      t.match(stdout, 'Organization:', 'contains human readable output');
-      const outputFileContents = readFileSync(
-        'snyk-direct-json-test-output.json',
-        'utf-8',
-      );
-      unlinkSync('./snyk-direct-json-test-output.json');
-      const jsonObj = JSON.parse(outputFileContents);
-      const okValue = jsonObj.ok as boolean;
-      t.ok(okValue, 'JSON output ok');
-    },
-  );
-});
-
-test('`test --json-file-output produces same JSON output as normal JSON output to stdout`', (t) => {
-  t.plan(1);
-
-  exec(
-    `node ${main} test --json --json-file-output=snyk-direct-json-test-output.json`,
-    (err, stdout) => {
-      if (err) {
-        throw err;
-      }
-      const stdoutJson = stdout;
-      const outputFileContents = readFileSync(
-        'snyk-direct-json-test-output.json',
-        'utf-8',
-      );
-      unlinkSync('./snyk-direct-json-test-output.json');
-      t.equals(stdoutJson, outputFileContents);
-    },
-  );
-});
-
-test('`test --json-file-output can handle a relative path`', (t) => {
-  t.plan(1);
-
-  // if 'test-output' doesn't exist, created it
-  if (!existsSync('test-output')) {
-    mkdirSync('test-output');
-  }
-
-  const tempFolder = uuidv4();
-  const outputPath = `test-output/${tempFolder}/snyk-direct-json-test-output.json`;
-
-  exec(
-    `node ${main} test --json --json-file-output=${outputPath}`,
-    (err, stdout) => {
-      if (err) {
-        throw err;
-      }
-      const stdoutJson = stdout;
-      const outputFileContents = readFileSync(outputPath, 'utf-8');
-      unlinkSync(outputPath);
-      rmdirSync(`test-output/${tempFolder}`);
-      t.equals(stdoutJson, outputFileContents);
-    },
-  );
-});
-
-test(
-  '`test --json-file-output can handle an absolute path`',
-  { skip: iswindows },
-  (t) => {
-    t.plan(1);
-
-    // if 'test-output' doesn't exist, created it
-    if (!existsSync('test-output')) {
-      mkdirSync('test-output');
-    }
-
-    const tempFolder = uuidv4();
-    const outputPath = join(
-      process.cwd(),
-      `test-output/${tempFolder}/snyk-direct-json-test-output.json`,
-    );
-
-    exec(
-      `node ${main} test --json --json-file-output=${outputPath}`,
-      (err, stdout) => {
-        if (err) {
-          throw err;
-        }
-        const stdoutJson = stdout;
-        const outputFileContents = readFileSync(outputPath, 'utf-8');
-        unlinkSync(outputPath);
-        rmdirSync(`test-output/${tempFolder}`);
-        t.equals(stdoutJson, outputFileContents);
-      },
-    );
-  },
-);

test/smoke/README.md

@@ -4,7 +4,10 @@ Design goal is to have a single test suite, that can detect if CLI is not workin
 
 CLI is being tested by a series of tests using [Shellspec](https://shellspec.info). See them in a `test/smoke/spec` folder.
 
-Spec in this folder is used as a 1) **"Smoke test" step in CircleCI** to verify that built CLI can run 2) **["Smoke Tests"](https://github.com/snyk/snyk/actions?query=workflow%3A%22Smoke+Tests%22) GitHub Action** to verify that our distribution channels are working.
+Spec in this folder is used as a
+
+1. **"Smoke test" step in CircleCI** to verify that built CLI can run
+2. **["Smoke Tests"](https://github.com/snyk/snyk/actions?query=workflow%3A%22Smoke+Tests%22) GitHub Action** to verify that our distribution channels are working.
 
 ## How to add a new smoke test
 
@@ -14,19 +17,29 @@ Before you start adding specs, those files are bash scripts, it's recommended to
 
 It's recommended to have a branch named `feat/smoke-test`, as [this branch will run the GitHub Action](https://github.com/snyk/snyk/blob/f35f39e96ef7aa69b22a846315dda015b12a4564/.github/workflows/smoke-tests.yml#L3-L5).
 
-To run these tests locally, install:
+To run these tests locally:
+
+1. Install:
 
-- [Shellspec](https://shellspec.info)
-- [jq](https://stedolan.github.io/jq/)
-- timeout (if not available on your platform)
+   - [Shellspec](https://shellspec.info)
+   - [jq](https://stedolan.github.io/jq/)
+   - timeout (if not available on your platform)
 
-cd into `test/smoke` folder and run:
+2. Run:
 
 ```sh
 cd test/smoke
 CI=1 SMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN shellspec -f d
 ```
 
+To run the Alpine test in Docker locally:
+
+```
+ docker build -f ./test/smoke/alpine/Dockerfile -t snyk-cli-alpine ./test/ && docker run --rm -eCI=1 -eSMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN snyk-cli-alpine
+```
+
+_Note: Alpine image is not copying/mounting everything, so you might need to add anything new to the `test/smoke/alpine/Dockerfile`_
+
 ## TODO
 
 ### Wishlist

test/smoke/alpine/Dockerfile

@@ -2,6 +2,7 @@ FROM shellspec/shellspec:latest
 
 COPY ./smoke/ /snyk/smoke/
 COPY ./fixtures/basic-npm/ /snyk/fixtures/basic-npm/
+COPY ./fixtures/empty/ /snyk/fixtures/empty/
 
 RUN shellspec --version
 RUN apk add curl jq libgcc libstdc++

test/smoke/spec/snyk_test_spec.sh

@@ -10,9 +10,28 @@ Describe "Snyk test command"
       snyk test
     }
 
+    run_test_in_empty_subfolder() {
+      cd ../fixtures/empty || return
+      snyk test
+    }
+
+    It "throws error when file does not exist"
+      When run snyk test --file=non-existent/package.json
+      The status should equal 2
+      The output should include "Could not find the specified file"
+      The stderr should equal ""
+    End
+
+    It "throws error when no suppored manifests detected"
+      When run run_test_in_empty_subfolder
+      The status should equal 3
+      The output should include "Could not detect supported target files in"
+      The stderr should equal ""
+    End
+
     It "finds vulns in a project in the same folder"
       When run run_test_in_subfolder
-      The status should be failure # issues found
+      The status should equal 1
       The output should include "https://snyk.io/vuln/npm:minimatch:20160620"
       The stderr should equal ""
     End

test/system/cli-json-file-output.test.ts

@@ -0,0 +1,114 @@
+import { test } from 'tap';
+import { exec } from 'child_process';
+import { sep, join } from 'path';
+import { readFileSync, unlinkSync, rmdirSync, mkdirSync, existsSync } from 'fs';
+import { v4 as uuidv4 } from 'uuid';
+
+const osName = require('os-name');
+
+const main = './dist/cli/index.js'.replace(/\//g, sep);
+const iswindows =
+  osName()
+    .toLowerCase()
+    .indexOf('windows') === 0;
+
+test('`test --json-file-output can save JSON output to file while sending human readable output to stdout`', (t) => {
+  t.plan(2);
+
+  exec(
+    `node ${main} test --json-file-output=snyk-direct-json-test-output.json`,
+    (err, stdout) => {
+      if (err) {
+        throw err;
+      }
+      t.match(stdout, 'Organization:', 'contains human readable output');
+      const outputFileContents = readFileSync(
+        'snyk-direct-json-test-output.json',
+        'utf-8',
+      );
+      unlinkSync('./snyk-direct-json-test-output.json');
+      const jsonObj = JSON.parse(outputFileContents);
+      const okValue = jsonObj.ok as boolean;
+      t.ok(okValue, 'JSON output ok');
+    },
+  );
+});
+
+test('`test --json-file-output produces same JSON output as normal JSON output to stdout`', (t) => {
+  t.plan(1);
+
+  exec(
+    `node ${main} test --json --json-file-output=snyk-direct-json-test-output.json`,
+    (err, stdout) => {
+      if (err) {
+        throw err;
+      }
+      const stdoutJson = stdout;
+      const outputFileContents = readFileSync(
+        'snyk-direct-json-test-output.json',
+        'utf-8',
+      );
+      unlinkSync('./snyk-direct-json-test-output.json');
+      t.equals(stdoutJson, outputFileContents);
+    },
+  );
+});
+
+test('`test --json-file-output can handle a relative path`', (t) => {
+  t.plan(1);
+
+  // if 'test-output' doesn't exist, created it
+  if (!existsSync('test-output')) {
+    mkdirSync('test-output');
+  }
+
+  const tempFolder = uuidv4();
+  const outputPath = `test-output/${tempFolder}/snyk-direct-json-test-output.json`;
+
+  exec(
+    `node ${main} test --json --json-file-output=${outputPath}`,
+    (err, stdout) => {
+      if (err) {
+        throw err;
+      }
+      const stdoutJson = stdout;
+      const outputFileContents = readFileSync(outputPath, 'utf-8');
+      unlinkSync(outputPath);
+      rmdirSync(`test-output/${tempFolder}`);
+      t.equals(stdoutJson, outputFileContents);
+    },
+  );
+});
+
+test(
+  '`test --json-file-output can handle an absolute path`',
+  { skip: iswindows },
+  (t) => {
+    t.plan(1);
+
+    // if 'test-output' doesn't exist, created it
+    if (!existsSync('test-output')) {
+      mkdirSync('test-output');
+    }
+
+    const tempFolder = uuidv4();
+    const outputPath = join(
+      process.cwd(),
+      `test-output/${tempFolder}/snyk-direct-json-test-output.json`,
+    );
+
+    exec(
+      `node ${main} test --json --json-file-output=${outputPath}`,
+      (err, stdout) => {
+        if (err) {
+          throw err;
+        }
+        const stdoutJson = stdout;
+        const outputFileContents = readFileSync(outputPath, 'utf-8');
+        unlinkSync(outputPath);
+        rmdirSync(`test-output/${tempFolder}`);
+        t.equals(stdoutJson, outputFileContents);
+      },
+    );
+  },
+);