diff --git a/help/help.txt b/help/help.txt index a0aa75dd31..a1de892ed7 100644 --- a/help/help.txt +++ b/help/help.txt @@ -175,6 +175,7 @@ Possible exit statuses and their meaning: - 0: success, no vulns found - 1: action_needed, vulns found - 2: failure, try to re-run command + - 3: failure, no supported projects detected Pro tip: use `snyk test` in your test scripts, if a vulnerability is found, the process will exit with a non-zero exit code. diff --git a/src/cli/index.ts b/src/cli/index.ts index 97077dc9e9..fac3922fe3 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -43,6 +43,7 @@ const debug = Debug('snyk'); const EXIT_CODES = { VULNS_FOUND: 1, ERROR: 2, + NO_SUPPORTED_MANIFESTS_FOUND: 3, }; async function runCommand(args: Args) { @@ -89,6 +90,13 @@ async function handleError(args, error) { spinner.clearAll(); let command = 'bad-command'; let exitCode = EXIT_CODES.ERROR; + const noSupportedManifestsFound = error.message?.includes( + 'Could not detect supported target files in', + ); + + if (noSupportedManifestsFound) { + exitCode = EXIT_CODES.NO_SUPPORTED_MANIFESTS_FOUND; + } const vulnsFound = error.code === 'VULNS'; if (vulnsFound) { diff --git a/src/lib/snyk-test/index.js b/src/lib/snyk-test/index.js index 307211de81..ae906bb710 100644 --- a/src/lib/snyk-test/index.js +++ b/src/lib/snyk-test/index.js @@ -51,7 +51,9 @@ function executeTest(root, options) { return results; }); } catch (error) { - return Promise.reject(chalk.red.bold(error)); + return Promise.reject( + chalk.red.bold(error.message ? error.message : error), + ); } } diff --git a/test/acceptance/cli-args.test.ts b/test/acceptance/cli-args.test.ts index 55743207e9..743595782f 100644 --- a/test/acceptance/cli-args.test.ts +++ b/test/acceptance/cli-args.test.ts @@ -1,8 +1,6 @@ import { test } from 'tap'; import { exec } from 'child_process'; -import { sep, join } from 'path'; -import { readFileSync, unlinkSync, rmdirSync, mkdirSync, existsSync } from 'fs'; -import { v4 as uuidv4 } from 'uuid'; +import { sep } from 'path'; const osName = require('os-name'); @@ -101,7 +99,7 @@ test('snyk test command should fail when iac file is not supported', (t) => { } t.match( stdout.trim(), - 'CustomError: Illegal infrastructure as code target file', + 'Illegal infrastructure as code target file', 'correct error output', ); }, @@ -118,7 +116,7 @@ test('snyk test command should fail when iac file is not supported', (t) => { } t.match( stdout.trim(), - 'CustomError: Not supported infrastructure as code target files in', + 'Not supported infrastructure as code target files in', 'correct error output', ); }, @@ -346,104 +344,3 @@ test('`test --json-file-output no value produces error message`', (t) => { optionsToTest.forEach(validate); }); - -test('`test --json-file-output can save JSON output to file while sending human readable output to stdout`', (t) => { - t.plan(2); - - exec( - `node ${main} test --json-file-output=snyk-direct-json-test-output.json`, - (err, stdout) => { - if (err) { - throw err; - } - t.match(stdout, 'Organization:', 'contains human readable output'); - const outputFileContents = readFileSync( - 'snyk-direct-json-test-output.json', - 'utf-8', - ); - unlinkSync('./snyk-direct-json-test-output.json'); - const jsonObj = JSON.parse(outputFileContents); - const okValue = jsonObj.ok as boolean; - t.ok(okValue, 'JSON output ok'); - }, - ); -}); - -test('`test --json-file-output produces same JSON output as normal JSON output to stdout`', (t) => { - t.plan(1); - - exec( - `node ${main} test --json --json-file-output=snyk-direct-json-test-output.json`, - (err, stdout) => { - if (err) { - throw err; - } - const stdoutJson = stdout; - const outputFileContents = readFileSync( - 'snyk-direct-json-test-output.json', - 'utf-8', - ); - unlinkSync('./snyk-direct-json-test-output.json'); - t.equals(stdoutJson, outputFileContents); - }, - ); -}); - -test('`test --json-file-output can handle a relative path`', (t) => { - t.plan(1); - - // if 'test-output' doesn't exist, created it - if (!existsSync('test-output')) { - mkdirSync('test-output'); - } - - const tempFolder = uuidv4(); - const outputPath = `test-output/${tempFolder}/snyk-direct-json-test-output.json`; - - exec( - `node ${main} test --json --json-file-output=${outputPath}`, - (err, stdout) => { - if (err) { - throw err; - } - const stdoutJson = stdout; - const outputFileContents = readFileSync(outputPath, 'utf-8'); - unlinkSync(outputPath); - rmdirSync(`test-output/${tempFolder}`); - t.equals(stdoutJson, outputFileContents); - }, - ); -}); - -test( - '`test --json-file-output can handle an absolute path`', - { skip: iswindows }, - (t) => { - t.plan(1); - - // if 'test-output' doesn't exist, created it - if (!existsSync('test-output')) { - mkdirSync('test-output'); - } - - const tempFolder = uuidv4(); - const outputPath = join( - process.cwd(), - `test-output/${tempFolder}/snyk-direct-json-test-output.json`, - ); - - exec( - `node ${main} test --json --json-file-output=${outputPath}`, - (err, stdout) => { - if (err) { - throw err; - } - const stdoutJson = stdout; - const outputFileContents = readFileSync(outputPath, 'utf-8'); - unlinkSync(outputPath); - rmdirSync(`test-output/${tempFolder}`); - t.equals(stdoutJson, outputFileContents); - }, - ); - }, -); diff --git a/test/fixtures/empty/not-supported.format b/test/fixtures/empty/not-supported.format new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/smoke/README.md b/test/smoke/README.md index af89fe8d90..bc16c47bf8 100644 --- a/test/smoke/README.md +++ b/test/smoke/README.md @@ -4,7 +4,10 @@ Design goal is to have a single test suite, that can detect if CLI is not workin CLI is being tested by a series of tests using [Shellspec](https://shellspec.info). See them in a `test/smoke/spec` folder. -Spec in this folder is used as a 1) **"Smoke test" step in CircleCI** to verify that built CLI can run 2) **["Smoke Tests"](https://github.com/snyk/snyk/actions?query=workflow%3A%22Smoke+Tests%22) GitHub Action** to verify that our distribution channels are working. +Spec in this folder is used as a + +1. **"Smoke test" step in CircleCI** to verify that built CLI can run +2. **["Smoke Tests"](https://github.com/snyk/snyk/actions?query=workflow%3A%22Smoke+Tests%22) GitHub Action** to verify that our distribution channels are working. ## How to add a new smoke test @@ -14,19 +17,29 @@ Before you start adding specs, those files are bash scripts, it's recommended to It's recommended to have a branch named `feat/smoke-test`, as [this branch will run the GitHub Action](https://github.com/snyk/snyk/blob/f35f39e96ef7aa69b22a846315dda015b12a4564/.github/workflows/smoke-tests.yml#L3-L5). -To run these tests locally, install: +To run these tests locally: + +1. Install: -- [Shellspec](https://shellspec.info) -- [jq](https://stedolan.github.io/jq/) -- timeout (if not available on your platform) + - [Shellspec](https://shellspec.info) + - [jq](https://stedolan.github.io/jq/) + - timeout (if not available on your platform) -cd into `test/smoke` folder and run: +2. Run: ```sh cd test/smoke CI=1 SMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN shellspec -f d ``` +To run the Alpine test in Docker locally: + +``` + docker build -f ./test/smoke/alpine/Dockerfile -t snyk-cli-alpine ./test/ && docker run --rm -eCI=1 -eSMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN snyk-cli-alpine +``` + +_Note: Alpine image is not copying/mounting everything, so you might need to add anything new to the `test/smoke/alpine/Dockerfile`_ + ## TODO ### Wishlist diff --git a/test/smoke/alpine/Dockerfile b/test/smoke/alpine/Dockerfile index 2c83c73075..5a6734e1b4 100644 --- a/test/smoke/alpine/Dockerfile +++ b/test/smoke/alpine/Dockerfile @@ -2,6 +2,7 @@ FROM shellspec/shellspec:latest COPY ./smoke/ /snyk/smoke/ COPY ./fixtures/basic-npm/ /snyk/fixtures/basic-npm/ +COPY ./fixtures/empty/ /snyk/fixtures/empty/ RUN shellspec --version RUN apk add curl jq libgcc libstdc++ diff --git a/test/smoke/spec/snyk_test_spec.sh b/test/smoke/spec/snyk_test_spec.sh index 5facc79dcb..66f430caa1 100644 --- a/test/smoke/spec/snyk_test_spec.sh +++ b/test/smoke/spec/snyk_test_spec.sh @@ -10,9 +10,28 @@ Describe "Snyk test command" snyk test } + run_test_in_empty_subfolder() { + cd ../fixtures/empty || return + snyk test + } + + It "throws error when file does not exist" + When run snyk test --file=non-existent/package.json + The status should equal 2 + The output should include "Could not find the specified file" + The stderr should equal "" + End + + It "throws error when no suppored manifests detected" + When run run_test_in_empty_subfolder + The status should equal 3 + The output should include "Could not detect supported target files in" + The stderr should equal "" + End + It "finds vulns in a project in the same folder" When run run_test_in_subfolder - The status should be failure # issues found + The status should equal 1 The output should include "https://snyk.io/vuln/npm:minimatch:20160620" The stderr should equal "" End diff --git a/test/system/cli-json-file-output.test.ts b/test/system/cli-json-file-output.test.ts new file mode 100644 index 0000000000..36e0e356dc --- /dev/null +++ b/test/system/cli-json-file-output.test.ts @@ -0,0 +1,114 @@ +import { test } from 'tap'; +import { exec } from 'child_process'; +import { sep, join } from 'path'; +import { readFileSync, unlinkSync, rmdirSync, mkdirSync, existsSync } from 'fs'; +import { v4 as uuidv4 } from 'uuid'; + +const osName = require('os-name'); + +const main = './dist/cli/index.js'.replace(/\//g, sep); +const iswindows = + osName() + .toLowerCase() + .indexOf('windows') === 0; + +test('`test --json-file-output can save JSON output to file while sending human readable output to stdout`', (t) => { + t.plan(2); + + exec( + `node ${main} test --json-file-output=snyk-direct-json-test-output.json`, + (err, stdout) => { + if (err) { + throw err; + } + t.match(stdout, 'Organization:', 'contains human readable output'); + const outputFileContents = readFileSync( + 'snyk-direct-json-test-output.json', + 'utf-8', + ); + unlinkSync('./snyk-direct-json-test-output.json'); + const jsonObj = JSON.parse(outputFileContents); + const okValue = jsonObj.ok as boolean; + t.ok(okValue, 'JSON output ok'); + }, + ); +}); + +test('`test --json-file-output produces same JSON output as normal JSON output to stdout`', (t) => { + t.plan(1); + + exec( + `node ${main} test --json --json-file-output=snyk-direct-json-test-output.json`, + (err, stdout) => { + if (err) { + throw err; + } + const stdoutJson = stdout; + const outputFileContents = readFileSync( + 'snyk-direct-json-test-output.json', + 'utf-8', + ); + unlinkSync('./snyk-direct-json-test-output.json'); + t.equals(stdoutJson, outputFileContents); + }, + ); +}); + +test('`test --json-file-output can handle a relative path`', (t) => { + t.plan(1); + + // if 'test-output' doesn't exist, created it + if (!existsSync('test-output')) { + mkdirSync('test-output'); + } + + const tempFolder = uuidv4(); + const outputPath = `test-output/${tempFolder}/snyk-direct-json-test-output.json`; + + exec( + `node ${main} test --json --json-file-output=${outputPath}`, + (err, stdout) => { + if (err) { + throw err; + } + const stdoutJson = stdout; + const outputFileContents = readFileSync(outputPath, 'utf-8'); + unlinkSync(outputPath); + rmdirSync(`test-output/${tempFolder}`); + t.equals(stdoutJson, outputFileContents); + }, + ); +}); + +test( + '`test --json-file-output can handle an absolute path`', + { skip: iswindows }, + (t) => { + t.plan(1); + + // if 'test-output' doesn't exist, created it + if (!existsSync('test-output')) { + mkdirSync('test-output'); + } + + const tempFolder = uuidv4(); + const outputPath = join( + process.cwd(), + `test-output/${tempFolder}/snyk-direct-json-test-output.json`, + ); + + exec( + `node ${main} test --json --json-file-output=${outputPath}`, + (err, stdout) => { + if (err) { + throw err; + } + const stdoutJson = stdout; + const outputFileContents = readFileSync(outputPath, 'utf-8'); + unlinkSync(outputPath); + rmdirSync(`test-output/${tempFolder}`); + t.equals(stdoutJson, outputFileContents); + }, + ); + }, +);