From 5b1ed7d08da878f9f0ae2d37c1333bcccdd4cbec Mon Sep 17 00:00:00 2001 From: ghe Date: Wed, 2 Sep 2020 10:35:08 +0100 Subject: [PATCH 1/2] feat: exiut code 3 for no detected projects --- src/cli/index.ts | 8 ++++++++ src/lib/snyk-test/index.js | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/cli/index.ts b/src/cli/index.ts index 97077dc9e9..fac3922fe3 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -43,6 +43,7 @@ const debug = Debug('snyk'); const EXIT_CODES = { VULNS_FOUND: 1, ERROR: 2, + NO_SUPPORTED_MANIFESTS_FOUND: 3, }; async function runCommand(args: Args) { @@ -89,6 +90,13 @@ async function handleError(args, error) { spinner.clearAll(); let command = 'bad-command'; let exitCode = EXIT_CODES.ERROR; + const noSupportedManifestsFound = error.message?.includes( + 'Could not detect supported target files in', + ); + + if (noSupportedManifestsFound) { + exitCode = EXIT_CODES.NO_SUPPORTED_MANIFESTS_FOUND; + } const vulnsFound = error.code === 'VULNS'; if (vulnsFound) { diff --git a/src/lib/snyk-test/index.js b/src/lib/snyk-test/index.js index 307211de81..ae906bb710 100644 --- a/src/lib/snyk-test/index.js +++ b/src/lib/snyk-test/index.js @@ -51,7 +51,9 @@ function executeTest(root, options) { return results; }); } catch (error) { - return Promise.reject(chalk.red.bold(error)); + return Promise.reject( + chalk.red.bold(error.message ? error.message : error), + ); } } From b6e602ba7df24c5f834f97db089dba756d5fd111 Mon Sep 17 00:00:00 2001 From: ghe Date: Thu, 3 Sep 2020 11:19:47 +0100 Subject: [PATCH 2/2] test: refactor & test exit codes --- test/acceptance/cli-args.test.ts | 109 +--------------------- test/fixtures/empty/not-supported.format | 0 test/smoke/spec/snyk_test_spec.sh | 21 ++++- test/system/cli-json-file-output.test.ts | 114 +++++++++++++++++++++++ 4 files changed, 137 insertions(+), 107 deletions(-) create mode 100644 test/fixtures/empty/not-supported.format create mode 100644 test/system/cli-json-file-output.test.ts diff --git a/test/acceptance/cli-args.test.ts b/test/acceptance/cli-args.test.ts index 55743207e9..743595782f 100644 --- a/test/acceptance/cli-args.test.ts +++ b/test/acceptance/cli-args.test.ts @@ -1,8 +1,6 @@ import { test } from 'tap'; import { exec } from 'child_process'; -import { sep, join } from 'path'; -import { readFileSync, unlinkSync, rmdirSync, mkdirSync, existsSync } from 'fs'; -import { v4 as uuidv4 } from 'uuid'; +import { sep } from 'path'; const osName = require('os-name'); @@ -101,7 +99,7 @@ test('snyk test command should fail when iac file is not supported', (t) => { } t.match( stdout.trim(), - 'CustomError: Illegal infrastructure as code target file', + 'Illegal infrastructure as code target file', 'correct error output', ); }, @@ -118,7 +116,7 @@ test('snyk test command should fail when iac file is not supported', (t) => { } t.match( stdout.trim(), - 'CustomError: Not supported infrastructure as code target files in', + 'Not supported infrastructure as code target files in', 'correct error output', ); }, @@ -346,104 +344,3 @@ test('`test --json-file-output no value produces error message`', (t) => { optionsToTest.forEach(validate); }); - -test('`test --json-file-output can save JSON output to file while sending human readable output to stdout`', (t) => { - t.plan(2); - - exec( - `node ${main} test --json-file-output=snyk-direct-json-test-output.json`, - (err, stdout) => { - if (err) { - throw err; - } - t.match(stdout, 'Organization:', 'contains human readable output'); - const outputFileContents = readFileSync( - 'snyk-direct-json-test-output.json', - 'utf-8', - ); - unlinkSync('./snyk-direct-json-test-output.json'); - const jsonObj = JSON.parse(outputFileContents); - const okValue = jsonObj.ok as boolean; - t.ok(okValue, 'JSON output ok'); - }, - ); -}); - -test('`test --json-file-output produces same JSON output as normal JSON output to stdout`', (t) => { - t.plan(1); - - exec( - `node ${main} test --json --json-file-output=snyk-direct-json-test-output.json`, - (err, stdout) => { - if (err) { - throw err; - } - const stdoutJson = stdout; - const outputFileContents = readFileSync( - 'snyk-direct-json-test-output.json', - 'utf-8', - ); - unlinkSync('./snyk-direct-json-test-output.json'); - t.equals(stdoutJson, outputFileContents); - }, - ); -}); - -test('`test --json-file-output can handle a relative path`', (t) => { - t.plan(1); - - // if 'test-output' doesn't exist, created it - if (!existsSync('test-output')) { - mkdirSync('test-output'); - } - - const tempFolder = uuidv4(); - const outputPath = `test-output/${tempFolder}/snyk-direct-json-test-output.json`; - - exec( - `node ${main} test --json --json-file-output=${outputPath}`, - (err, stdout) => { - if (err) { - throw err; - } - const stdoutJson = stdout; - const outputFileContents = readFileSync(outputPath, 'utf-8'); - unlinkSync(outputPath); - rmdirSync(`test-output/${tempFolder}`); - t.equals(stdoutJson, outputFileContents); - }, - ); -}); - -test( - '`test --json-file-output can handle an absolute path`', - { skip: iswindows }, - (t) => { - t.plan(1); - - // if 'test-output' doesn't exist, created it - if (!existsSync('test-output')) { - mkdirSync('test-output'); - } - - const tempFolder = uuidv4(); - const outputPath = join( - process.cwd(), - `test-output/${tempFolder}/snyk-direct-json-test-output.json`, - ); - - exec( - `node ${main} test --json --json-file-output=${outputPath}`, - (err, stdout) => { - if (err) { - throw err; - } - const stdoutJson = stdout; - const outputFileContents = readFileSync(outputPath, 'utf-8'); - unlinkSync(outputPath); - rmdirSync(`test-output/${tempFolder}`); - t.equals(stdoutJson, outputFileContents); - }, - ); - }, -); diff --git a/test/fixtures/empty/not-supported.format b/test/fixtures/empty/not-supported.format new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/smoke/spec/snyk_test_spec.sh b/test/smoke/spec/snyk_test_spec.sh index 5facc79dcb..66f430caa1 100644 --- a/test/smoke/spec/snyk_test_spec.sh +++ b/test/smoke/spec/snyk_test_spec.sh @@ -10,9 +10,28 @@ Describe "Snyk test command" snyk test } + run_test_in_empty_subfolder() { + cd ../fixtures/empty || return + snyk test + } + + It "throws error when file does not exist" + When run snyk test --file=non-existent/package.json + The status should equal 2 + The output should include "Could not find the specified file" + The stderr should equal "" + End + + It "throws error when no suppored manifests detected" + When run run_test_in_empty_subfolder + The status should equal 3 + The output should include "Could not detect supported target files in" + The stderr should equal "" + End + It "finds vulns in a project in the same folder" When run run_test_in_subfolder - The status should be failure # issues found + The status should equal 1 The output should include "https://snyk.io/vuln/npm:minimatch:20160620" The stderr should equal "" End diff --git a/test/system/cli-json-file-output.test.ts b/test/system/cli-json-file-output.test.ts new file mode 100644 index 0000000000..36e0e356dc --- /dev/null +++ b/test/system/cli-json-file-output.test.ts @@ -0,0 +1,114 @@ +import { test } from 'tap'; +import { exec } from 'child_process'; +import { sep, join } from 'path'; +import { readFileSync, unlinkSync, rmdirSync, mkdirSync, existsSync } from 'fs'; +import { v4 as uuidv4 } from 'uuid'; + +const osName = require('os-name'); + +const main = './dist/cli/index.js'.replace(/\//g, sep); +const iswindows = + osName() + .toLowerCase() + .indexOf('windows') === 0; + +test('`test --json-file-output can save JSON output to file while sending human readable output to stdout`', (t) => { + t.plan(2); + + exec( + `node ${main} test --json-file-output=snyk-direct-json-test-output.json`, + (err, stdout) => { + if (err) { + throw err; + } + t.match(stdout, 'Organization:', 'contains human readable output'); + const outputFileContents = readFileSync( + 'snyk-direct-json-test-output.json', + 'utf-8', + ); + unlinkSync('./snyk-direct-json-test-output.json'); + const jsonObj = JSON.parse(outputFileContents); + const okValue = jsonObj.ok as boolean; + t.ok(okValue, 'JSON output ok'); + }, + ); +}); + +test('`test --json-file-output produces same JSON output as normal JSON output to stdout`', (t) => { + t.plan(1); + + exec( + `node ${main} test --json --json-file-output=snyk-direct-json-test-output.json`, + (err, stdout) => { + if (err) { + throw err; + } + const stdoutJson = stdout; + const outputFileContents = readFileSync( + 'snyk-direct-json-test-output.json', + 'utf-8', + ); + unlinkSync('./snyk-direct-json-test-output.json'); + t.equals(stdoutJson, outputFileContents); + }, + ); +}); + +test('`test --json-file-output can handle a relative path`', (t) => { + t.plan(1); + + // if 'test-output' doesn't exist, created it + if (!existsSync('test-output')) { + mkdirSync('test-output'); + } + + const tempFolder = uuidv4(); + const outputPath = `test-output/${tempFolder}/snyk-direct-json-test-output.json`; + + exec( + `node ${main} test --json --json-file-output=${outputPath}`, + (err, stdout) => { + if (err) { + throw err; + } + const stdoutJson = stdout; + const outputFileContents = readFileSync(outputPath, 'utf-8'); + unlinkSync(outputPath); + rmdirSync(`test-output/${tempFolder}`); + t.equals(stdoutJson, outputFileContents); + }, + ); +}); + +test( + '`test --json-file-output can handle an absolute path`', + { skip: iswindows }, + (t) => { + t.plan(1); + + // if 'test-output' doesn't exist, created it + if (!existsSync('test-output')) { + mkdirSync('test-output'); + } + + const tempFolder = uuidv4(); + const outputPath = join( + process.cwd(), + `test-output/${tempFolder}/snyk-direct-json-test-output.json`, + ); + + exec( + `node ${main} test --json --json-file-output=${outputPath}`, + (err, stdout) => { + if (err) { + throw err; + } + const stdoutJson = stdout; + const outputFileContents = readFileSync(outputPath, 'utf-8'); + unlinkSync(outputPath); + rmdirSync(`test-output/${tempFolder}`); + t.equals(stdoutJson, outputFileContents); + }, + ); + }, +);