chore(test): use shellspec for regression test

.circleci/config.yml

@@ -70,6 +70,11 @@ commands:
       - run:
           name: Use correct version of Node
           command: nvm use << parameters.node_version >>
+  install_shellspec:
+    description: Install Shellspec
+    steps:
+      - run: curl -fsSL https://git.io/shellspec | sh -s -- -y
+      - run: sudo ln -s ${HOME}/.local/lib/shellspec/shellspec /usr/local/bin/shellspec
   show_node_version:
     description: Log Node and npm version
     steps:
@@ -81,15 +86,13 @@ commands:
           command: npm --version
 
 jobs:
-  common:
+  smoke-test:
     <<: *defaults
     docker:
       - image: circleci/node:<< parameters.node_version >>
     steps:
-      - show_node_version
       - checkout
-      - attach_workspace:
-          at: ~/snyk
+      - install_shellspec
       - install_deps
       - build_ts
       - run:
@@ -98,6 +101,33 @@ jobs:
       - run:
           name: Run tests
           command: npm run test:common
+      - run:
+          name: Clear Snyk config
+          command: rm ~/.config/configstore/snyk.json
+      - run:
+          name: Clear node_modules
+          command: rm -rf node_modules/
+      - run:
+          name: Install only production dependencies
+          command: npm install
+          environment:
+            NODE_ENV: production
+      - run:
+          name: npm install for fixture project
+          working_directory: ~/snyk/test/fixtures/basic-npm
+          command: npm install
+      - run:
+          name: Setup alias for snyk
+          working_directory: ~/snyk/
+          command: sudo ./test/smoke/setup-alias-for-snyk.sh
+      - run:
+          name: Run shellspec
+          working_directory: ~/snyk/test/smoke
+          command: |
+            echo "Checkout the README in test/smoke folder for more details about this step"
+            unset SNYK_API
+            unset SNYK_API_KEY
+            shellspec -f d
 
   danger:
     <<: *defaults
@@ -157,32 +187,6 @@ jobs:
           name: Run tests
           command: npm test
 
-  test-regression:
-    <<: *defaults
-    docker:
-      - image: circleci/node:<< parameters.node_version >>
-    steps:
-      - checkout
-      - install_deps
-      - build_ts
-      - run:
-          name: Clear node_modules
-          command: rm -rf node_modules/
-      - run:
-          name: Install only production dependencies
-          command: npm install
-          environment:
-            NODE_ENV: production
-      - run:
-          name: Run auth
-          command: node ~/snyk/dist/cli/index.js auth $SNYK_API_KEY
-      - run:
-          name: Regression test in test command
-          command: node ~/snyk/dist/cli/index.js test test/acceptance/workspaces/npm-package-no-vulns
-      - run:
-          name: Regression test in monitor command
-          command: node ~/snyk/dist/cli/index.js monitor test/acceptance/workspaces/npm-package-no-vulns
-
   release:
     <<: *defaults
     docker:
@@ -200,17 +204,19 @@ workflows:
   version: 2
   test_and_release:
     jobs:
-      - common:
-          name: Common
-          context: nodejs-install
+      - danger:
+          name: Danger Zone
           node_version: '8'
           filters:
             branches:
               ignore:
                 - master
-      - danger:
-          name: Danger Zone
+      - smoke-test:
+          name: Smoke Test
+          context: nodejs-install
           node_version: '8'
+          requires:
+            - Danger Zone
           filters:
             branches:
               ignore:
@@ -220,8 +226,7 @@ workflows:
           context: nodejs-install
           node_version: '12.0.0'
           requires:
-            - Common
-            - Danger Zone
+            - Smoke Test
           filters:
             branches:
               ignore:
@@ -231,8 +236,7 @@ workflows:
           context: nodejs-install
           node_version: '14'
           requires:
-            - Common
-            - Danger Zone
+            - Smoke Test
           filters:
             branches:
               ignore:
@@ -242,8 +246,7 @@ workflows:
           context: nodejs-install
           node_version: '10.21.0'
           requires:
-            - Common
-            - Danger Zone
+            - Smoke Test
           filters:
             branches:
               ignore:
@@ -253,8 +256,7 @@ workflows:
           context: nodejs-install
           node_version: '8.17.0'
           requires:
-            - Common
-            - Danger Zone
+            - Smoke Test
           filters:
             branches:
               ignore:
@@ -264,8 +266,7 @@ workflows:
           context: nodejs-install
           node_version: '12.16.2'
           requires:
-            - Common
-            - Danger Zone
+            - Smoke Test
           filters:
             branches:
               ignore:
@@ -275,8 +276,7 @@ workflows:
           context: nodejs-install
           node_version: '14'
           requires:
-            - Common
-            - Danger Zone
+            - Smoke Test
           filters:
             branches:
               ignore:
@@ -286,8 +286,7 @@ workflows:
           context: nodejs-install
           node_version: '10'
           requires:
-            - Common
-            - Danger Zone
+            - Smoke Test
           filters:
             branches:
               ignore:
@@ -297,25 +296,7 @@ workflows:
           context: nodejs-install
           node_version: '8'
           requires:
-            - Common
-            - Danger Zone
-          filters:
-            branches:
-              ignore:
-                - master
-      - test-regression:
-          name: Regression Test
-          context: nodejs-install
-          node_version: '8'
-          requires:
-            - Unix Tests for Node v8 support
-            - Unix Tests for Node v10 support
-            - Unix Tests for Node v12 support
-            - Unix Tests for Node v14 support
-            - Windows Tests for Node v8 support
-            - Windows Tests for Node v10 support
-            - Windows Tests for Node v12 support
-            - Windows Tests for Node v14 support
+            - Smoke Test
           filters:
             branches:
               ignore:

test/smoke/README.md

@@ -0,0 +1,33 @@
+# Snyk CLI Smoke Tests
+
+Design goal is to have a single test suite, that can detect if CLI is not working properly - before and after it's released. Defects it should catch are e.g. [broken pkg builds for a specific platform](https://github.com/snyk/snyk/issues/670), [misaligned dependencies](https://github.com/snyk/snyk/issues/1261) or [issues with a specific installation targets](https://github.com/snyk/snyk/issues/1270).
+
+CLI is being tested by a series of tests using [Shellspec](https://shellspec.info). See them in a `test/smoke/spec` folder.
+
+Spec in this folder is used as a 1) **"Smoke test" step in CircleCI** to verify that built CLI can run 2) **["Smoke Tests"](https://github.com/snyk/snyk/actions?query=workflow%3A%22Smoke+Tests%22) GitHub Action** to verify that our distribution channels are working.
+
+## How to add a new smoke test
+
+Smoke tests should be fast. Ideally tests only things that could break when CLI is being built and packaged into binaries. Functionality should be tested our _other_ tests.
+
+Before you start adding specs, those files are bash scripts, it's recommended to have a [ShellCheck](https://www.shellcheck.net) installed and running in your IDE. See [Shellspec reference](https://github.com/shellspec/shellspec/blob/master/docs/references.md#expectation) for available commands.
+
+It's recommended to have a branch named `feat/smoke-test`, as [this branch will run the GitHub Action](https://github.com/snyk/snyk/blob/f35f39e96ef7aa69b22a846315dda015b12a4564/.github/workflows/smoke-tests.yml#L3-L5).
+
+## TODO
+
+### Missing scenarios
+
+- [x] basics: version, help, config
+- [x] auth [TOKEN]
+- [ ] test [--json][npm project, java-goof]
+- [ ] policy, ignore
+- [ ] monitor
+- [ ] wizard - possibly impossible? (maybe a basic test that it even loads)
+
+### Missing environments
+
+- [ ] Alpine binary
+- [ ] Docker: current images can't output a clear stderr, because of an extraneous --json flag. Also released version is currently lagging behind the latest GitHub tag by a few hours
+- [ ] yarn installation (see https://github.com/snyk/snyk/issues/1270)
+- [ ] scoop package

test/smoke/setup-alias-for-snyk.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -e
+
+# Force CI to have this built Snyk version available in shell used
+echo "node ${PWD}/dist/cli/index.js \"\$@\"" > /usr/local/bin/snyk
+chmod +x /usr/local/bin/snyk

test/smoke/spec/snyk_basic_spec.sh

@@ -3,15 +3,15 @@
 Describe "Snyk CLI basics"
   Describe "snyk version"
     It "prints version"
-      When run "snyk" version
+      When run snyk version
       The output should include "${EXPECTED_SNYK_VERSION}"
       The status should be success
       # TODO: unusable with our current docker issues
       The stderr should equal ""
     End
 
-    It "prints version"
-      When run "snyk" --version
+    It "prints version with --version flag"
+      When run snyk --version
       The output should include "${EXPECTED_SNYK_VERSION}"
       The status should be success
       # TODO: unusable with our current docker issues