diff --git a/.circleci/config.yml b/.circleci/config.yml index 85bccffff7..811c4ff0a6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -359,7 +359,7 @@ jobs: command: | latest_version=$(cat lerna.json | jq .version -r) new_tag="v${latest_version}" - gh release create ${new_tag} --title "${new_tag}" -n "Release notes not implemented yet" --prerelease + gh release create ${new_tag} --title "${new_tag}" -n "Release Notes pending" - run: name: Add Assets to GitHub Release command: | diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d5c77235bf..66fdc0d7e7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,7 +4,6 @@ src/cli/commands/test/formatters/format-reachability.ts @snyk/flow help/ @snyk/hammer * @snyk/hammer @snyk/boost src/cli/commands/test/iac-output.ts @snyk/cloudconfig -src/cli/commands/test/iac-local-execution/ @snyk/cloudconfig src/lib/cloud-config-projects.ts @snyk/cloudconfig src/lib/iac/ @snyk/cloudconfig src/lib/snyk-test/iac-test-result.ts @snyk/cloudconfig @@ -13,7 +12,6 @@ src/lib/snyk-test/run-iac-test.ts @snyk/cloudconfig test/acceptance/cli-test/iac/ @snyk/cloudconfig test/fixtures/iac/ @snyk/cloudconfig test/smoke/spec/iac/ @snyk/cloudconfig -test/smoke/.iac-data/ @snyk/cloudconfig src/lib/errors/invalid-iac-file.ts @snyk/cloudconfig src/lib/errors/unsupported-options-iac-error.ts @snyk/cloudconfig help/commands-docs/iac-examples.md @snyk/cloudconfig diff --git a/.gitignore b/.gitignore index c89256230a..d3f044f2cc 100644 --- a/.gitignore +++ b/.gitignore @@ -29,5 +29,3 @@ report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json /test/acceptance/workspaces/**/target/ test/acceptance/workspaces/**/.gradle test/**/.gradle -.iac-data -!test/smoke/.iac-data \ No newline at end of file diff --git a/package.json b/package.json index b0a161a49e..0ac99b8a07 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,6 @@ "author": "snyk.io", "license": "Apache-2.0", "dependencies": { - "@open-policy-agent/opa-wasm": "git+https://github.com/open-policy-agent/npm-opa-wasm.git#f4a21fe6f4d70706f85106dc6ea867983747e040", "@snyk/cli-interface": "2.11.0", "@snyk/dep-graph": "1.21.0", "@snyk/gemfile": "1.2.0", diff --git a/src/cli/commands/test/iac-local-execution/index.ts b/src/cli/commands/test/iac-local-execution/index.ts deleted file mode 100644 index d009b848f5..0000000000 --- a/src/cli/commands/test/iac-local-execution/index.ts +++ /dev/null @@ -1,86 +0,0 @@ -import * as fs from 'fs'; -import * as YAML from 'js-yaml'; -import { isLocalFolder } from '../../../../lib/detect'; -import { getFileType } from '../../../../lib/iac/iac-parser'; -import * as util from 'util'; -import { IacFileTypes } from '../../../../lib/iac/constants'; -import { IacFileScanResult, IacFileMetadata, IacFileData } from './types'; -import { buildPolicyEngine } from './policy-engine'; -import { formatResults } from './results-formatter'; - -const readFileContentsAsync = util.promisify(fs.readFile); -const REQUIRED_K8S_FIELDS = ['apiVersion', 'kind', 'metadata']; - -// this method executes the local processing engine and then formats the results to adapt with the CLI output. -// the current version is dependent on files to be present locally which are not part of the source code. -// without these files this method would fail. -// if you're interested in trying out the experimental local execution model for IaC scanning, please reach-out. -export async function test(pathToScan: string, options) { - // TODO: add support for proper typing of old TestResult interface. - const results = await localProcessing(pathToScan); - const formattedResults = formatResults(results, options); - const singleFileFormattedResult = formattedResults[0]; - - return singleFileFormattedResult as any; -} - -async function localProcessing( - pathToScan: string, -): Promise { - const policyEngine = await buildPolicyEngine(); - const filePathsToScan = await getFilePathsToScan(pathToScan); - const fileDataToScan = await parseFileContentsForPolicyEngine( - filePathsToScan, - ); - const scanResults = await policyEngine.scanFiles(fileDataToScan); - return scanResults; -} - -async function getFilePathsToScan(pathToScan): Promise { - if (isLocalFolder(pathToScan)) { - throw new Error( - 'IaC Experimental version does not support directory scan yet.', - ); - } - if (getFileType(pathToScan) === 'tf') { - throw new Error( - 'IaC Experimental version does not support Terraform scan yet.', - ); - } - - return [ - { filePath: pathToScan, fileType: getFileType(pathToScan) as IacFileTypes }, - ]; -} - -async function parseFileContentsForPolicyEngine( - filesMetadata: IacFileMetadata[], -): Promise { - const parsedFileData: Array = []; - for (const fileMetadata of filesMetadata) { - const fileContent = await readFileContentsAsync( - fileMetadata.filePath, - 'utf-8', - ); - const yamlDocuments = YAML.safeLoadAll(fileContent); - - yamlDocuments.forEach((parsedYamlDocument, docId) => { - if ( - REQUIRED_K8S_FIELDS.every((requiredField) => - parsedYamlDocument.hasOwnProperty(requiredField), - ) - ) { - parsedFileData.push({ - ...fileMetadata, - fileContent: fileContent, - jsonContent: parsedYamlDocument, - docId, - }); - } else { - throw new Error('Invalid K8s File!'); - } - }); - } - - return parsedFileData; -} diff --git a/src/cli/commands/test/iac-local-execution/policy-engine.ts b/src/cli/commands/test/iac-local-execution/policy-engine.ts deleted file mode 100644 index 468be9e2e5..0000000000 --- a/src/cli/commands/test/iac-local-execution/policy-engine.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { - OpaWasmInstance, - IacFileData, - IacFileScanResult, - PolicyMetadata, -} from './types'; -import { loadPolicy } from '@open-policy-agent/opa-wasm'; -import * as fs from 'fs'; -import * as path from 'path'; - -const LOCAL_POLICY_ENGINE_DIR = `.iac-data`; -const LOCAL_POLICY_ENGINE_WASM_PATH = `${LOCAL_POLICY_ENGINE_DIR}${path.sep}policy.wasm`; -const LOCAL_POLICY_ENGINE_DATA_PATH = `${LOCAL_POLICY_ENGINE_DIR}${path.sep}data.json`; - -export async function buildPolicyEngine(): Promise { - const policyEngineCoreDataPath = `${process.cwd()}/${LOCAL_POLICY_ENGINE_WASM_PATH}`; - const policyEngineMetaDataPath = `${process.cwd()}/${LOCAL_POLICY_ENGINE_DATA_PATH}`; - try { - const wasmFile = fs.readFileSync(policyEngineCoreDataPath); - const policyMetaData = fs.readFileSync(policyEngineMetaDataPath); - const policyMetadataAsJson: Record = JSON.parse( - policyMetaData.toString(), - ); - - const opaWasmInstance: OpaWasmInstance = await loadPolicy( - Buffer.from(wasmFile), - ); - opaWasmInstance.setData(policyMetadataAsJson); - - return new PolicyEngine(opaWasmInstance); - } catch (err) { - throw new Error( - `Failed to build policy engine from path: ${LOCAL_POLICY_ENGINE_DIR}: \n err: ${err.message}`, - ); - } -} - -class PolicyEngine { - constructor(private opaWasmInstance: OpaWasmInstance) { - this.opaWasmInstance = opaWasmInstance; - } - - private evaluate(data: Record): PolicyMetadata[] { - return this.opaWasmInstance.evaluate(data)[0].result; - } - - public async scanFiles( - filesToScan: IacFileData[], - ): Promise { - try { - return filesToScan.map((iacFile: IacFileData) => { - const violatedPolicies = this.evaluate(iacFile.jsonContent); - return { - ...iacFile, - violatedPolicies, - }; - }); - } catch (err) { - // TODO: to distinguish between different failure reasons - throw new Error(`Failed to run policy engine: ${err}`); - } - } -} diff --git a/src/cli/commands/test/iac-local-execution/results-formatter.ts b/src/cli/commands/test/iac-local-execution/results-formatter.ts deleted file mode 100644 index 4bbd17f75d..0000000000 --- a/src/cli/commands/test/iac-local-execution/results-formatter.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { IacFileScanResult, PolicyMetadata } from './types'; -import { SEVERITY } from '../../../../lib/snyk-test/common'; -// import { -// issuesToLineNumbers, -// CloudConfigFileTypes, -// } from '@snyk/cloud-config-parser'; - -const SEVERITIES = [SEVERITY.LOW, SEVERITY.MEDIUM, SEVERITY.HIGH]; - -export function formatResults( - iacLocalExecutionResults: Array, - options: { severityThreshold?: SEVERITY }, -) { - const iacLocalExecutionGroupedResults = groupMultiDocResults( - iacLocalExecutionResults, - ); - return iacLocalExecutionGroupedResults.map((iacScanResult) => - iacLocalFileScanToFormattedResult(iacScanResult, options.severityThreshold), - ); -} - -// -// function getFileTypeForLineNumber( -// fileType: string, -// ): CloudConfigFileTypes { -// switch (fileType) { -// case 'yaml': -// case 'yml': -// return CloudConfigFileTypes.YAML; -// case 'json': -// return CloudConfigFileTypes.JSON; -// default: -// return CloudConfigFileTypes.YAML; -// } -// } - -function iacLocalFileScanToFormattedResult( - iacFileScanResult: IacFileScanResult, - severityThreshold?: SEVERITY, -) { - const formattedIssues = iacFileScanResult.violatedPolicies.map((policy) => { - // TODO: make sure we handle this issue with annotations: - // https://github.com/snyk/registry/pull/17277 - const cloudConfigPath = [`[DocId:${iacFileScanResult.docId}]`].concat( - policy.msg.split('.'), - ); - const lineNumber = -1; - // TODO: once package becomes public, restore the commented out code for having the issue-to-line-number functionality - // try { - // lineNumber = issuesToLineNumbers( - // iacFileScanResult.fileContent, - // getFileTypeForLineNumber(iacFileScanResult.fileType), - // cloudConfigPath, - // ); - // } catch (err) { - // // - // } - - return { - ...policy, - id: policy.publicId, - from: [], - name: policy.title, - cloudConfigPath, - isIgnored: false, - iacDescription: { - issue: policy.issue, - impact: policy.impact, - resolve: policy.resolve, - }, - severity: policy.severity, - lineNumber: lineNumber, - }; - }); - return { - result: { - cloudConfigResults: filterPoliciesBySeverity( - formattedIssues, - severityThreshold, - ), - }, - isPrivate: true, - packageManager: 'k8sconfig', - targetFile: iacFileScanResult.filePath, - }; -} - -function groupMultiDocResults( - scanResults: Array, -): Array { - const groupedData = scanResults.reduce((memo, result) => { - if (memo[result.filePath]) { - memo[result.filePath].violatedPolicies = memo[ - result.filePath - ].violatedPolicies.concat(result.violatedPolicies); - } else { - memo[result.filePath] = result; - } - - return memo; - }, {} as IacFileScanResult); - - return Object.values(groupedData); -} - -function filterPoliciesBySeverity( - violatedPolicies: PolicyMetadata[], - severityThreshold?: SEVERITY, -): PolicyMetadata[] { - if (!severityThreshold || severityThreshold === SEVERITY.LOW) { - return violatedPolicies; - } - - const severitiesToInclude = SEVERITIES.slice( - SEVERITIES.indexOf(severityThreshold), - ); - - return violatedPolicies.filter((policy) => - severitiesToInclude.includes(policy.severity), - ); -} diff --git a/src/cli/commands/test/iac-local-execution/types.ts b/src/cli/commands/test/iac-local-execution/types.ts deleted file mode 100644 index 4b46229335..0000000000 --- a/src/cli/commands/test/iac-local-execution/types.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { SEVERITY } from '../../../../lib/snyk-test/common'; -import { IacFileInDirectory } from '../../../../lib/types'; - -export type IacFileMetadata = IacFileInDirectory; -export interface IacFileData extends IacFileMetadata { - jsonContent: Record; - fileContent: string; - docId?: number; -} -export interface IacFileScanResult extends IacFileData { - violatedPolicies: PolicyMetadata[]; -} - -export interface OpaWasmInstance { - evaluate: (data: Record) => { results: PolicyMetadata[] }; - setData: (data: Record) => void; -} - -export interface PolicyMetadata { - id: string; - publicId: string; - type: string; - subType: string; - title: string; - description: string; - severity: SEVERITY; - msg: string; - policyEngineType: 'opa'; - issue: string; - impact: string; - resolve: string; - references: string[]; -} diff --git a/src/cli/commands/test/index.ts b/src/cli/commands/test/index.ts index f40a3b97f2..ec434d76c1 100644 --- a/src/cli/commands/test/index.ts +++ b/src/cli/commands/test/index.ts @@ -51,8 +51,6 @@ import { getDisplayedOutput, } from './formatters/format-test-results'; -import * as iacLocalExecution from './iac-local-execution'; - const debug = Debug('snyk-test'); const SEPARATOR = '\n-------------------------------------------------------\n'; @@ -135,13 +133,7 @@ async function test(...args: MethodArgs): Promise { let res: (TestResult | TestResult[]) | Error; try { - if (options.iac && options.experimental) { - // this path is an experimental feature feature for IaC which does issue scanning locally without sending files to our Backend servers. - // once ready for GA, it is aimed to deprecate our remote-processing model, so IaC file scanning in the CLI is done locally. - res = await iacLocalExecution.test(path, options); - } else { - res = await snyk.test(path, testOpts); - } + res = await snyk.test(path, testOpts); if (testOpts.iacDirFiles) { options.iacDirFiles = testOpts.iacDirFiles; } diff --git a/src/lib/snyk-test/common.ts b/src/lib/snyk-test/common.ts index 6362b7d55d..b6549973e2 100644 --- a/src/lib/snyk-test/common.ts +++ b/src/lib/snyk-test/common.ts @@ -20,7 +20,7 @@ export function assembleQueryString(options) { return Object.keys(qs).length !== 0 ? qs : null; } -export enum SEVERITY { +enum SEVERITY { LOW = 'low', MEDIUM = 'medium', HIGH = 'high', diff --git a/test/smoke/.iac-data/data.json b/test/smoke/.iac-data/data.json deleted file mode 100644 index 5b960981f5..0000000000 --- a/test/smoke/.iac-data/data.json +++ /dev/null @@ -1 +0,0 @@ -{".circleci":{"jobs":{"run_tests":{"docker":[{"image":"circleci/golang:1.14"}],"steps":["checkout",{"restore_cache":{"keys":["go-mod-v4-{{ checksum \"go.sum\" }}"]}},{"run":"go build"},{"save_cache":{"key":"go-mod-v4-{{ checksum \"go.sum\" }}","paths":["/go/pkg/mod"]}},{"run":{"command":"./cloud-config-opa-policies test .\n","name":"Run Tests"}}]}},"version":2.1,"workflows":{"build_and_push":{"jobs":["run_tests"]},"version":2}},"ecosystems":{"kubernetes":{"SNYK_CC_K8S_1":{"description":"","id":"1","impact":"Compromised container could potentially modify the underlying host’s kernel by loading unauthorized modules (i.e. drivers).","issue":"Container is running in privileged mode","policyEngineType":"opa","publicId":"SNYK-CC-K8S-1","references":["CIS Kubernetes Benchmark 1.6.0 - 5.2.1 Minimize the admission of privileged containers","https://kubernetes.io/docs/concepts/policy/pod-security-policy/#privileged","https://kubernetes.io/blog/2016/08/security-best-practices-kubernetes-deployment/"],"resolve":"Remove `securityContext.privileged` attribute, or set value to `false`","severity":"high","subType":"Deployment","title":"Container is running in privileged mode","type":"k8s"}}}} diff --git a/test/smoke/.iac-data/policy.wasm b/test/smoke/.iac-data/policy.wasm deleted file mode 100644 index e78f98ff48..0000000000 Binary files a/test/smoke/.iac-data/policy.wasm and /dev/null differ diff --git a/test/smoke/spec/iac/snyk_test_local_exec_spec.sh b/test/smoke/spec/iac/snyk_test_local_exec_spec.sh deleted file mode 100644 index a00260d4f7..0000000000 --- a/test/smoke/spec/iac/snyk_test_local_exec_spec.sh +++ /dev/null @@ -1,50 +0,0 @@ -#shellcheck shell=sh - -Describe "Snyk iac test --experimental command" - Before snyk_login - After snyk_logout - - Describe "k8s single file scan" - It "finds issues in k8s file" - When run snyk iac test ../fixtures/iac/kubernetes/pod-privileged.yaml --experimental - The status should be failure # issues found - The output should include "Testing ../fixtures/iac/kubernetes/pod-privileged.yaml..." - - # Outputs issues - The output should include "Infrastructure as code issues:" - The output should include "✗ Container is running in privileged mode [High Severity] [SNYK-CC-K8S-1] in Deployment" - The output should include " introduced by input > spec > containers[example] > securityContext > privileged" - End - - It "filters out issues when using severity threshold" - When run snyk iac test ../fixtures/iac/kubernetes/pod-privileged.yaml --experimental --severity-threshold=high - The status should be failure # one issue found - The output should include "Testing ../fixtures/iac/kubernetes/pod-privileged.yaml..." - - The output should include "Infrastructure as code issues:" - The output should include "✗ Container is running in privileged mode [High Severity] [SNYK-CC-K8S-1] in Deployment" - The output should include "introduced by input > spec > containers[example] > securityContext > privileged" - End - - It "outputs an error for files with no valid k8s objects" - When run snyk iac test ../fixtures/iac/kubernetes/pod-invalid.yaml --experimental - The status should be failure - The output should include "Invalid K8s File!" - End - - It "outputs the expected text when running with --sarif flag" - When run snyk iac test ../fixtures/iac/kubernetes/pod-privileged.yaml --experimental --sarif - The status should be failure - The output should include '"id": "SNYK-CC-K8S-1",' - The output should include '"ruleId": "SNYK-CC-K8S-1",' - End - - It "outputs the expected text when running with --json flag" - When run snyk iac test ../fixtures/iac/kubernetes/pod-privileged.yaml --experimental --json - The status should be failure - The output should include '"id": "SNYK-CC-K8S-1",' - The output should include '"packageManager": "k8sconfig",' - The result of function check_valid_json should be success - End - End -End