refactor: create only one project group per execution

snyk · Jun 1, 2022 · fd3c0ce · fd3c0ce
1 parent 7887ba7
commit fd3c0ce
Show file tree

Hide file tree

Showing 22 changed files with 959 additions and 176 deletions.
diff --git a/src/cli/commands/test/iac/index.ts b/src/cli/commands/test/iac/index.ts
@@ -1,28 +1,16 @@
 import * as Debug from 'debug';
 import { EOL } from 'os';
-import * as cloneDeep from 'lodash.clonedeep';
-import * as assign from 'lodash.assign';
 import chalk from 'chalk';
 
-import {
-  IacFileInDirectory,
-  IacOutputMeta,
-  Options,
-  TestOptions,
-} from '../../../../lib/types';
 import { MethodArgs } from '../../../args';
 import { TestCommandResult } from '../../types';
-import {
-  LegacyVulnApiResult,
-  TestResult,
-} from '../../../../lib/snyk-test/legacy';
+import { LegacyVulnApiResult } from '../../../../lib/snyk-test/legacy';
 import { mapIacTestResult } from '../../../../lib/snyk-test/iac-test-result';
 
 import {
   summariseErrorResults,
   summariseVulnerableResults,
 } from '../../../../lib/formatters';
-import * as utils from '../utils';
 import {
   failuresTipOutput,
   formatIacTestFailures,
@@ -33,43 +21,30 @@ import {
   getIacDisplayErrorFileOutput,
   iacTestTitle,
   shouldLogUserMessages,
-  spinnerMessage,
   spinnerSuccessMessage,
   IaCTestFailure,
 } from '../../../../lib/formatters/iac-output';
 import { extractDataToSendFromResults } from '../../../../lib/formatters/test/format-test-results';
 
-import { test as iacTest } from './local-execution';
 import { validateCredentials } from '../validate-credentials';
 import { validateTestOptions } from '../validate-test-options';
 import { setDefaultTestOptions } from '../set-default-test-options';
 import { processCommandArgs } from '../../process-command-args';
-import { formatTestError } from '../format-test-error';
 import { displayResult } from '../../../../lib/formatters/test/display-result';
 
-import {
-  assertIaCOptionsFlags,
-  isIacShareResultsOptions,
-} from './local-execution/assert-iac-options-flag';
+import { isIacShareResultsOptions } from './local-execution/assert-iac-options-flag';
 import { hasFeatureFlag } from '../../../../lib/feature-flags';
-import {
-  buildDefaultOciRegistry,
-  initRules,
-} from './local-execution/rules/rules';
-import {
-  cleanLocalCache,
-  getIacOrgSettings,
-} from './local-execution/measurable-methods';
+import { buildDefaultOciRegistry } from './local-execution/rules/rules';
+import { getIacOrgSettings } from './local-execution/measurable-methods';
 import config from '../../../../lib/config';
 import { UnsupportedEntitlementError } from '../../../../lib/errors/unsupported-entitlement-error';
 import * as ora from 'ora';
 import { CustomError, FormattedCustomError } from '../../../../lib/errors';
+import { scan } from './scan';
 
 const debug = Debug('snyk-test');
 const SEPARATOR = '\n-------------------------------------------------------\n';
 
-// TODO: avoid using `as any` whenever it's possible
-
 // The hardcoded `isReportCommand` argument is temporary and will be removed together with the `snyk iac report` command deprecation
 export default async function(
   isReportCommand: boolean,
@@ -88,18 +63,10 @@ export default async function(
     throw new UnsupportedEntitlementError('infrastructureAsCode');
   }
 
-  const ociRegistryBuilder = () => buildDefaultOciRegistry(iacOrgSettings);
+  const buildOciRegistry = () => buildDefaultOciRegistry(iacOrgSettings);
 
   let testSpinner: ora.Ora | undefined;
 
-  const resultOptions: Array<Options & TestOptions> = [];
-  const results = [] as any[];
-
-  // Holds an array of scanned file metadata for output.
-  let iacScanFailures: IacFileInDirectory[] = [];
-  let iacIgnoredIssuesCount = 0;
-  let iacOutputMeta: IacOutputMeta | undefined;
-
   const isNewIacOutputSupported =
     config.IAC_OUTPUT_V2 ||
     (await hasFeatureFlag('iacCliOutputRelease', options));
@@ -110,76 +77,25 @@ export default async function(
     testSpinner = ora({ isSilent: options.quiet, stream: process.stdout });
   }
 
-  try {
-    const rulesOrigin = await initRules(
-      ociRegistryBuilder,
-      iacOrgSettings,
-      options,
-    );
-
-    testSpinner?.start(spinnerMessage);
-
-    for (const path of paths) {
-      // Create a copy of the options so a specific test can
-      // modify them i.e. add `options.file` etc. We'll need
-      // these options later.
-      const testOpts = cloneDeep(options);
-      testOpts.path = path;
-      testOpts.projectName = testOpts['project-name'];
-
-      let res: (TestResult | TestResult[]) | Error;
-      try {
-        assertIaCOptionsFlags(process.argv);
-        const { results, failures, ignoreCount } = await iacTest(
-          path,
-          testOpts,
-          orgPublicId,
-          iacOrgSettings,
-          rulesOrigin,
-        );
-        iacOutputMeta = {
-          orgName: results[0]?.org,
-          projectName: results[0]?.projectName,
-          gitRemoteUrl: results[0]?.meta?.gitRemoteUrl,
-        };
-
-        res = results;
-        iacScanFailures = [...iacScanFailures, ...(failures || [])];
-        iacIgnoredIssuesCount += ignoreCount;
-      } catch (error) {
-        res = formatTestError(error);
-      }
-
-      // Not all test results are arrays in order to be backwards compatible
-      // with scripts that use a callback with test. Coerce results/errors to be arrays
-      // and add the result options to each to be displayed
-      const resArray: any[] = Array.isArray(res) ? res : [res];
-
-      for (let i = 0; i < resArray.length; i++) {
-        const pathWithOptionalProjectName = utils.getPathWithOptionalProjectName(
-          path,
-          resArray[i],
-        );
-        results.push(
-          assign(resArray[i], { path: pathWithOptionalProjectName }),
-        );
-        // currently testOpts are identical for each test result returned even if it's for multiple projects.
-        // we want to return the project names, so will need to be crafty in a way that makes sense.
-        if (!testOpts.projectNames) {
-          resultOptions.push(testOpts);
-        } else {
-          resultOptions.push(
-            assign(cloneDeep(testOpts), {
-              projectName: testOpts.projectNames[i],
-            }),
-          );
-        }
-      }
-    }
-  } finally {
-    cleanLocalCache();
+  if (!iacOrgSettings.entitlements?.infrastructureAsCode) {
+    throw new UnsupportedEntitlementError('infrastructureAsCode');
   }
 
+  const {
+    iacOutputMeta,
+    iacScanFailures,
+    iacIgnoredIssuesCount,
+    results,
+    resultOptions,
+  } = await scan(
+    iacOrgSettings,
+    options,
+    testSpinner,
+    paths,
+    orgPublicId,
+    buildOciRegistry,
+  );
+
   // this is any[] to follow the resArray type above
   const successResults: any[] = [],
     errorResults: any[] = [];

diff --git a/src/cli/commands/test/iac/local-execution/index.ts b/src/cli/commands/test/iac/local-execution/index.ts
@@ -23,7 +23,7 @@ import {
 } from './measurable-methods';
 import { findAndLoadPolicy } from '../../../../../lib/policy';
 import { NoFilesToScanError } from './file-loader';
-import { processResults } from './process-results';
+import { ResultsProcessor } from './process-results';
 import { generateProjectAttributes, generateTags } from '../../../monitor';
 import {
   getAllDirectoriesForPath,
@@ -35,9 +35,9 @@ import { getErrorStringCode } from './error-utils';
 // this method executes the local processing engine and then formats the results to adapt with the CLI output.
 // this flow is the default GA flow for IAC scanning.
 export async function test(
+  resultsProcessor: ResultsProcessor,
   pathToScan: string,
   options: IaCTestFlags,
-  orgPublicId: string,
   iacOrgSettings: IacOrgSettings,
   rulesOrigin: RulesOrigin,
 ): Promise<TestReturnValue> {
@@ -110,15 +110,11 @@ export async function test(
     iacOrgSettings.customPolicies,
   );
 
-  const { filteredIssues, ignoreCount } = await processResults(
+  const { filteredIssues, ignoreCount } = await resultsProcessor.processResults(
     resultsWithCustomSeverities,
-    orgPublicId,
-    iacOrgSettings,
     policy,
     tags,
     attributes,
-    options,
-    pathToScan,
   );
 
   try {

diff --git a/src/cli/commands/test/iac/local-execution/measurable-methods.ts b/src/cli/commands/test/iac/local-execution/measurable-methods.ts
@@ -1,6 +1,7 @@
 import { parseFiles } from './file-parser';
 import { scanFiles } from './file-scanner';
-import { formatScanResults } from './process-results/results-formatter';
+import { formatScanResults as formatScanResultsV1 } from './process-results/v1/results-formatter';
+import { formatScanResults as formatScanResultsV2 } from './process-results/v2/results-formatter';
 import { trackUsage } from './usage-tracking';
 import { cleanLocalCache, initLocalCache } from './local-cache';
 import { applyCustomSeverities } from './org-settings/apply-custom-severities';
@@ -82,8 +83,13 @@ const measurableCleanLocalCache = performanceAnalyticsDecorator(
   PerformanceAnalyticsKey.CacheCleanup,
 );
 
-const measurableFormatScanResults = performanceAnalyticsDecorator(
-  formatScanResults,
+const measurableFormatScanResultsV1 = performanceAnalyticsDecorator(
+  formatScanResultsV1,
+  PerformanceAnalyticsKey.ResultFormatting,
+);
+
+const measurableFormatScanResultsV2 = performanceAnalyticsDecorator(
+  formatScanResultsV2,
   PerformanceAnalyticsKey.ResultFormatting,
 );
 
@@ -109,7 +115,8 @@ export {
   measurableScanFiles as scanFiles,
   measurableGetIacOrgSettings as getIacOrgSettings,
   measurableApplyCustomSeverities as applyCustomSeverities,
-  measurableFormatScanResults as formatScanResults,
+  measurableFormatScanResultsV1 as formatScanResultsV1,
+  measurableFormatScanResultsV2 as formatScanResultsV2,
   measurableTrackUsage as trackUsage,
   measurableCleanLocalCache as cleanLocalCache,
   measurableLocalTest as localTest,

diff --git a/src/cli/commands/test/iac/local-execution/process-results/index.ts b/src/cli/commands/test/iac/local-execution/process-results/index.ts
@@ -1,6 +1,3 @@
-import { filterIgnoredIssues } from './policy';
-import { formatAndShareResults } from './share-results';
-import { formatScanResults } from '../measurable-methods';
 import { Policy } from '../../../../../../lib/policy/find-and-load-policy';
 import { ProjectAttributes, Tag } from '../../../../../../lib/types';
 import {
@@ -9,42 +6,71 @@ import {
   IacOrgSettings,
   IaCTestFlags,
 } from '../types';
+import { processResults as processResultsV2 } from './v2';
+import { processResults as processResultsV1 } from './v1';
 
-export async function processResults(
-  resultsWithCustomSeverities: IacFileScanResult[],
-  orgPublicId: string,
-  iacOrgSettings: IacOrgSettings,
-  policy: Policy | undefined,
-  tags: Tag[] | undefined,
-  attributes: ProjectAttributes | undefined,
-  options: IaCTestFlags,
-  pathToScan: string,
-): Promise<{
-  filteredIssues: FormattedResult[];
-  ignoreCount: number;
-}> {
-  let projectPublicIds: Record<string, string> = {};
-  let gitRemoteUrl: string | undefined;
+export interface ResultsProcessor {
+  processResults(
+    resultsWithCustomSeverities: IacFileScanResult[],
+    policy: Policy | undefined,
+    tags: Tag[] | undefined,
+    attributes: ProjectAttributes | undefined,
+  ): Promise<{
+    filteredIssues: FormattedResult[];
+    ignoreCount: number;
+  }>;
+}
+
+export class SingleGroupResultsProcessor implements ResultsProcessor {
+  constructor(
+    private projectRoot: string,
+    private orgPublicId: string,
+    private iacOrgSettings: IacOrgSettings,
+    private options: IaCTestFlags,
+  ) {}
 
-  if (options.report) {
-    ({ projectPublicIds, gitRemoteUrl } = await formatAndShareResults({
-      results: resultsWithCustomSeverities,
-      options,
-      orgPublicId,
+  processResults(
+    resultsWithCustomSeverities: IacFileScanResult[],
+    policy: Policy | undefined,
+    tags: Tag[] | undefined,
+    attributes: ProjectAttributes | undefined,
+  ): Promise<{ filteredIssues: FormattedResult[]; ignoreCount: number }> {
+    return processResultsV2(
+      resultsWithCustomSeverities,
+      this.orgPublicId,
+      this.iacOrgSettings,
       policy,
       tags,
       attributes,
-      pathToScan,
-    }));
+      this.options,
+      this.projectRoot,
+    );
   }
+}
 
-  const formattedResults = formatScanResults(
-    resultsWithCustomSeverities,
-    options,
-    iacOrgSettings.meta,
-    projectPublicIds,
-    gitRemoteUrl,
-  );
+export class MultipleGroupsResultsProcessor implements ResultsProcessor {
+  constructor(
+    private pathToScan: string,
+    private orgPublicId: string,
+    private iacOrgSettings: IacOrgSettings,
+    private options: IaCTestFlags,
+  ) {}
 
-  return filterIgnoredIssues(policy, formattedResults);
+  processResults(
+    resultsWithCustomSeverities: IacFileScanResult[],
+    policy: Policy | undefined,
+    tags: Tag[] | undefined,
+    attributes: ProjectAttributes | undefined,
+  ): Promise<{ filteredIssues: FormattedResult[]; ignoreCount: number }> {
+    return processResultsV1(
+      resultsWithCustomSeverities,
+      this.orgPublicId,
+      this.iacOrgSettings,
+      policy,
+      tags,
+      attributes,
+      this.options,
+      this.pathToScan,
+    );
+  }
 }
diff --git a/...on/process-results/extract-line-number.ts → ...process-results/v1/extract-line-number.ts b/...on/process-results/extract-line-number.ts → ...process-results/v1/extract-line-number.ts
@@ -1,14 +1,14 @@
-import { IaCErrorCodes } from '../types';
-import { CustomError } from '../../../../../../lib/errors';
+import { IaCErrorCodes } from '../../types';
+import { CustomError } from '../../../../../../../lib/errors';
 import {
   CloudConfigFileTypes,
   MapsDocIdToTree,
   getLineNumber,
 } from '@snyk/cloud-config-parser';
-import { UnsupportedFileTypeError } from '../file-parser';
-import * as analytics from '../../../../../../lib/analytics';
+import { UnsupportedFileTypeError } from '../../file-parser';
+import * as analytics from '../../../../../../../lib/analytics';
 import * as Debug from 'debug';
-import { getErrorStringCode } from '../error-utils';
+import { getErrorStringCode } from '../../error-utils';
 const debug = Debug('iac-extract-line-number');
 
 export function getFileTypeForParser(fileType: string): CloudConfigFileTypes {