From f0e51f2c06bb3a7904874cbe46200a9daddc438a Mon Sep 17 00:00:00 2001 From: Liliana Kastilio Date: Fri, 2 Aug 2019 19:58:30 +0100 Subject: [PATCH 1/3] feat: prune graph on test if asked --- src/lib/monitor.ts | 4 ++-- src/lib/snyk-test/run-test.ts | 17 +++++++++++++++-- src/lib/types.ts | 1 + 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/lib/monitor.ts b/src/lib/monitor.ts index 4ad532f04c..3eee0275f7 100644 --- a/src/lib/monitor.ts +++ b/src/lib/monitor.ts @@ -332,14 +332,14 @@ export async function monitorGraph( }); } -function countPathsToGraphRoot(graph: depGraphLib.DepGraph): number { +export function countPathsToGraphRoot(graph: depGraphLib.DepGraph): number { return graph .getPkgs() .map((pkg) => graph.countPathsToRoot(pkg)) .reduce((acc, pathsToRoot) => acc + pathsToRoot, 0); } -async function pruneGraph( +export async function pruneGraph( depGraph: depGraphLib.DepGraph, packageManager: SupportedPackageManagers, ): Promise { diff --git a/src/lib/snyk-test/run-test.ts b/src/lib/snyk-test/run-test.ts index b0e1eba04b..a2a116f799 100644 --- a/src/lib/snyk-test/run-test.ts +++ b/src/lib/snyk-test/run-test.ts @@ -25,6 +25,7 @@ import { } from '../errors'; import { maybePrintDeps } from '../print-deps'; import { SupportedPackageManagers } from '../package-managers'; +import { countPathsToGraphRoot, pruneGraph } from '../monitor'; // tslint:disable-next-line:no-var-requires const debug = require('debug')('snyk'); @@ -72,7 +73,6 @@ async function runTest(packageManager: SupportedPackageManagers, if (payload.body && payload.body.docker && payload.body.docker.dockerfilePackages) { dockerfilePackages = payload.body.docker.dockerfilePackages; } - await spinner.clear(spinnerLbl)(); await spinner(spinnerLbl); analytics.add('depGraph', !!depGraph); analytics.add('isDocker', !!(payload.body && payload.body.docker)); @@ -333,9 +333,22 @@ async function assembleLocalPayloads(root, options: Options & TestOptions): Prom // Graphs are more compact and robust representations. // Legacy parts of the code are still using trees, but will eventually be fully migrated. debug('converting dep-tree to dep-graph', {name: pkg.name, targetFile: depRoot.targetFile || options.file}); - const depGraph = await depGraphLib.legacy.depTreeToGraph( + let depGraph = await depGraphLib.legacy.depTreeToGraph( pkg, options.packageManager); + debug('done converting dep-tree to dep-graph', {uniquePkgsCount: depGraph.getPkgs().length}); + if (options['prune-repeated-subdependencies']) { + debug('Trying to prune the graph'); + const prePruneDepCount = countPathsToGraphRoot(depGraph); + debug('pre prunedPathsCount: ' + prePruneDepCount); + + depGraph = await pruneGraph(depGraph, options.packageManager); + + analytics.add('prePrunedPathsCount', prePruneDepCount); + const postPruneDepCount = countPathsToGraphRoot(depGraph); + debug('post prunedPathsCount: ' + prePruneDepCount); + analytics.add('postPrunedPathsCount', postPruneDepCount); + } body.depGraph = depGraph; } diff --git a/src/lib/types.ts b/src/lib/types.ts index 2f4376a636..c1a14d30c4 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -62,6 +62,7 @@ export function isMultiResult(pet: SingleDepRootResult | MultiDepRootsResult): p export interface TestOptions { traverseNodeModules: boolean; interactive: boolean; + 'prune-repeated-subdependencies'?: boolean; } export interface ProtectOptions { loose: boolean; From 842ef7f7ff57b2818f62bfa4724fc9a9d270237b Mon Sep 17 00:00:00 2001 From: Daniel Kontorovskyi Date: Sun, 4 Aug 2019 17:09:05 +0300 Subject: [PATCH 2/3] feat: move prune logic in separate file --- src/lib/monitor.ts | 41 ++------------------------------- src/lib/prune.ts | 43 +++++++++++++++++++++++++++++++++++ src/lib/snyk-test/run-test.ts | 4 ++-- 3 files changed, 47 insertions(+), 41 deletions(-) create mode 100644 src/lib/prune.ts diff --git a/src/lib/monitor.ts b/src/lib/monitor.ts index 3eee0275f7..67fbaf3320 100644 --- a/src/lib/monitor.ts +++ b/src/lib/monitor.ts @@ -12,7 +12,8 @@ import { SingleDepRootResult, DepTree, MonitorMeta, MonitorResult } from './type import * as projectMetadata from './project-metadata'; import * as path from 'path'; import {MonitorError, ConnectionTimeoutError} from './errors'; -import { SupportedPackageManagers, GRAPH_SUPPORTED_PACKAGE_MANAGERS } from './package-managers'; +import { countPathsToGraphRoot, pruneGraph } from './prune'; +import { GRAPH_SUPPORTED_PACKAGE_MANAGERS } from './package-managers'; const debug = Debug('snyk'); @@ -332,44 +333,6 @@ export async function monitorGraph( }); } -export function countPathsToGraphRoot(graph: depGraphLib.DepGraph): number { - return graph - .getPkgs() - .map((pkg) => graph.countPathsToRoot(pkg)) - .reduce((acc, pathsToRoot) => acc + pathsToRoot, 0); -} - -export async function pruneGraph( - depGraph: depGraphLib.DepGraph, - packageManager: SupportedPackageManagers, - ): Promise { - try { - const threshold = config.PRUNE_DEPS_THRESHOLD; // Arbitrary threshold for maximum number of elements in the tree - const prunedTree: DepTree = (await depGraphLib.legacy.graphToDepTree( - depGraph, - packageManager, - { deduplicateWithinTopLevelDeps: true }, - )) as DepTree; - - const prunedGraph = await depGraphLib.legacy.depTreeToGraph( - prunedTree, - packageManager, - ); - const count = countPathsToGraphRoot(prunedGraph); - debug('prunedPathsCount: ' + count); - - if (count < threshold) { - return prunedGraph; - } - debug('Too many vulnerable paths to monitor the project'); - // TODO: create a custom error for this - throw new Error('Too many vulnerable paths to monitor the project'); - } catch (e) { - debug('Failed to prune the graph, returning original:' + e); - return depGraph; - } -} - function pluckPolicies(pkg) { if (!pkg) { return null; diff --git a/src/lib/prune.ts b/src/lib/prune.ts new file mode 100644 index 0000000000..9ee617568a --- /dev/null +++ b/src/lib/prune.ts @@ -0,0 +1,43 @@ +import * as _debug from 'debug'; +import { DepGraph, legacy } from '@snyk/dep-graph'; + +import { DepTree } from './types'; +import * as config from './config'; +import { SupportedPackageManagers } from './package-managers'; + +const debug = _debug('snyk:prune'); + +const {depTreeToGraph, graphToDepTree} = legacy; + +export function countPathsToGraphRoot(graph: DepGraph): number { + return graph.getPkgs().reduce((acc, pkg) => acc + graph.countPathsToRoot(pkg), 0); +} + +export async function pruneGraph( + depGraph: DepGraph, + packageManager: SupportedPackageManagers, +): Promise { + try { + // Arbitrary threshold for maximum number of elements in the tree + const threshold = config.PRUNE_DEPS_THRESHOLD; + const prunedTree = (await graphToDepTree( + depGraph, + packageManager, + { deduplicateWithinTopLevelDeps: true }, + )) as DepTree; + + const prunedGraph = await depTreeToGraph(prunedTree, packageManager); + const count = countPathsToGraphRoot(prunedGraph); + debug('prunedPathsCount: ' + count); + + if (count < threshold) { + return prunedGraph; + } + + debug('Too many vulnerable paths to process the project'); + throw new Error('Too many vulnerable paths to process the project'); + } catch (e) { + debug('Failed to prune the graph, returning original: ' + e); + return depGraph; + } +} diff --git a/src/lib/snyk-test/run-test.ts b/src/lib/snyk-test/run-test.ts index a2a116f799..ed79154c90 100644 --- a/src/lib/snyk-test/run-test.ts +++ b/src/lib/snyk-test/run-test.ts @@ -25,7 +25,7 @@ import { } from '../errors'; import { maybePrintDeps } from '../print-deps'; import { SupportedPackageManagers } from '../package-managers'; -import { countPathsToGraphRoot, pruneGraph } from '../monitor'; +import { countPathsToGraphRoot, pruneGraph } from '../prune'; // tslint:disable-next-line:no-var-requires const debug = require('debug')('snyk'); @@ -346,7 +346,7 @@ async function assembleLocalPayloads(root, options: Options & TestOptions): Prom analytics.add('prePrunedPathsCount', prePruneDepCount); const postPruneDepCount = countPathsToGraphRoot(depGraph); - debug('post prunedPathsCount: ' + prePruneDepCount); + debug('post prunedPathsCount: ' + postPruneDepCount); analytics.add('postPrunedPathsCount', postPruneDepCount); } body.depGraph = depGraph; From 82e7ca6af82d8c0291c3e1293198792f7679b54c Mon Sep 17 00:00:00 2001 From: Daniel Kontorovskyi Date: Sun, 4 Aug 2019 17:09:39 +0300 Subject: [PATCH 3/3] feat: special error for too many vuln paths --- src/lib/errors/index.ts | 1 + src/lib/errors/too-many-vuln-paths.ts | 14 ++++++++++++++ src/lib/prune.ts | 3 ++- 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 src/lib/errors/too-many-vuln-paths.ts diff --git a/src/lib/errors/index.ts b/src/lib/errors/index.ts index 2e601d0033..06a4262e99 100644 --- a/src/lib/errors/index.ts +++ b/src/lib/errors/index.ts @@ -12,3 +12,4 @@ export {FailedToGetVulnerabilitiesError} from './failed-to-get-vulnerabilities-e export {UnsupportedFeatureFlagError} from './unsupported-feature-flag-error'; export {UnsupportedPackageManagerError} from './unsupported-package-manager-error'; export {FailedToRunTestError} from './failed-to-run-test-error'; +export {TooManyVulnPaths} from './too-many-vuln-paths'; diff --git a/src/lib/errors/too-many-vuln-paths.ts b/src/lib/errors/too-many-vuln-paths.ts new file mode 100644 index 0000000000..6e54bc577d --- /dev/null +++ b/src/lib/errors/too-many-vuln-paths.ts @@ -0,0 +1,14 @@ +import {CustomError} from './custom-error'; + +export class TooManyVulnPaths extends CustomError { + private static ERROR_CODE: number = 413; + private static ERROR_STRING_CODE: string = 'TOO_MANY_VULN_PATHS'; + private static ERROR_MESSAGE: string = 'Too many vulnerable paths to process the project'; + + constructor() { + super(TooManyVulnPaths.ERROR_MESSAGE); + this.code = TooManyVulnPaths.ERROR_CODE; + this.strCode = TooManyVulnPaths.ERROR_STRING_CODE; + this.userMessage = TooManyVulnPaths.ERROR_MESSAGE; + } +} diff --git a/src/lib/prune.ts b/src/lib/prune.ts index 9ee617568a..4d08258246 100644 --- a/src/lib/prune.ts +++ b/src/lib/prune.ts @@ -3,6 +3,7 @@ import { DepGraph, legacy } from '@snyk/dep-graph'; import { DepTree } from './types'; import * as config from './config'; +import { TooManyVulnPaths } from './errors'; import { SupportedPackageManagers } from './package-managers'; const debug = _debug('snyk:prune'); @@ -35,7 +36,7 @@ export async function pruneGraph( } debug('Too many vulnerable paths to process the project'); - throw new Error('Too many vulnerable paths to process the project'); + throw new TooManyVulnPaths(); } catch (e) { debug('Failed to prune the graph, returning original: ' + e); return depGraph;