feat: sequential fix

packages/snyk-fix/src/plugins/python/handlers/ensure-has-updates.ts → packages/snyk-fix/src/plugins/python/handlers/fail-if-no-updates-applied.ts

@@ -6,7 +6,7 @@ import { isSuccessfulChange } from './attempted-changes-summary';
 
 const debug = debugLib('snyk-fix:python:ensure-changes-applied');
 
-export function ensureHasUpdates(changes: FixChangesSummary[]) {
+export function failIfNoUpdatesApplied(changes: FixChangesSummary[]) {
   if (!changes.length) {
     throw new NoFixesCouldBeAppliedError();
   }

packages/snyk-fix/src/plugins/python/handlers/pipenv-pipfile/update-dependencies/generate-upgrades.ts

@@ -0,0 +1,17 @@
+import { EntityToFix } from '../../../../../types';
+import { standardizePackageName } from '../../../standardize-package-name';
+import { validateRequiredData } from '../../validate-required-data';
+
+export function generateUpgrades(entity: EntityToFix): { upgrades: string[] } {
+  const { remediation } = validateRequiredData(entity);
+  const { pin: pins } = remediation;
+
+  const upgrades: string[] = [];
+  for (const pkgAtVersion of Object.keys(pins)) {
+    const pin = pins[pkgAtVersion];
+    const newVersion = pin.upgradeTo.split('@')[1];
+    const [pkgName] = pkgAtVersion.split('@');
+    upgrades.push(`${standardizePackageName(pkgName)}==${newVersion}`);
+  }
+  return { upgrades };
+}

packages/snyk-fix/src/plugins/python/handlers/pipenv-pipfile/update-dependencies/index.ts

@@ -6,37 +6,26 @@ import {
   FixChangesSummary,
   FixOptions,
 } from '../../../../../types';
-import { NoFixesCouldBeAppliedError } from '../../../../../lib/errors/no-fixes-applied';
-import { standardizePackageName } from '../../../standardize-package-name';
-import { validateRequiredData } from '../../validate-required-data';
 
-import { ensureHasUpdates } from '../../ensure-has-updates';
+import { failIfNoUpdatesApplied } from '../../fail-if-no-updates-applied';
+import { NoFixesCouldBeAppliedError } from '../../../../../lib/errors/no-fixes-applied';
+import { generateUpgrades } from './generate-upgrades';
 import { pipenvAdd } from './pipenv-add';
 
 const debug = debugLib('snyk-fix:python:Pipfile');
 
+function chooseFixStrategy(options: FixOptions) {
+  return options.sequentialFix ? fixSequentially : fixAll;
+}
+
 export async function updateDependencies(
   entity: EntityToFix,
   options: FixOptions,
 ): Promise<PluginFixResponse> {
-  const handlerResult = await fixAll(entity, options);
+  const handlerResult = await chooseFixStrategy(options)(entity, options);
   return handlerResult;
 }
 
-export function generateUpgrades(entity: EntityToFix): { upgrades: string[] } {
-  const { remediation } = validateRequiredData(entity);
-  const { pin: pins } = remediation;
-
-  const upgrades: string[] = [];
-  for (const pkgAtVersion of Object.keys(pins)) {
-    const pin = pins[pkgAtVersion];
-    const newVersion = pin.upgradeTo.split('@')[1];
-    const [pkgName] = pkgAtVersion.split('@');
-    upgrades.push(`${standardizePackageName(pkgName)}==${newVersion}`);
-  }
-  return { upgrades };
-}
-
 async function fixAll(
   entity: EntityToFix,
   options: FixOptions,
@@ -46,14 +35,14 @@ async function fixAll(
     failed: [],
     skipped: [],
   };
-  const { upgrades } = await generateUpgrades(entity);
-  if (!upgrades.length) {
-    throw new NoFixesCouldBeAppliedError(
-      'Failed to calculate package updates to apply',
-    );
-  }
   const changes: FixChangesSummary[] = [];
   try {
+    const { upgrades } = await generateUpgrades(entity);
+    if (!upgrades.length) {
+      throw new NoFixesCouldBeAppliedError(
+        'Failed to calculate package updates to apply',
+      );
+    }
     // TODO: for better support we need to:
     // 1. parse the manifest and extract original requirements, version spec etc
     // 2. swap out only the version and retain original spec
@@ -64,7 +53,7 @@ async function fixAll(
       changes.push(...(await pipenvAdd(entity, options, upgrades)));
     }
 
-    ensureHasUpdates(changes);
+    failIfNoUpdatesApplied(changes);
     handlerResult.succeeded.push({
       original: entity,
       changes,
@@ -81,3 +70,53 @@ async function fixAll(
   }
   return handlerResult;
 }
+
+async function fixSequentially(
+  entity: EntityToFix,
+  options: FixOptions,
+): Promise<PluginFixResponse> {
+  const handlerResult: PluginFixResponse = {
+    succeeded: [],
+    failed: [],
+    skipped: [],
+  };
+  const { upgrades } = await generateUpgrades(entity);
+  // TODO: for better support we need to:
+  // 1. parse the manifest and extract original requirements, version spec etc
+  // 2. swap out only the version and retain original spec
+  // 3. re-lock the lockfile
+  // at the moment we do not parse Pipfile and therefore can't tell the difference
+  // between prod & dev updates
+  const changes: FixChangesSummary[] = [];
+
+  try {
+    if (!upgrades.length) {
+      throw new NoFixesCouldBeAppliedError(
+        'Failed to calculate package updates to apply',
+      );
+    }
+    // update prod dependencies first
+    if (upgrades.length) {
+      for (const upgrade of upgrades) {
+        changes.push(...(await pipenvAdd(entity, options, [upgrade])));
+      }
+    }
+
+    failIfNoUpdatesApplied(changes);
+
+    handlerResult.succeeded.push({
+      original: entity,
+      changes,
+    });
+  } catch (error) {
+    debug(
+      `Failed to fix ${entity.scanResult.identity.targetFile}.\nERROR: ${error}`,
+    );
+    handlerResult.failed.push({
+      original: entity,
+      tip: error.tip,
+      error,
+    });
+  }
+  return handlerResult;
+}

packages/snyk-fix/src/plugins/python/handlers/poetry/update-dependencies/generate-upgrades.ts

@@ -0,0 +1,70 @@
+import * as pathLib from 'path';
+import * as toml from 'toml';
+
+import * as debugLib from 'debug';
+
+import { EntityToFix } from '../../../../../types';
+
+import { validateRequiredData } from '../../validate-required-data';
+import { standardizePackageName } from '../../../standardize-package-name';
+
+const debug = debugLib('snyk-fix:python:Poetry');
+
+interface PyProjectToml {
+  tool: {
+    poetry: {
+      name: string;
+      version: string;
+      description: string;
+      authors: string[];
+      dependencies?: object;
+      'dev-dependencies'?: object;
+    };
+  };
+}
+
+export async function generateUpgrades(
+  entity: EntityToFix,
+): Promise<{ upgrades: string[]; devUpgrades: string[] }> {
+  const { remediation, targetFile } = validateRequiredData(entity);
+  const pins = remediation.pin;
+
+  const targetFilePath = pathLib.resolve(entity.workspace.path, targetFile);
+  const { dir } = pathLib.parse(targetFilePath);
+  const pyProjectTomlRaw = await entity.workspace.readFile(
+    pathLib.resolve(dir, 'pyproject.toml'),
+  );
+  const pyProjectToml: PyProjectToml = toml.parse(pyProjectTomlRaw);
+
+  const prodTopLevelDeps = Object.keys(
+    pyProjectToml.tool.poetry.dependencies ?? {},
+  ).map((dep) => standardizePackageName(dep));
+  const devTopLevelDeps = Object.keys(
+    pyProjectToml.tool.poetry['dev-dependencies'] ?? {},
+  ).map((dep) => standardizePackageName(dep));
+
+  const upgrades: string[] = [];
+  const devUpgrades: string[] = [];
+  for (const pkgAtVersion of Object.keys(pins)) {
+    const pin = pins[pkgAtVersion];
+    const newVersion = pin.upgradeTo.split('@')[1];
+    const [pkgName] = pkgAtVersion.split('@');
+
+    const upgrade = `${standardizePackageName(pkgName)}==${newVersion}`;
+
+    if (pin.isTransitive || prodTopLevelDeps.includes(pkgName)) {
+      // transitive and it could have come from a dev or prod dep
+      // since we can't tell right now let be pinned into production deps
+      upgrades.push(upgrade);
+    } else if (prodTopLevelDeps.includes(pkgName)) {
+      upgrades.push(upgrade);
+    } else if (entity.options.dev && devTopLevelDeps.includes(pkgName)) {
+      devUpgrades.push(upgrade);
+    } else {
+      debug(
+        `Could not determine what type of upgrade ${upgrade} is. When choosing between: transitive upgrade, production or dev direct upgrade. `,
+      );
+    }
+  }
+  return { upgrades, devUpgrades };
+}