Skip to content

Commit

Permalink
fix(lambda-nodejs): esbuild detection with Yarn 2 in PnP mode (aws#14739
Browse files Browse the repository at this point in the history
)
  • Loading branch information
jogold authored and hollanddd committed Aug 26, 2021
1 parent dcfc0ae commit 007d326
Show file tree
Hide file tree
Showing 9 changed files with 317 additions and 194 deletions.
170 changes: 94 additions & 76 deletions packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@ import * as os from 'os';
import * as path from 'path';
import { AssetCode, Code, Runtime } from '@aws-cdk/aws-lambda';
import * as cdk from '@aws-cdk/core';
import { EsbuildInstallation } from './esbuild-installation';
import { PackageManager } from './package-manager';
import { BundlingOptions } from './types';
import { exec, extractDependencies, findUp, getEsBuildVersion, LockFile } from './util';
import { exec, extractDependencies, findUp } from './util';

const ESBUILD_VERSION = '0';
const ESBUILD_MAJOR_VERSION = '0';

/**
* Bundling properties
Expand Down Expand Up @@ -41,11 +43,11 @@ export class Bundling implements cdk.BundlingOptions {
});
}

public static clearRunsLocallyCache(): void {
this.runsLocally = undefined;
public static clearEsbuildInstallationCache(): void {
this.esbuildInstallation = undefined;
}

private static runsLocally?: boolean;
private static esbuildInstallation?: EsbuildInstallation;

// Core bundling options
public readonly image: cdk.DockerImage;
Expand All @@ -54,20 +56,22 @@ export class Bundling implements cdk.BundlingOptions {
public readonly workingDirectory: string;
public readonly local?: cdk.ILocalBundling;

private readonly projectRoot: string;
private readonly relativeEntryPath: string;
private readonly relativeTsconfigPath?: string;
private readonly externals: string[];
private readonly packageManager: PackageManager;

constructor(private readonly props: BundlingProps) {
Bundling.runsLocally = Bundling.runsLocally
?? getEsBuildVersion()?.startsWith(ESBUILD_VERSION)
?? false;
this.packageManager = PackageManager.fromLockFile(props.depsLockFilePath);

const projectRoot = path.dirname(props.depsLockFilePath);
this.relativeEntryPath = path.relative(projectRoot, path.resolve(props.entry));
Bundling.esbuildInstallation = Bundling.esbuildInstallation ?? EsbuildInstallation.detect();

this.projectRoot = path.dirname(props.depsLockFilePath);
this.relativeEntryPath = path.relative(this.projectRoot, path.resolve(props.entry));

if (props.tsconfig) {
this.relativeTsconfigPath = path.relative(projectRoot, path.resolve(props.tsconfig));
this.relativeTsconfigPath = path.relative(this.projectRoot, path.resolve(props.tsconfig));
}

this.externals = [
Expand All @@ -76,18 +80,23 @@ export class Bundling implements cdk.BundlingOptions {
];

// Docker bundling
const shouldBuildImage = props.forceDockerBundling || !Bundling.runsLocally;
const shouldBuildImage = props.forceDockerBundling || !Bundling.esbuildInstallation;
this.image = shouldBuildImage
? props.dockerImage ?? cdk.DockerImage.fromBuild(path.join(__dirname, '../lib'), {
buildArgs: {
...props.buildArgs ?? {},
IMAGE: props.runtime.bundlingDockerImage.image,
ESBUILD_VERSION: props.esbuildVersion ?? ESBUILD_VERSION,
IMAGE: props.runtime.bundlingImage.image,
ESBUILD_VERSION: props.esbuildVersion ?? ESBUILD_MAJOR_VERSION,
},
})
: cdk.DockerImage.fromRegistry('dummy'); // Do not build if we don't need to

const bundlingCommand = this.createBundlingCommand(cdk.AssetStaging.BUNDLING_INPUT_DIR, cdk.AssetStaging.BUNDLING_OUTPUT_DIR);
const bundlingCommand = this.createBundlingCommand({
inputDir: cdk.AssetStaging.BUNDLING_INPUT_DIR,
outputDir: cdk.AssetStaging.BUNDLING_OUTPUT_DIR,
esbuildRunner: 'esbuild', // esbuild is installed globally in the docker image
osPlatform: 'linux', // linux docker image
});
this.command = ['bash', '-c', bundlingCommand];
this.environment = props.environment;
// Bundling sets the working directory to cdk.AssetStaging.BUNDLING_INPUT_DIR
Expand All @@ -96,66 +105,34 @@ export class Bundling implements cdk.BundlingOptions {

// Local bundling
if (!props.forceDockerBundling) { // only if Docker is not forced
const osPlatform = os.platform();
const createLocalCommand = (outputDir: string) => this.createBundlingCommand(projectRoot, outputDir, osPlatform);

this.local = {
tryBundle(outputDir: string) {
if (Bundling.runsLocally === false) {
process.stderr.write('esbuild cannot run locally. Switching to Docker bundling.\n');
return false;
}

const localCommand = createLocalCommand(outputDir);

exec(
osPlatform === 'win32' ? 'cmd' : 'bash',
[
osPlatform === 'win32' ? '/c' : '-c',
localCommand,
],
{
env: { ...process.env, ...props.environment ?? {} },
stdio: [ // show output
'ignore', // ignore stdio
process.stderr, // redirect stdout to stderr
'inherit', // inherit stderr
],
cwd: path.dirname(props.entry),
windowsVerbatimArguments: osPlatform === 'win32',
});

return true;
},
};
this.local = this.getLocalBundlingProvider();
}
}

public createBundlingCommand(inputDir: string, outputDir: string, osPlatform: NodeJS.Platform = 'linux'): string {
const pathJoin = osPathJoin(osPlatform);
private createBundlingCommand(options: BundlingCommandOptions): string {
const pathJoin = osPathJoin(options.osPlatform);

const npx = osPlatform === 'win32' ? 'npx.cmd' : 'npx';
const loaders = Object.entries(this.props.loader ?? {});
const defines = Object.entries(this.props.define ?? {});

const esbuildCommand: string = [
npx, 'esbuild',
'--bundle', `"${pathJoin(inputDir, this.relativeEntryPath)}"`,
const esbuildCommand: string[] = [
options.esbuildRunner,
'--bundle', `"${pathJoin(options.inputDir, this.relativeEntryPath)}"`,
`--target=${this.props.target ?? toTarget(this.props.runtime)}`,
'--platform=node',
`--outfile="${pathJoin(outputDir, 'index.js')}"`,
`--outfile="${pathJoin(options.outputDir, 'index.js')}"`,
...this.props.minify ? ['--minify'] : [],
...this.props.sourceMap ? ['--sourcemap'] : [],
...this.externals.map(external => `--external:${external}`),
...loaders.map(([ext, name]) => `--loader:${ext}=${name}`),
...defines.map(([key, value]) => `--define:${key}=${JSON.stringify(value)}`),
...this.props.logLevel ? [`--log-level=${this.props.logLevel}`] : [],
...this.props.keepNames ? ['--keep-names'] : [],
...this.relativeTsconfigPath ? [`--tsconfig=${pathJoin(inputDir, this.relativeTsconfigPath)}`] : [],
...this.props.metafile ? [`--metafile=${pathJoin(outputDir, 'index.meta.json')}`] : [],
...this.relativeTsconfigPath ? [`--tsconfig=${pathJoin(options.inputDir, this.relativeTsconfigPath)}`] : [],
...this.props.metafile ? [`--metafile=${pathJoin(options.outputDir, 'index.meta.json')}`] : [],
...this.props.banner ? [`--banner:js=${JSON.stringify(this.props.banner)}`] : [],
...this.props.footer ? [`--footer:js=${JSON.stringify(this.props.footer)}`] : [],
].join(' ');
];

let depsCommand = '';
if (this.props.nodeModules) {
Expand All @@ -168,37 +145,78 @@ export class Bundling implements cdk.BundlingOptions {

// Determine dependencies versions, lock file and installer
const dependencies = extractDependencies(pkgPath, this.props.nodeModules);
let installer = Installer.NPM;
let lockFile = LockFile.NPM;
if (this.props.depsLockFilePath.endsWith(LockFile.YARN)) {
lockFile = LockFile.YARN;
installer = Installer.YARN;
}

const osCommand = new OsCommand(osPlatform);
const osCommand = new OsCommand(options.osPlatform);

// Create dummy package.json, copy lock file if any and then install
depsCommand = chain([
osCommand.writeJson(pathJoin(outputDir, 'package.json'), { dependencies }),
osCommand.copy(pathJoin(inputDir, lockFile), pathJoin(outputDir, lockFile)),
osCommand.changeDirectory(outputDir),
`${installer} install`,
osCommand.writeJson(pathJoin(options.outputDir, 'package.json'), { dependencies }),
osCommand.copy(pathJoin(options.inputDir, this.packageManager.lockFile), pathJoin(options.outputDir, this.packageManager.lockFile)),
osCommand.changeDirectory(options.outputDir),
this.packageManager.installCommand.join(' '),
]);
}

return chain([
...this.props.commandHooks?.beforeBundling(inputDir, outputDir) ?? [],
esbuildCommand,
...(this.props.nodeModules && this.props.commandHooks?.beforeInstall(inputDir, outputDir)) ?? [],
...this.props.commandHooks?.beforeBundling(options.inputDir, options.outputDir) ?? [],
esbuildCommand.join(' '),
...(this.props.nodeModules && this.props.commandHooks?.beforeInstall(options.inputDir, options.outputDir)) ?? [],
depsCommand,
...this.props.commandHooks?.afterBundling(inputDir, outputDir) ?? [],
...this.props.commandHooks?.afterBundling(options.inputDir, options.outputDir) ?? [],
]);
}

private getLocalBundlingProvider(): cdk.ILocalBundling {
const osPlatform = os.platform();
const createLocalCommand = (outputDir: string, esbuild: EsbuildInstallation) => this.createBundlingCommand({
inputDir: this.projectRoot,
outputDir,
esbuildRunner: esbuild.isLocal ? this.packageManager.runBinCommand('esbuild') : 'esbuild',
osPlatform,
});
const environment = this.props.environment ?? {};
const cwd = path.dirname(this.props.entry);

return {
tryBundle(outputDir: string) {
if (!Bundling.esbuildInstallation) {
process.stderr.write('esbuild cannot run locally. Switching to Docker bundling.\n');
return false;
}

if (!Bundling.esbuildInstallation.version.startsWith(`${ESBUILD_MAJOR_VERSION}.`)) {
throw new Error(`Expected esbuild version ${ESBUILD_MAJOR_VERSION}.x but got ${Bundling.esbuildInstallation.version}`);
}

const localCommand = createLocalCommand(outputDir, Bundling.esbuildInstallation);

exec(
osPlatform === 'win32' ? 'cmd' : 'bash',
[
osPlatform === 'win32' ? '/c' : '-c',
localCommand,
],
{
env: { ...process.env, ...environment },
stdio: [ // show output
'ignore', // ignore stdio
process.stderr, // redirect stdout to stderr
'inherit', // inherit stderr
],
cwd,
windowsVerbatimArguments: osPlatform === 'win32',
});

return true;
},
};
}
}

enum Installer {
NPM = 'npm',
YARN = 'yarn',
interface BundlingCommandOptions {
readonly inputDir: string;
readonly outputDir: string;
readonly esbuildRunner: string;
readonly osPlatform: NodeJS.Platform;
}

/**
Expand Down
35 changes: 35 additions & 0 deletions packages/@aws-cdk/aws-lambda-nodejs/lib/esbuild-installation.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
import { spawnSync } from 'child_process';
import { tryGetModuleVersion } from './util';

/**
* An esbuild installation
*/
export abstract class EsbuildInstallation {
public static detect(): EsbuildInstallation | undefined {
try {
// Check local version first
const version = tryGetModuleVersion('esbuild');
if (version) {
return {
isLocal: true,
version,
};
}

// Fallback to a global version
const esbuild = spawnSync('esbuild', ['--version']);
if (esbuild.status === 0 && !esbuild.error) {
return {
isLocal: false,
version: esbuild.stdout.toString().trim(),
};
}
return undefined;
} catch (err) {
return undefined;
}
}

public abstract readonly isLocal: boolean;
public abstract readonly version: string;
}
5 changes: 3 additions & 2 deletions packages/@aws-cdk/aws-lambda-nodejs/lib/function.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@ import * as fs from 'fs';
import * as path from 'path';
import * as lambda from '@aws-cdk/aws-lambda';
import { Bundling } from './bundling';
import { PackageManager } from './package-manager';
import { BundlingOptions } from './types';
import { callsites, findUp, LockFile } from './util';
import { callsites, findUp } from './util';

// keep this import separate from other imports to reduce chance for merge conflicts with v2-main
// eslint-disable-next-line no-duplicate-imports, import/order
Expand Down Expand Up @@ -94,7 +95,7 @@ export class NodejsFunction extends lambda.Function {
}
depsLockFilePath = path.resolve(props.depsLockFilePath);
} else {
const lockFile = findUp(LockFile.YARN) ?? findUp(LockFile.NPM);
const lockFile = findUp(PackageManager.YARN.lockFile) ?? findUp(PackageManager.NPM.lockFile);
if (!lockFile) {
throw new Error('Cannot find a package lock file (`yarn.lock` or `package-lock.json`). Please specify it with `depsFileLockPath`.');
}
Expand Down
57 changes: 57 additions & 0 deletions packages/@aws-cdk/aws-lambda-nodejs/lib/package-manager.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
import * as os from 'os';
import * as path from 'path';

interface PackageManagerProps {
readonly lockFile: string;
readonly installCommand: string[];
readonly runCommand: string[];
}

/**
* A node package manager
*/
export class PackageManager {
public static NPM = new PackageManager({
lockFile: 'package-lock.json',
installCommand: ['npm', 'install'],
runCommand: ['npx', '--no-install'],
});

public static YARN = new PackageManager({
lockFile: 'yarn.lock',
installCommand: ['yarn', 'install'],
runCommand: ['yarn', 'run'],
});

public static fromLockFile(lockFilePath: string): PackageManager {
const lockFile = path.basename(lockFilePath);

switch (lockFile) {
case PackageManager.NPM.lockFile:
return PackageManager.NPM;
case PackageManager.YARN.lockFile:
return PackageManager.YARN;
default:
return PackageManager.NPM;
}
}

public readonly lockFile: string;
public readonly installCommand: string[];
public readonly runCommand: string[];

constructor(props: PackageManagerProps) {
this.lockFile = props.lockFile;
this.installCommand = props.installCommand;
this.runCommand = props.runCommand;
}

public runBinCommand(bin: string): string {
const [runCommand, ...runArgs] = this.runCommand;
return [
os.platform() === 'win32' ? `${runCommand}.cmd` : runCommand,
...runArgs,
bin,
].join(' ');
}
}
Loading

0 comments on commit 007d326

Please sign in to comment.