diff --git a/packages/@aws-cdk/assets/lib/fs/copy.ts b/packages/@aws-cdk/assets/lib/fs/copy.ts index 543b64a4e5f22..0796ca5698c96 100644 --- a/packages/@aws-cdk/assets/lib/fs/copy.ts +++ b/packages/@aws-cdk/assets/lib/fs/copy.ts @@ -2,11 +2,11 @@ import fs = require('fs'); import path = require('path'); import { CopyOptions } from './copy-options'; import { FollowMode } from './follow-mode'; -import { shouldExclude, shouldFollow } from './utils'; +import { mkdirpSync } from './mkdirpSync'; +import { listFilesRecursively, shouldFollow } from './utils'; export function copyDirectory(srcDir: string, destDir: string, options: CopyOptions = { }, rootDir?: string) { const follow = options.follow !== undefined ? options.follow : FollowMode.EXTERNAL; - const exclude = options.exclude || []; rootDir = rootDir || srcDir; @@ -14,44 +14,26 @@ export function copyDirectory(srcDir: string, destDir: string, options: CopyOpti throw new Error(`${srcDir} is not a directory`); } - const files = fs.readdirSync(srcDir); - for (const file of files) { - const sourceFilePath = path.join(srcDir, file); + for (const assetFile of listFilesRecursively(srcDir, {...options, follow}, rootDir)) { + const filePath = assetFile.relativePath; + const destFilePath = path.join(destDir, filePath); - if (shouldExclude(exclude, path.relative(rootDir, sourceFilePath))) { - continue; - } - - const destFilePath = path.join(destDir, file); - - let stat: fs.Stats | undefined = follow === FollowMode.ALWAYS - ? fs.statSync(sourceFilePath) - : fs.lstatSync(sourceFilePath); - - if (stat && stat.isSymbolicLink()) { - const target = fs.readlinkSync(sourceFilePath); + if (follow !== FollowMode.ALWAYS) { + if (assetFile.isSymbolicLink) { + const targetPath = path.normalize(path.resolve(srcDir, assetFile.symlinkTarget)); + if (!shouldFollow(follow, rootDir, targetPath)) { + fs.symlinkSync(assetFile.symlinkTarget, destFilePath); - // determine if this is an external link (i.e. the target's absolute path - // is outside of the root directory). - const targetPath = path.normalize(path.resolve(srcDir, target)); - - if (shouldFollow(follow, rootDir, targetPath)) { - stat = fs.statSync(sourceFilePath); - } else { - fs.symlinkSync(target, destFilePath); - stat = undefined; + continue; + } } } - if (stat && stat.isDirectory()) { - fs.mkdirSync(destFilePath); - copyDirectory(sourceFilePath, destFilePath, options, rootDir); - stat = undefined; - } - - if (stat && stat.isFile()) { - fs.copyFileSync(sourceFilePath, destFilePath); - stat = undefined; + if (!assetFile.isDirectory) { + mkdirpSync(path.dirname(destFilePath)); + fs.copyFileSync(assetFile.absolutePath, destFilePath); + } else { + mkdirpSync(destFilePath); } } } diff --git a/packages/@aws-cdk/assets/lib/fs/fingerprint.ts b/packages/@aws-cdk/assets/lib/fs/fingerprint.ts index 750e541b94eaf..b660a9213ab2a 100644 --- a/packages/@aws-cdk/assets/lib/fs/fingerprint.ts +++ b/packages/@aws-cdk/assets/lib/fs/fingerprint.ts @@ -3,7 +3,7 @@ import fs = require('fs'); import path = require('path'); import { CopyOptions } from './copy-options'; import { FollowMode } from './follow-mode'; -import { shouldExclude, shouldFollow } from './utils'; +import { listFilesRecursively, shouldFollow } from './utils'; const BUFFER_SIZE = 8 * 1024; const CTRL_SOH = '\x01'; @@ -38,40 +38,28 @@ export function fingerprint(fileOrDirectory: string, options: FingerprintOptions const rootDirectory = fs.statSync(fileOrDirectory).isDirectory() ? fileOrDirectory : path.dirname(fileOrDirectory); - const exclude = options.exclude || []; - _processFileOrDirectory(fileOrDirectory); - return hash.digest('hex'); - - function _processFileOrDirectory(symbolicPath: string, realPath = symbolicPath) { - if (shouldExclude(exclude, symbolicPath)) { - return; - } - - const stat = fs.lstatSync(realPath); - const relativePath = path.relative(fileOrDirectory, symbolicPath); + for (const assetFile of listFilesRecursively(fileOrDirectory, {...options, follow}, rootDirectory)) { + const relativePath = path.relative(fileOrDirectory, assetFile.absolutePath); - if (stat.isSymbolicLink()) { - const linkTarget = fs.readlinkSync(realPath); - const resolvedLinkTarget = path.resolve(path.dirname(realPath), linkTarget); - if (shouldFollow(follow, rootDirectory, resolvedLinkTarget)) { - _processFileOrDirectory(symbolicPath, resolvedLinkTarget); + if (assetFile.isSymbolicLink) { + const resolvedLinkTarget = path.resolve(path.dirname(assetFile.absolutePath), assetFile.symlinkTarget); + if (!shouldFollow(follow, rootDirectory, resolvedLinkTarget)) { + _hashField(hash, `link:${relativePath}`, assetFile.symlinkTarget); } else { - _hashField(hash, `link:${relativePath}`, linkTarget); + _hashField(hash, `file:${relativePath}`, _contentFingerprint(assetFile.absolutePath, assetFile.size)); } - } else if (stat.isFile()) { - _hashField(hash, `file:${relativePath}`, _contentFingerprint(realPath, stat)); - } else if (stat.isDirectory()) { - for (const item of fs.readdirSync(realPath).sort()) { - _processFileOrDirectory(path.join(symbolicPath, item), path.join(realPath, item)); - } - } else { - throw new Error(`Unable to hash ${symbolicPath}: it is neither a file nor a directory`); + } else if (assetFile.isFile) { + _hashField(hash, `file:${relativePath}`, _contentFingerprint(assetFile.absolutePath, assetFile.size)); + } else if (!assetFile.isDirectory) { + throw new Error(`Unable to hash ${assetFile.absolutePath}: it is neither a file nor a directory`); } } + + return hash.digest('hex'); } -function _contentFingerprint(file: string, stat: fs.Stats): string { +function _contentFingerprint(file: string, size: number): string { const hash = crypto.createHash('sha256'); const buffer = Buffer.alloc(BUFFER_SIZE); // tslint:disable-next-line: no-bitwise @@ -85,7 +73,7 @@ function _contentFingerprint(file: string, stat: fs.Stats): string { } finally { fs.closeSync(fd); } - return `${stat.size}:${hash.digest('hex')}`; + return `${size}:${hash.digest('hex')}`; } function _hashField(hash: crypto.Hash, header: string, value: string | Buffer | DataView) { diff --git a/packages/@aws-cdk/assets/lib/fs/mkdirpSync.ts b/packages/@aws-cdk/assets/lib/fs/mkdirpSync.ts new file mode 100644 index 0000000000000..94f68354e23b4 --- /dev/null +++ b/packages/@aws-cdk/assets/lib/fs/mkdirpSync.ts @@ -0,0 +1,67 @@ +// Slightly refactored version of fs-extra mkdirpSync +// https://github.com/jprichardson/node-fs-extra/blob/d1a01e735e81688e08688557d7a254fa8297d98e/lib/mkdirs/mkdirs.js + +import fs = require('fs'); +import path = require('path'); + +const INVALID_PATH_CHARS = /[<>:"|?*]/; +const o777 = parseInt('0777', 8); + +function getRootPath(p: string) { + const paths = path.normalize(path.resolve(p)).split(path.sep); + if (paths.length > 0) { return paths[0]; } + return null; +} + +function invalidWin32Path(p: string) { + const rp = getRootPath(p); + p = p.replace(rp || '', ''); + return INVALID_PATH_CHARS.test(p); +} + +export function mkdirpSync(p: string, opts?: any, made?: any) { + if (!opts || typeof opts !== 'object') { + opts = { mode: opts }; + } + + let mode = opts.mode; + const xfs = opts.fs || fs; + + if (process.platform === 'win32' && invalidWin32Path(p)) { + const errInval = new Error(p + ' contains invalid WIN32 path characters.'); + // @ts-ignore + errInval.code = 'EINVAL'; + throw errInval; + } + + if (mode === undefined) { + // tslint:disable-next-line: no-bitwise + mode = o777 & (~process.umask()); + } + if (!made) { made = null; } + + p = path.resolve(p); + + try { + xfs.mkdirSync(p, mode); + made = made || p; + } catch (err0) { + if (err0.code === 'ENOENT') { + if (path.dirname(p) === p) { throw err0; } + made = mkdirpSync(path.dirname(p), opts, made); + mkdirpSync(p, opts, made); + } else { + // In the case of any other error, just see if there's a dir there + // already. If so, then hooray! If not, then something is borked. + let stat; + try { + stat = xfs.statSync(p); + } catch (err1) { + throw err0; + } + if (!stat.isDirectory()) { throw err0; } + } + } + + return made; +} \ No newline at end of file diff --git a/packages/@aws-cdk/assets/lib/fs/utils.ts b/packages/@aws-cdk/assets/lib/fs/utils.ts index a63717f46e49a..1bff5c4729e33 100644 --- a/packages/@aws-cdk/assets/lib/fs/utils.ts +++ b/packages/@aws-cdk/assets/lib/fs/utils.ts @@ -1,6 +1,7 @@ import fs = require('fs'); import minimatch = require('minimatch'); import path = require('path'); +import { CopyOptions } from './copy-options'; import { FollowMode } from './follow-mode'; /** @@ -8,27 +9,140 @@ import { FollowMode } from './follow-mode'; * exclusion glob patterns. * * @param exclude exclusion patterns - * @param filePath file apth to be assessed against the pattern + * @param filePath file path to be assessed against the pattern * * @returns `true` if the file should be excluded + * + * @deprecated see {@link ExcludeRules.excludeFile} */ export function shouldExclude(exclude: string[], filePath: string): boolean { - let excludeOutput = false; + const [_shouldExclude] = ExcludeRules.evaluateFile(exclude, filePath); + return _shouldExclude; +} + +/** + * Set of exclusion evaluation methods + */ +export class ExcludeRules { + /** + * Determines whether a given file should be excluded or not based on given + * exclusion glob patterns. + * + * @param patterns exclusion patterns + * @param filePath file path to be assessed against the pattern + * + * @returns `true` if the file should be excluded, followed by the index of the rule applied + */ + public static evaluateFile(patterns: string[], filePath: string): [boolean, number] { + let _shouldExclude = false; + let exclusionIndex = -1; - for (const pattern of exclude) { - const negate = pattern.startsWith('!'); - const match = minimatch(filePath, pattern, { matchBase: true, flipNegate: true }); + patterns.map((pattern, patternIndex) => { + if (minimatch(filePath, pattern, { matchBase: true, flipNegate: true })) { + [_shouldExclude, exclusionIndex] = [!pattern.startsWith('!'), patternIndex]; + } + }); - if (!negate && match) { - excludeOutput = true; + return [_shouldExclude, exclusionIndex]; + } + + /** + * Splits a file or directory path in an array of elements + * containing each path component (directories and file) + * + * @param filePath the path to split + * @returns an array containing each path component + * + * @example ExcludeRules.getPathComponents('a/b/c') = ['a', 'b', 'c'] + */ + private static getPathComponents = (filePath: string): string[] => filePath.split(path.sep); + + /** + * Splits a file or directory path in an array of elements + * containing each partial path up to that point + * + * @param filePath the path to split + * @returns an array containing each path component + * + * @example ExcludeRules.getAccumulatedPathComponents('a/b/c') = ['a', 'a/b', 'a/b/c'] + */ + private static getAccumulatedPathComponents(filePath: string): string[] { + const accComponents: string[] = []; + for (const component of ExcludeRules.getPathComponents(filePath)) { + accComponents.push(accComponents.length ? + [accComponents[accComponents.length - 1], component].join(path.sep) : + component + ); } + return accComponents; + } + + private readonly patternComponents: string[][] = this.patterns.map(ExcludeRules.getPathComponents); + private get accumulatedPatternComponents(): string[][] { + const patternComponentsLength = this.patternComponents.map(({ length }) => length); + const maxPatternLength = Math.max(...patternComponentsLength); + + const accPatternComponents: string[][] = []; + for (let pattenComponentsLength = 1; pattenComponentsLength <= maxPatternLength; ++pattenComponentsLength) { + accPatternComponents.push(this.patternComponents.map((pattern) => pattern.slice(0, pattenComponentsLength).join(path.sep))); + } + + return accPatternComponents; + } + + public constructor(private readonly patterns: string[]) { } + + /** + * Determines whether a given file should be excluded,taking into account deep file structures + * + * @param filePath file path to be assessed against the pattern + */ + public excludeFile(relativePath: string): boolean { + let accExclude = false; + let accPriority = -1; + + for (const accPath of ExcludeRules.getAccumulatedPathComponents(relativePath)) { + const [shouldExcludeIt, priorityIt] = ExcludeRules.evaluateFile(this.patterns, accPath); + if (priorityIt > accPriority) { + [accExclude, accPriority] = [shouldExcludeIt, priorityIt]; + } + } + + return accExclude; + } + + /** + * Determines whether a given directory should be excluded and not explored further + * This might be `true` even if the directory is explicitly excluded, + * but one of its children might be inclunded + * + * @param directoryPath directory path to be assessed against the pattern + */ + public excludeDirectory(directoryPath: string): boolean { + let _shouldExclude: boolean | null = null; - if (negate && match) { - excludeOutput = false; + for (const accPath of ExcludeRules.getAccumulatedPathComponents(directoryPath)) { + this.accumulatedPatternComponents.map((accumulatedPatterns, accumulatedIndex) => { + const [shouldExcludeIt, patternIndex] = ExcludeRules.evaluateFile(accumulatedPatterns, accPath); + if (patternIndex < 0) { + return; + } + + if (shouldExcludeIt) { + if (_shouldExclude === null) { + _shouldExclude = true; + } + } else if (accumulatedIndex < this.patternComponents[patternIndex].length - 1) { + _shouldExclude = shouldExcludeIt; + } else if (!accumulatedPatterns[patternIndex].includes('**')) { + _shouldExclude = true; + } + }); } + + return _shouldExclude || false; } - return excludeOutput; } /** @@ -58,3 +172,92 @@ export function shouldFollow(mode: FollowMode, sourceRoot: string, realPath: str return path.resolve(realPath).startsWith(path.resolve(sourceRoot)); } } + +type AssetFile = { + absolutePath: string; + relativePath: string; + isFile: boolean; + isDirectory: boolean; + size: number; +} & ({ isSymbolicLink: false } | { isSymbolicLink: true; symlinkTarget: string }); + +const generateAssetFile = (rootDir: string, fullFilePath: string, stat: fs.Stats): AssetFile => ({ + absolutePath: fullFilePath, + relativePath: path.relative(rootDir, fullFilePath) || path.relative(path.dirname(rootDir), fullFilePath), + isFile: stat.isFile(), + isDirectory: stat.isDirectory(), + size: stat.size, + isSymbolicLink: false, +}); + +const generateAssetSymlinkFile = (rootDir: string, fullFilePath: string, stat: fs.Stats, symlinkTarget: string): AssetFile => ({ + ...generateAssetFile(rootDir, fullFilePath, stat), + isSymbolicLink: true, + symlinkTarget, +}); + +export function listFilesRecursively( + dirOrFile: string, + options: CopyOptions & Required>, _rootDir?: string +): AssetFile[] { + const files: AssetFile[] = []; + const exclude = options.exclude || []; + const rootDir = _rootDir || dirOrFile; + const followStatsFn = options.follow === FollowMode.ALWAYS ? fs.statSync : fs.lstatSync; + + const excludeRules = new ExcludeRules(exclude); + + recurse(dirOrFile); + + function recurse(currentPath: string, _currentStat?: fs.Stats): void { + const currentStat = _currentStat || fs.statSync(currentPath); + if (!currentStat) { + return; + } + + for (const file of currentStat.isDirectory() ? fs.readdirSync(currentPath) : ['']) { + const fullFilePath = path.join(currentPath, file); + const relativeFilePath = path.relative(rootDir, fullFilePath); + + let stat: fs.Stats | undefined = followStatsFn(fullFilePath); + if (!stat) { + continue; + } + + const isExcluded = excludeRules.excludeFile(relativeFilePath); + if (!isExcluded) { + let target = ''; + if (stat.isSymbolicLink()) { + target = fs.readlinkSync(fullFilePath); + + // determine if this is an external link (i.e. the target's absolute path is outside of the root directory). + const targetPath = path.normalize(path.resolve(currentPath, target)); + if (shouldFollow(options.follow, rootDir, targetPath)) { + stat = fs.statSync(fullFilePath); + if (!stat) { + continue; + } + } + } + + if (stat.isFile()) { + files.push(generateAssetFile(rootDir, fullFilePath, stat)); + } else if (stat.isSymbolicLink()) { + files.push(generateAssetSymlinkFile(rootDir, fullFilePath, stat, target)); + } + } + + if (stat.isDirectory() && (!isExcluded || !excludeRules.excludeDirectory(relativeFilePath))) { + const previousLength = files.length; + recurse(fullFilePath, stat); + + if (files.length === previousLength && !isExcluded) { + // helps "copy" create an empty directory + files.push(generateAssetFile(rootDir, fullFilePath, stat)); + } + } + } + } + + return files; +} diff --git a/packages/@aws-cdk/assets/test/fs/fs-utils.ts b/packages/@aws-cdk/assets/test/fs/fs-utils.ts new file mode 100644 index 0000000000000..222af8a5e22ba --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fs-utils.ts @@ -0,0 +1,119 @@ +import fs = require('fs'); +import os = require('os'); +import path = require('path'); + +interface FromTreeOutput { + /** + * Absolute path of the created temporary directory, containing the generated structure + */ + readonly directory: string; + /** + * Cleanup function that will remove the generated files once called + */ + readonly cleanup: () => void; +} + +/** + * Collection of file-system utility methods + */ +export class FsUtils { + /** + * Generates a filesystem structure from a string, + * formatted like the output of a `tree` shell command + * + * @param tmpPrefix temp directory prefix, used by {@link fs.mkdtemp} + * @param tree + * @param content the content + * + * @returns an array containing the absolute path of the created directory, + * and a cleanup function that will remove the generated files on invocation + */ + public static fromTree(tmpPrefix: string, tree: string, content = 'content'): FromTreeOutput { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), tmpPrefix)); + + const directories: string[] = []; + const files: string[] = []; + const symlinks: Array<[string, string]> = []; + + // we push an element at the end because we push the files/directories during the previous iteration + const lines = [...tree.replace(/^\n+/, '').trimRight().split('\n'), '']; + const initialIndentLevel = (lines[0].match(/^\s*/) || [''])[0].length; + + lines.reduce<[string, number, boolean]>(([previousDir, previousIndentLevel, wasDirectory], line) => { + const indentCharacters = (line.match(FsUtils.INDENT_CHARACTERS_REGEX) || [''])[0]; + const indentLevel = (indentCharacters.length - initialIndentLevel) / 4; + + const fileName = line.slice(indentCharacters.length).replace(FsUtils.TRAILING_CHARACTERS_REGEX, '').trimRight(); + + const current = indentLevel <= previousIndentLevel ? + path.join(...previousDir.split(path.sep).slice(0, indentLevel - 1), fileName) : + path.join(previousDir, fileName); + + if (previousDir) { + // Because of the structure of a tree output, we need the next line + // to tell whether the current one is a directory or not. + // If the indentation as increased (or it was forcefully marked as a directory by ending with "(D)") + // then we know the previous file is a directory + if (indentLevel > previousIndentLevel || wasDirectory) { + directories.push(previousDir); + } else if (FsUtils.SYMBOLIC_LINK_REGEX.test(previousDir)) { + const [link, target] = previousDir.split(FsUtils.SYMBOLIC_LINK_REGEX); + symlinks.push([link, target]); + } else { + files.push(previousDir); + } + } + + return [current, indentLevel, FsUtils.IS_DIRECTORY_REGEX.test(line)]; + }, ['', 0, false]); + + // we create the directories first, as they're needed to store the files + for (const _directory of directories) { + fs.mkdirSync(path.join(directory, _directory)); + } + // we create the files first, as they're needed to create the symlinks + for (const file of files) { + fs.writeFileSync(path.join(directory, file), content); + } + for (const [link, target] of symlinks) { + fs.symlinkSync(target, path.join(directory, link)); + } + + return { + directory, + cleanup: () => { + // reverse order of the creation, we need to empty the directories before rmdir + for (const [link] of symlinks) { + fs.unlinkSync(path.join(directory, link)); + } + for (const file of files) { + fs.unlinkSync(path.join(directory, file)); + } + for (const _directory of directories.reverse()) { + fs.rmdirSync(path.join(directory, _directory)); + } + + // finally, we delete the directory created by mkdtempSync + fs.rmdirSync(directory); + } + }; + } + + /** + * RegExp matching characters used to indent the tree, indicating the line depth + */ + private static readonly INDENT_CHARACTERS_REGEX = /^[\s├─│└]+/; + /** + * RegExp matching characters trailing a tree line + */ + private static readonly TRAILING_CHARACTERS_REGEX = /(\/|\(D\))$/i; + /** + * RegExp determining whether a given line is an explicit directory + */ + private static readonly IS_DIRECTORY_REGEX = /\(D\)\s*$/i; + /** + * RegExp determining whether a given line is a symblic link + */ + private static readonly SYMBOLIC_LINK_REGEX = /\s*[=-]>\s*/; + +} \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/test.fs-utils.ts b/packages/@aws-cdk/assets/test/fs/test.fs-utils.ts new file mode 100644 index 0000000000000..a284281765b3a --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/test.fs-utils.ts @@ -0,0 +1,131 @@ +import fs = require('fs'); +import { Test } from 'nodeunit'; +import os = require('os'); +import path = require('path'); +import { FsUtils } from './fs-utils'; + +export = { + fromTree: { + 'basic usage'(test: Test) { + // GIVEN + const tree = ` + ├── foo + └── dir + └── subdir/ + └── bar.txt`; + + // THEN + const { directory, cleanup } = FsUtils.fromTree('basic-usage', tree); + + test.ok(fs.existsSync(path.join(directory, 'foo'))); + test.ok(fs.existsSync(path.join(directory, 'dir', 'subdir', 'bar.txt'))); + + cleanup(); + + test.ok(!fs.existsSync(path.join(directory, 'foo'))); + test.ok(!fs.existsSync(path.join(directory, 'dir', 'subdir', 'bar.txt'))); + test.ok(!fs.existsSync(directory)); + + test.done(); + }, + 'symlinks'(test: Test) { + // GIVEN + const tree = ` + ├── link -> target + ├── other_link=>target + ├── target + └── foo.txt`; + + // THEN + const { directory, cleanup } = FsUtils.fromTree('symlink', tree); + + test.ok(fs.existsSync(path.join(directory, 'target', 'foo.txt'))); + test.ok(fs.existsSync(path.join(directory, 'link', 'foo.txt'))); + test.ok(fs.existsSync(path.join(directory, 'other_link', 'foo.txt'))); + test.equal(fs.readlinkSync(path.join(directory, 'link')), 'target'); + test.equal(fs.readlinkSync(path.join(directory, 'other_link')), 'target'); + + cleanup(); + + test.ok(!fs.existsSync(path.join(directory, 'target'))); + test.ok(!fs.existsSync(path.join(directory, 'link'))); + test.ok(!fs.existsSync(path.join(directory, 'other_link'))); + test.ok(!fs.existsSync(directory)); + + test.done(); + }, + 'external smylinks'(test: Test) { + // GIVEN + const externalTree = FsUtils.fromTree('external', ` + ├── external_dir + │   ├── foobar.txt`); + + // THEN + + const externalRelativeDirectory = path.relative(os.tmpdir(), externalTree.directory); + const externalLink = `../${externalRelativeDirectory}/external_dir`; + + const internalTree = FsUtils.fromTree('internal', ` + ├── external_link -> ${externalLink}`); + + test.ok(fs.existsSync(path.join(externalTree.directory, 'external_dir', 'foobar.txt'))); + test.ok(fs.existsSync(path.join(internalTree.directory, 'external_link', 'foobar.txt'))); + test.equal(fs.readlinkSync(path.join(internalTree.directory, 'external_link')), externalLink); + + externalTree.cleanup(); + internalTree.cleanup(); + + test.ok(!fs.existsSync(path.join(externalTree.directory, 'external_dir'))); + test.ok(!fs.existsSync(path.join(internalTree.directory, 'external_link'))); + test.ok(!fs.existsSync(internalTree.directory)); + test.ok(!fs.existsSync(externalTree.directory)); + + test.done(); + }, + 'empty directory'(test: Test) { + // GIVEN + const tree = ` + ├── dir (D)`; + + // THEN + const { directory, cleanup } = FsUtils.fromTree('empty-directory', tree); + + test.ok(fs.existsSync(path.join(directory, 'dir'))); + + cleanup(); + + test.ok(!fs.existsSync(path.join(directory, 'dir'))); + test.ok(!fs.existsSync(directory)); + + test.done(); + }, + 'works with any indent'(test: Test) { + // GIVEN + const tree = ` + + + + ├── foo + └── dir + └── subdir + └── bar.txt + + + `; + + // THEN + const { directory, cleanup } = FsUtils.fromTree('any-indent', tree); + + test.ok(fs.existsSync(path.join(directory, 'foo'))); + test.ok(fs.existsSync(path.join(directory, 'dir', 'subdir', 'bar.txt'))); + + cleanup(); + + test.ok(!fs.existsSync(path.join(directory, 'foo'))); + test.ok(!fs.existsSync(path.join(directory, 'dir', 'subdir', 'bar.txt'))); + test.ok(!fs.existsSync(directory)); + + test.done(); + }, + }, +}; \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/test.utils.ts b/packages/@aws-cdk/assets/test/fs/test.utils.ts index d0d4cde73dcea..51f83f611e3ae 100644 --- a/packages/@aws-cdk/assets/test/fs/test.utils.ts +++ b/packages/@aws-cdk/assets/test/fs/test.utils.ts @@ -1,9 +1,11 @@ import fs = require('fs'); import { Test } from 'nodeunit'; +import os = require('os'); import path = require('path'); import { ImportMock } from 'ts-mock-imports'; import { FollowMode } from '../../lib/fs'; import util = require('../../lib/fs/utils'); +import { FsUtils } from './fs-utils'; export = { shouldExclude: { @@ -192,4 +194,340 @@ export = { }, } }, + + shouldExcludeDeep: { + 'without pattern'(test: Test) { + testShouldExcludeDeep(test, [], [], ['foo.txt']); + testShouldExcludeDeep(test, [''], [], ['foo.txt']); + testShouldExcludeDeep(test, ['# comment'], [], ['foo.txt']); + + test.done(); + }, + 'basic usage'(test: Test) { + testShouldExcludeDeep(test, ['foo.txt'], [ + 'foo.txt', + 'foo.txt/file', + 'dir/foo.txt', + ], [ + 'bar.txt', + 'foo', + 'foo.txt.old', + ]); + + test.done(); + }, + 'negative contridactory'(test: Test) { + testShouldExcludeDeep(test, ['foo.txt', '!foo.txt'], [], ['foo.txt']); + + test.done(); + }, + 'positive contridactory'(test: Test) { + testShouldExcludeDeep(test, ['!foo.txt', 'foo.txt'], ['foo.txt'], []); + + test.done(); + }, + 'dir single wildcard'(test: Test) { + testShouldExcludeDeep(test, ['d?r'], [ + 'dir', + 'dir/exclude', + 'dir/exclude/file', + ], [ + 'door', + 'door/file', + ]); + + test.done(); + }, + 'dir wildcard'(test: Test) { + testShouldExcludeDeep(test, ['d*r'], [ + 'dir', + 'dir/file', + 'door', + 'door/file', + ], [ + 'dog', + 'dog/file', + ]); + + test.done(); + }, + 'dir deep wildcard'(test: Test) { + testShouldExcludeDeep(test, ['dir/**/*', '!dir/include/**/*'], [ + 'dir/deep', + 'dir/deep/file', + 'dir/deep/deeper/file', + 'dir/include', + ], [ + 'dir', + 'dir/include/deep', + 'dir/include/deep/deeper', + ]); + + test.done(); + }, + 'deep structure'(test: Test) { + testShouldExcludeDeep(test, ['deep/exclude'], [ + 'deep/exclude', + 'deep/exclude/file', + ], [ + 'deep', + 'deep/include', + 'deep/include/file', + ]); + + test.done(); + }, + 'inverted pattern'(test: Test) { + testShouldExcludeDeep(test, ['*', '!foo.txt', '!d?r', 'dir/exclude'], [ + 'bar.txt', + 'dir/exclude', + 'dir/exclude/file', + ], [ + '.hidden-file', + 'foo.txt', + 'dir', + 'dir/include', + 'dir/include/subdir', + 'exclude/foo.txt', + ]); + + test.done(); + }, + }, + + shouldExcludeDirectory: { + 'without pattern'(test: Test) { + testShouldExcludeDirectory(test, [], [], ['dir']); + testShouldExcludeDirectory(test, [''], [], ['dir']); + testShouldExcludeDirectory(test, ['# comment'], [], ['dir']); + + test.done(); + }, + 'basic usage'(test: Test) { + const pattern = ['dir', '!dir/*', 'other_dir']; + + testShouldExcludeDeep(test, pattern, ['dir', 'other_dir'], ['dir/file']); + testShouldExcludeDirectory(test, pattern, ['dir/deep', 'other_dir'], ['dir']); + + test.done(); + }, + 'deep structure'(test: Test) { + const pattern = ['dir', '!dir/subdir/?', 'other_dir', 'really/deep/structure/of/files/and/dirs']; + + testShouldExcludeDeep(test, pattern, + ['dir', 'dir/subdir', 'other_dir'], + ['dir/subdir/a'] + ); + testShouldExcludeDirectory(test, pattern, + ['other_dir', 'dir/subdir/d'], + ['dir', 'dir/subdir'] + ); + + test.done(); + }, + 'wildcard pattern'(test: Test) { + const pattern = ['dir', '!dir/*/*', 'other_dir']; + + testShouldExcludeDeep(test, pattern, + ['dir', 'other_dir', 'dir/file'], + ['dir/file/deep'] + ); + testShouldExcludeDirectory(test, pattern, + ['other_dir', 'dir/deep/struct'], + ['dir', 'dir/deep', 'dir/deep'] + ); + + test.done(); + }, + 'deep wildcard'(test: Test) { + const pattern = ['dir', '!dir/**/*', 'other_dir']; + + testShouldExcludeDeep(test, pattern, + ['dir', 'other_dir'], + ['dir/file', 'dir/file/deep'] + ); + testShouldExcludeDirectory(test, pattern, + ['other_dir'], + ['dir', 'dir/deep', 'dir/deep/struct', 'dir/really/really/really/really/deep'] + ); + + test.done(); + }, + }, + + listFilesRecursively: { + 'basic usage'(test: Test) { + const exclude = ['']; + const follow = FollowMode.ALWAYS; + const tree = ` + ├── directory + │   ├── foo.txt + │   └── bar.txt + ├── empty-dir (D) + ├── deep + │   ├── dir + │   │   └── struct + │   │   └── qux.txt + ├── foobar.txt`; + + const { directory, cleanup } = FsUtils.fromTree('basic', tree); + const paths = util.listFilesRecursively(directory, { exclude, follow }).map(({ relativePath }) => relativePath); + + test.deepEqual(paths, [ + 'deep/dir/struct/qux.txt', + 'directory/bar.txt', + 'directory/foo.txt', + 'empty-dir', + 'foobar.txt', + ]); + + cleanup(); + test.done(); + }, + 'exclude'(test: Test) { + const exclude = ['foobar.txt', 'deep', '!deep/foo.txt']; + const follow = FollowMode.ALWAYS; + const tree = ` + ├── directory + │   ├── foo.txt + │   └── bar.txt + ├── deep + │   ├── dir + │   │   └── struct + │   │   └── qux.txt + │   ├── foo.txt + │   └── bar.txt + ├── foobar.txt`; + + const { directory, cleanup } = FsUtils.fromTree('exclude', tree); + const paths = util.listFilesRecursively(directory, { exclude, follow }).map(({ relativePath }) => relativePath); + + test.deepEqual(paths, [ + 'deep/foo.txt', + 'directory/bar.txt', + 'directory/foo.txt', + ]); + + cleanup(); + test.done(); + }, + 'symlinks': { + 'without exclusion'(test: Test) { + const exclude = ['']; + const externalTree = FsUtils.fromTree('exclude', ` + ├── external_dir + │   ├── foobar.txt`); + + const internalTree = FsUtils.fromTree('exclude', ` + ├── directory + │   ├── foo.txt + ├── internal_link -> directory + ├── external_link -> ../${path.relative(os.tmpdir(), externalTree.directory)}/external_dir`); + + const expected: { [followMode in FollowMode]: string[] } = { + [FollowMode.NEVER]: [ + 'directory/foo.txt', + 'external_link', + 'internal_link', + ], + [FollowMode.ALWAYS]: [ + 'directory/foo.txt', + 'external_link/foobar.txt', + 'internal_link/foo.txt', + ], + [FollowMode.EXTERNAL]: [ + 'directory/foo.txt', + 'external_link/foobar.txt', + 'internal_link' + ], + [FollowMode.BLOCK_EXTERNAL]: [ + 'directory/foo.txt', + 'external_link', + 'internal_link/foo.txt', + ], + }; + + for (const follow of Object.values(FollowMode)) { + const paths = util.listFilesRecursively(internalTree.directory, { exclude, follow }).map(({ relativePath }) => relativePath); + + test.deepEqual(paths, expected[follow], follow); + } + + internalTree.cleanup(); + externalTree.cleanup(); + test.done(); + }, + }, + 'exclude targets'(test: Test) { + const exclude = ['external_dir', 'directory']; + const follow = FollowMode.ALWAYS; + const externalTree = FsUtils.fromTree('exclude', ` + ├── external_dir + │   ├── foobar.txt`); + + const internalTree = FsUtils.fromTree('exclude', ` + ├── directory + │   ├── foo.txt + ├── internal_link -> directory + ├── external_link -> ../${path.relative(os.tmpdir(), externalTree.directory)}/external_dir`); + const paths = util.listFilesRecursively(internalTree.directory, { exclude, follow }).map(({ relativePath }) => relativePath); + + test.deepEqual(paths, [ + 'external_link/foobar.txt', + 'internal_link/foo.txt' + ]); + + internalTree.cleanup(); + externalTree.cleanup(); + test.done(); + }, + 'exclude links'(test: Test) { + const exclude = ['internal_link', 'external_link']; + const follow = FollowMode.ALWAYS; + const externalTree = FsUtils.fromTree('exclude', ` + ├── external_dir + │   ├── foobar.txt`); + + const internalTree = FsUtils.fromTree('exclude', ` + ├── directory + │   ├── foo.txt + ├── internal_link -> directory + ├── external_link -> ../${path.relative(os.tmpdir(), externalTree.directory)}/external_dir`); + const paths = util.listFilesRecursively(internalTree.directory, { exclude, follow }).map(({ relativePath }) => relativePath); + + test.deepEqual(paths, [ + 'directory/foo.txt' + ]); + + internalTree.cleanup(); + externalTree.cleanup(); + test.done(); + }, + }, +}; + +const testShouldExclude = ( + test: Test, + pattern: string[], + expectExclude: string[], + expectInclude: string[], + shouldExcludeMethod: (pattern: string[], path: string) => boolean) => { + for (const exclude of expectExclude) { + test.ok(shouldExcludeMethod(pattern, exclude), `${exclude} should have been excluded, but wasn't`); + } + for (const include of expectInclude) { + test.ok(!shouldExcludeMethod(pattern, include), `${include} should have been included, but wasn't`); + } }; + +const testShouldExcludeDeep = (test: Test, pattern: string[], expectExclude: string[], expectInclude: string[]) => + testShouldExclude(test, pattern, expectExclude, expectInclude, (patterns: string[], _path: string) => { + const excludeRules = new util.ExcludeRules(patterns); + return excludeRules.excludeFile(_path); + }); + +const testShouldExcludeDirectory = (test: Test, pattern: string[], expectExclude: string[], expectInclude: string[]) => + testShouldExclude(test, pattern, expectExclude, expectInclude, (patterns: string[], _path: string) => { + const excludeRules = new util.ExcludeRules(patterns); + return excludeRules.excludeDirectory(_path); + }); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-ecr-assets/lib/image-asset.ts b/packages/@aws-cdk/aws-ecr-assets/lib/image-asset.ts index bc51c8f13a0c6..c1c0694fb2ece 100644 --- a/packages/@aws-cdk/aws-ecr-assets/lib/image-asset.ts +++ b/packages/@aws-cdk/aws-ecr-assets/lib/image-asset.ts @@ -80,7 +80,12 @@ export class DockerImageAsset extends Construct implements assets.IAsset { const ignore = path.join(dir, '.dockerignore'); if (fs.existsSync(ignore)) { - exclude = [...exclude, ...fs.readFileSync(ignore).toString().split('\n').filter(e => !!e)]; + exclude = [ + ...exclude, + ...fs.readFileSync(ignore).toString().split('\n').filter(e => !!e), + // prevents accidentally excluding Dockerfile with a "*" + '!Dockerfile', + ]; } const staging = new assets.Staging(this, 'Staging', { diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/.dockerignore b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/.dockerignore deleted file mode 100644 index b7c7139ddb1a4..0000000000000 --- a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -foobar.txt diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/Dockerfile b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/Dockerfile deleted file mode 100644 index 123b5670febc8..0000000000000 --- a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM python:3.6 -EXPOSE 8000 -WORKDIR /src -ADD . /src -CMD python3 index.py diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/foobar.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/foobar.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/index.py b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/index.py deleted file mode 100644 index 2ccedfce3ab76..0000000000000 --- a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/index.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/python -import sys -import textwrap -import http.server -import socketserver - -PORT = 8000 - - -class Handler(http.server.SimpleHTTPRequestHandler): - def do_GET(self): - self.send_response(200) - self.send_header('Content-Type', 'text/html') - self.end_headers() - self.wfile.write(textwrap.dedent('''\ - - It works - -

Hello from the integ test container

-

This container got built and started as part of the integ test.

- - - ''').encode('utf-8')) - - -def main(): - httpd = http.server.HTTPServer(("", PORT), Handler) - print("serving at port", PORT) - httpd.serve_forever() - - -if __name__ == '__main__': - main() diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/subdirectory/baz.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/subdirectory/baz.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts b/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts index 5ba6e96e2ab3e..e8d7b47f9345e 100644 --- a/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts +++ b/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts @@ -1,4 +1,5 @@ import { expect, haveResource, SynthUtils } from '@aws-cdk/assert'; +import { FsUtils } from '@aws-cdk/assets/test/fs/fs-utils'; import iam = require('@aws-cdk/aws-iam'); import { App, Construct, Lazy, Resource, Stack } from '@aws-cdk/core'; import { ASSET_METADATA } from '@aws-cdk/cx-api'; @@ -234,20 +235,39 @@ export = { const app = new App(); const stack = new Stack(app, 'stack'); - const image = new DockerImageAsset(stack, 'MyAsset', { - directory: path.join(__dirname, 'dockerignore-image') - }); + const {directory, cleanup} = FsUtils.fromTree('dockerignore-image', ` + ├── Dockerfile + ├── .dockerignore + ├── foobar.txt + ├── index.py + └── subdirectory + └── baz.txt`); + fs.writeFileSync(path.join(directory, '.dockerignore'), 'foobar.txt'); + + const image = new DockerImageAsset(stack, 'MyAsset', { directory }); const session = app.synth(); - // .dockerignore itself should be included in output to be processed during docker build - test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, '.dockerignore'))); - test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, `Dockerfile`))); - test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, 'index.py'))); - test.ok(!fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, 'foobar.txt'))); - test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, 'subdirectory'))); - test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, 'subdirectory', 'baz.txt'))); + const expectedFiles = [ + // .dockerignore itself should be included in output to be processed during docker build + '.dockerignore', + 'Dockerfile', + 'index.py', + 'subdirectory', + path.join('subdirectory', 'baz.txt'), + ]; + const unexpectedFiles = [ + 'foobar.txt', + ]; + + for (const expectedFile of expectedFiles) { + test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, expectedFile)), expectedFile); + } + for (const unexpectedFile of unexpectedFiles) { + test.ok(!fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, unexpectedFile)), unexpectedFile); + } + cleanup(); test.done(); }, @@ -255,20 +275,185 @@ export = { const app = new App(); const stack = new Stack(app, 'stack'); + const {directory, cleanup} = FsUtils.fromTree('dockerignore-image', ` + ├── Dockerfile + ├── .dockerignore + ├── foobar.txt + ├── index.py + └── subdirectory + └── baz.txt`); + fs.writeFileSync(path.join(directory, '.dockerignore'), 'foobar.txt'); + const image = new DockerImageAsset(stack, 'MyAsset', { - directory: path.join(__dirname, 'dockerignore-image'), + directory, exclude: ['subdirectory'] }); const session = app.synth(); - test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, '.dockerignore'))); - test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, `Dockerfile`))); - test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, 'index.py'))); - test.ok(!fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, 'foobar.txt'))); - test.ok(!fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, 'subdirectory'))); - test.ok(!fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, 'subdirectory', 'baz.txt'))); + const expectedFiles = [ + '.dockerignore', + 'Dockerfile', + 'index.py', + ]; + const unexpectedFiles = [ + 'foobar.txt', + 'subdirectory', + path.join('subdirectory', 'baz.txt'), + ]; + + for (const expectedFile of expectedFiles) { + test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, expectedFile)), expectedFile); + } + for (const unexpectedFile of unexpectedFiles) { + test.ok(!fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, unexpectedFile)), unexpectedFile); + } + + cleanup(); + test.done(); + }, + + 'advanced .dockerignore test case'(test: Test) { + const app = new App(); + const stack = new Stack(app, 'stack'); + + // GIVEN + const {directory, cleanup} = FsUtils.fromTree('dockerignore-image-advanced', ` + ├── config + │   ├── config-prod.txt + │   ├── config-test.txt + │   └── config.txt + ├── deep + │   ├── dir + │   │   └── struct + │   │   └── qux.txt + │   └── include_me + │   └── sub + │   └── dir + │   └── quuz.txt + ├── foobar.txt + ├── foo.txt + ├── .dockerignore + ├── Dockerfile + ├── index.py + ├── .hidden-file + └── empty-directory (D) + └── subdirectory + ├── baz.txt + └── quux.txt`); + + fs.writeFileSync(path.join(directory, '.dockerignore'), ` + # This a comment, followed by an empty line + + # The following line should be ignored + #index.py + + # This shouldn't ignore foo.txt + foo.? + # This shoul ignore foobar.txt + foobar.??? + # This should catch qux.txt + deep/**/*.txt + # but quuz should be added back + !deep/include_me/** + + # baz and quux should be ignored + subdirectory/** + # but baz should be added back + !subdirectory/baz* + + config/config*.txt + !config/config-*.txt + config/config-test.txt + `.split('\n').map(line => line.trim()).join('\n')); + + const image = new DockerImageAsset(stack, 'MyAsset', { directory }); + const session = app.synth(); + + const expectedFiles = [ + '.dockerignore', + '.hidden-file', + 'Dockerfile', + 'index.py', + 'foo.txt', + 'empty-directory', + path.join('subdirectory', 'baz.txt'), + path.join('deep', 'include_me', 'sub', 'dir', 'quuz.txt'), + path.join('config', 'config-prod.txt'), + ]; + const unexpectedFiles = [ + 'foobar.txt', + path.join('deep', 'dir', 'struct', 'qux.txt'), + path.join('subdirectory', 'quux.txt'), + path.join('config', 'config.txt'), + path.join('config', 'config-test.txt'), + ]; + + for (const expectedFile of expectedFiles) { + test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, expectedFile)), expectedFile); + } + for (const unexpectedFile of unexpectedFiles) { + test.ok(!fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, unexpectedFile)), unexpectedFile); + } + + cleanup(); + test.done(); + }, + + 'negative .dockerignore test case'(test: Test) { + const app = new App(); + const stack = new Stack(app, 'stack'); + + const {directory, cleanup} = FsUtils.fromTree('dockerignore-image-advanced', ` + ├── deep + │   └── dir + │   └── struct + │   └── qux.txt + ├── Dockerfile + ├── .dockerignore + ├── foobar.txt + ├── index.py + └── subdirectory + ├── baz.txt + └── foo.txt`); + + fs.writeFileSync(path.join(directory, '.dockerignore'), ` + # Comment + + * + !index.py + !subdirectory + subdirectory/foo.txt + + # Dockerfile isn't explicitly included, but we'll add it anyway to build the image + `.split('\n').map(line => line.trim()).join('\n')); + + const image = new DockerImageAsset(stack, 'MyAsset', { directory }); + + const session = app.synth(); + + const expectedFiles = [ + 'index.py', + // Dockerfile is always added + 'Dockerfile', + path.join('subdirectory', 'baz.txt'), + // "*" doesn't match ".*" without "dot: true" in minimist + '.dockerignore', + ]; + const unexpectedFiles = [ + 'foobar.txt', + path.join('deep', 'dir', 'struct', 'qux.txt'), + path.join('subdirectory', 'foo.txt'), + ]; + + for (const expectedFile of expectedFiles) { + test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, expectedFile)), expectedFile); + } + for (const unexpectedFile of unexpectedFiles) { + test.ok(!fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, unexpectedFile)), unexpectedFile); + } + cleanup(); test.done(); },