From 3af6b91157e8a0c03ceee4e40e61df6f63da3076 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Fri, 11 Oct 2019 10:31:40 +0200 Subject: [PATCH 01/35] chore: better test --- .../aws-ecr-assets/lib/image-asset.ts | 1 + .../dockerignore-image-advanced/.dockerignore | 18 ++++++++++ .../dockerignore-image-advanced/Dockerfile | 5 +++ .../deep/dir/struct/qux.txt | 0 .../deep/include_me/sub/dir/quuz.txt | 0 .../test/dockerignore-image-advanced/foo.txt | 0 .../dockerignore-image-advanced/foobar.txt | 0 .../test/dockerignore-image-advanced/index.py | 33 ++++++++++++++++++ .../subdirectory/baz.txt | 0 .../subdirectory/quux.txt | 0 .../aws-ecr-assets/test/test.image-asset.ts | 34 +++++++++++++++++++ 11 files changed, 91 insertions(+) create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/.dockerignore create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/Dockerfile create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/deep/dir/struct/qux.txt create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/deep/include_me/sub/dir/quuz.txt create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/foo.txt create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/foobar.txt create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/index.py create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/subdirectory/baz.txt create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/subdirectory/quux.txt diff --git a/packages/@aws-cdk/aws-ecr-assets/lib/image-asset.ts b/packages/@aws-cdk/aws-ecr-assets/lib/image-asset.ts index 662883cc72bd0..25d995978bded 100644 --- a/packages/@aws-cdk/aws-ecr-assets/lib/image-asset.ts +++ b/packages/@aws-cdk/aws-ecr-assets/lib/image-asset.ts @@ -83,6 +83,7 @@ export class DockerImageAsset extends Construct implements assets.IAsset { exclude = [...exclude, ...fs.readFileSync(ignore).toString().split('\n').filter(e => !!e)]; } + console.log(exclude); const staging = new assets.Staging(this, 'Staging', { ...props, exclude, diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/.dockerignore b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/.dockerignore new file mode 100644 index 0000000000000..76531ae9927e2 --- /dev/null +++ b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/.dockerignore @@ -0,0 +1,18 @@ +# This a comment, followed by an empty line + +# The following line should be ignored +#index.py + +# This shouldn't ignore foo.txt +foo.? +# This shoul ignore foobar.txt +foobar.??? +# This should catch qux.txt +deep/**/*.txt +# but quuz should be added back +!deep/include_me/** + +# baz and quux should be ignored +subdirectory/** +# but baz should be added back +!subdirectory/baz* \ No newline at end of file diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/Dockerfile b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/Dockerfile new file mode 100644 index 0000000000000..123b5670febc8 --- /dev/null +++ b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.6 +EXPOSE 8000 +WORKDIR /src +ADD . /src +CMD python3 index.py diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/deep/dir/struct/qux.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/deep/dir/struct/qux.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/deep/include_me/sub/dir/quuz.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/deep/include_me/sub/dir/quuz.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/foo.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/foo.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/foobar.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/foobar.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/index.py b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/index.py new file mode 100644 index 0000000000000..2ccedfce3ab76 --- /dev/null +++ b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/index.py @@ -0,0 +1,33 @@ +#!/usr/bin/python +import sys +import textwrap +import http.server +import socketserver + +PORT = 8000 + + +class Handler(http.server.SimpleHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header('Content-Type', 'text/html') + self.end_headers() + self.wfile.write(textwrap.dedent('''\ + + It works + +

Hello from the integ test container

+

This container got built and started as part of the integ test.

+ + + ''').encode('utf-8')) + + +def main(): + httpd = http.server.HTTPServer(("", PORT), Handler) + print("serving at port", PORT) + httpd.serve_forever() + + +if __name__ == '__main__': + main() diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/subdirectory/baz.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/subdirectory/baz.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/subdirectory/quux.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/subdirectory/quux.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts b/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts index 955d0e8e3d98a..67428ebb3ff39 100644 --- a/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts +++ b/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts @@ -250,6 +250,40 @@ export = { test.done(); }, + 'advanced .dockerignore test case'(test: Test) { + const app = new App(); + const stack = new Stack(app, 'stack'); + + const image = new DockerImageAsset(stack, 'MyAsset', { + directory: path.join(__dirname, 'dockerignore-image-advanced') + }); + + const session = app.synth(); + + const expectedFiles = [ + '.dockerignore', + 'Dockerfile', + 'index.py', + 'foo.txt', + path.join('subdirectory', 'baz.txt'), + path.join('deep', 'include_me', 'sub', 'dir', 'quuz.txt'), + ]; + const unexpectedFiles = [ + 'foobar.txt', + path.join('deep', 'dir', 'struct', 'qux.txt'), + path.join('subdirectory', 'quux.txt'), + ]; + + for (const expectedFile of expectedFiles) { + test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, expectedFile)), expectedFile); + } + for (const unexpectedFile of unexpectedFiles) { + test.ok(!fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, unexpectedFile)), unexpectedFile); + } + + test.done(); + }, + 'fails if using tokens in build args keys or values'(test: Test) { // GIVEN const stack = new Stack(); From e0738c301f8ce885ca46f6f1cf36fdb389d8e1a8 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Fri, 11 Oct 2019 14:56:37 +0200 Subject: [PATCH 02/35] fix: folder exclusion * additional test cases * refactor existing tests --- packages/@aws-cdk/assets/lib/fs/copy.ts | 38 +++++++-- .../aws-ecr-assets/lib/image-asset.ts | 8 +- .../dockerignore-image-advanced/.dockerignore | 6 +- .../aws-ecr-assets/test/test.image-asset.ts | 85 ++++++++++++++++--- 4 files changed, 114 insertions(+), 23 deletions(-) diff --git a/packages/@aws-cdk/assets/lib/fs/copy.ts b/packages/@aws-cdk/assets/lib/fs/copy.ts index 543b64a4e5f22..da84f6ba56bba 100644 --- a/packages/@aws-cdk/assets/lib/fs/copy.ts +++ b/packages/@aws-cdk/assets/lib/fs/copy.ts @@ -6,7 +6,7 @@ import { shouldExclude, shouldFollow } from './utils'; export function copyDirectory(srcDir: string, destDir: string, options: CopyOptions = { }, rootDir?: string) { const follow = options.follow !== undefined ? options.follow : FollowMode.EXTERNAL; - const exclude = options.exclude || []; + let exclude = [...(options.exclude || [])]; rootDir = rootDir || srcDir; @@ -17,17 +17,35 @@ export function copyDirectory(srcDir: string, destDir: string, options: CopyOpti const files = fs.readdirSync(srcDir); for (const file of files) { const sourceFilePath = path.join(srcDir, file); - - if (shouldExclude(exclude, path.relative(rootDir, sourceFilePath))) { - continue; - } - const destFilePath = path.join(destDir, file); + const filePath = path.relative(rootDir, sourceFilePath); let stat: fs.Stats | undefined = follow === FollowMode.ALWAYS ? fs.statSync(sourceFilePath) : fs.lstatSync(sourceFilePath); + // we've just discovered that we have a directory + if (stat && stat.isDirectory()) { + // to help future shouldExclude calls, we're changing the exlusion patterns + // by expliciting "dir" exclusions to "dir/*" (same with "!dir" -> "!dir/*") + exclude = exclude.reduce((res, pattern) => { + res.push(pattern); + if (pattern.trim().replace(/^!/, '') === filePath) { + // we add the pattern immediately after to preserve the exclusion order + res.push(`${pattern}/*`); + } + + return res; + }, []); + } + + const isExcluded = shouldExclude(exclude, filePath); + if (isExcluded) { + if (!stat || !stat.isDirectory()) { + continue; + } + } + if (stat && stat.isSymbolicLink()) { const target = fs.readlinkSync(sourceFilePath); @@ -45,7 +63,13 @@ export function copyDirectory(srcDir: string, destDir: string, options: CopyOpti if (stat && stat.isDirectory()) { fs.mkdirSync(destFilePath); - copyDirectory(sourceFilePath, destFilePath, options, rootDir); + copyDirectory(sourceFilePath, destFilePath, { ...options, exclude }, rootDir); + + // FIXME kind of ugly + if (isExcluded && !fs.readdirSync(destFilePath).length) { + fs.rmdirSync(destFilePath); + } + stat = undefined; } diff --git a/packages/@aws-cdk/aws-ecr-assets/lib/image-asset.ts b/packages/@aws-cdk/aws-ecr-assets/lib/image-asset.ts index 25d995978bded..8189a5fa8a9ba 100644 --- a/packages/@aws-cdk/aws-ecr-assets/lib/image-asset.ts +++ b/packages/@aws-cdk/aws-ecr-assets/lib/image-asset.ts @@ -80,10 +80,14 @@ export class DockerImageAsset extends Construct implements assets.IAsset { const ignore = path.join(dir, '.dockerignore'); if (fs.existsSync(ignore)) { - exclude = [...exclude, ...fs.readFileSync(ignore).toString().split('\n').filter(e => !!e)]; + exclude = [ + ...exclude, + ...fs.readFileSync(ignore).toString().split('\n').filter(e => !!e), + // prevents accidentally excluding Dockerfile with a "*" + '!Dockerfile', + ]; } - console.log(exclude); const staging = new assets.Staging(this, 'Staging', { ...props, exclude, diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/.dockerignore b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/.dockerignore index 76531ae9927e2..6cae4e26f057c 100644 --- a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/.dockerignore +++ b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/.dockerignore @@ -15,4 +15,8 @@ deep/**/*.txt # baz and quux should be ignored subdirectory/** # but baz should be added back -!subdirectory/baz* \ No newline at end of file +!subdirectory/baz* + +config/config*.txt +!config/config-*.txt +config/config-test.txt \ No newline at end of file diff --git a/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts b/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts index 67428ebb3ff39..85f9dfe588dc6 100644 --- a/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts +++ b/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts @@ -218,13 +218,24 @@ export = { const session = app.synth(); - // .dockerignore itself should be included in output to be processed during docker build - test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, '.dockerignore'))); - test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, `Dockerfile`))); - test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, 'index.py'))); - test.ok(!fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, 'foobar.txt'))); - test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, 'subdirectory'))); - test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, 'subdirectory', 'baz.txt'))); + const expectedFiles = [ + // .dockerignore itself should be included in output to be processed during docker build + '.dockerignore', + 'Dockerfile', + 'index.py', + 'subdirectory', + path.join('subdirectory', 'baz.txt'), + ]; + const unexpectedFiles = [ + 'foobar.txt', + ]; + + for (const expectedFile of expectedFiles) { + test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, expectedFile)), expectedFile); + } + for (const unexpectedFile of unexpectedFiles) { + test.ok(!fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, unexpectedFile)), unexpectedFile); + } test.done(); }, @@ -240,12 +251,23 @@ export = { const session = app.synth(); - test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, '.dockerignore'))); - test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, `Dockerfile`))); - test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, 'index.py'))); - test.ok(!fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, 'foobar.txt'))); - test.ok(!fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, 'subdirectory'))); - test.ok(!fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, 'subdirectory', 'baz.txt'))); + const expectedFiles = [ + '.dockerignore', + 'Dockerfile', + 'index.py', + ]; + const unexpectedFiles = [ + 'foobar.txt', + 'subdirectory', + path.join('subdirectory', 'baz.txt'), + ]; + + for (const expectedFile of expectedFiles) { + test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, expectedFile)), expectedFile); + } + for (const unexpectedFile of unexpectedFiles) { + test.ok(!fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, unexpectedFile)), unexpectedFile); + } test.done(); }, @@ -267,11 +289,48 @@ export = { 'foo.txt', path.join('subdirectory', 'baz.txt'), path.join('deep', 'include_me', 'sub', 'dir', 'quuz.txt'), + path.join('config', 'config-prod.txt'), ]; const unexpectedFiles = [ 'foobar.txt', path.join('deep', 'dir', 'struct', 'qux.txt'), path.join('subdirectory', 'quux.txt'), + path.join('config', 'config.txt'), + path.join('config', 'config-test.txt'), + ]; + + for (const expectedFile of expectedFiles) { + test.ok(fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, expectedFile)), expectedFile); + } + for (const unexpectedFile of unexpectedFiles) { + test.ok(!fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, unexpectedFile)), unexpectedFile); + } + + test.done(); + }, + + 'negagive .dockerignore test case'(test: Test) { + const app = new App(); + const stack = new Stack(app, 'stack'); + + const image = new DockerImageAsset(stack, 'MyAsset', { + directory: path.join(__dirname, 'dockerignore-image-negative') + }); + + const session = app.synth(); + + const expectedFiles = [ + 'index.py', + // Dockerfile is always added + 'Dockerfile', + path.join('subdirectory', 'baz.txt'), + // "*" doesn't match ".*" without "dot: true" in minimist + '.dockerignore', + ]; + const unexpectedFiles = [ + 'foobar.txt', + path.join('deep', 'dir', 'struct', 'qux.txt'), + path.join('subdirectory', 'foo.txt'), ]; for (const expectedFile of expectedFiles) { From dce18c602ced8f9bcf89e7b377af972edb21dc12 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Fri, 11 Oct 2019 14:57:38 +0200 Subject: [PATCH 03/35] chore: add new test files --- .../config/config-prod.txt | 0 .../config/config-test.txt | 0 .../config/config.txt | 0 .../dockerignore-image-negative/.dockerignore | 8 +++++ .../dockerignore-image-negative/Dockerfile | 5 +++ .../deep/dir/struct/qux.txt | 0 .../dockerignore-image-negative/foobar.txt | 0 .../test/dockerignore-image-negative/index.py | 33 +++++++++++++++++++ .../subdirectory/baz.txt | 0 .../subdirectory/foo.txt | 0 10 files changed, 46 insertions(+) create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/config/config-prod.txt create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/config/config-test.txt create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/config/config.txt create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/.dockerignore create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/Dockerfile create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/deep/dir/struct/qux.txt create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/foobar.txt create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/index.py create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/subdirectory/baz.txt create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/subdirectory/foo.txt diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/config/config-prod.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/config/config-prod.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/config/config-test.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/config/config-test.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/config/config.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/config/config.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/.dockerignore b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/.dockerignore new file mode 100644 index 0000000000000..d8b3df3b4c01a --- /dev/null +++ b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/.dockerignore @@ -0,0 +1,8 @@ +# Comment + +* +!index.py +!subdirectory +subdirectory/foo.txt + +# Dockerfile isn't explicitly included, but we'll add it anyway to build the image \ No newline at end of file diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/Dockerfile b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/Dockerfile new file mode 100644 index 0000000000000..123b5670febc8 --- /dev/null +++ b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.6 +EXPOSE 8000 +WORKDIR /src +ADD . /src +CMD python3 index.py diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/deep/dir/struct/qux.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/deep/dir/struct/qux.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/foobar.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/foobar.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/index.py b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/index.py new file mode 100644 index 0000000000000..2ccedfce3ab76 --- /dev/null +++ b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/index.py @@ -0,0 +1,33 @@ +#!/usr/bin/python +import sys +import textwrap +import http.server +import socketserver + +PORT = 8000 + + +class Handler(http.server.SimpleHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header('Content-Type', 'text/html') + self.end_headers() + self.wfile.write(textwrap.dedent('''\ + + It works + +

Hello from the integ test container

+

This container got built and started as part of the integ test.

+ + + ''').encode('utf-8')) + + +def main(): + httpd = http.server.HTTPServer(("", PORT), Handler) + print("serving at port", PORT) + httpd.serve_forever() + + +if __name__ == '__main__': + main() diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/subdirectory/baz.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/subdirectory/baz.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/subdirectory/foo.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/subdirectory/foo.txt new file mode 100644 index 0000000000000..e69de29bb2d1d From 5faf3ca085140e10c7fe9e13fad12b7c6030dd18 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Fri, 11 Oct 2019 16:57:37 +0200 Subject: [PATCH 04/35] fix: listFilesRecursively refactor (wip) --- packages/@aws-cdk/assets/lib/fs/copy.ts | 73 +++++------------ .../@aws-cdk/assets/lib/fs/fingerprint.ts | 6 +- packages/@aws-cdk/assets/lib/fs/utils.ts | 80 +++++++++++++++++++ 3 files changed, 104 insertions(+), 55 deletions(-) diff --git a/packages/@aws-cdk/assets/lib/fs/copy.ts b/packages/@aws-cdk/assets/lib/fs/copy.ts index da84f6ba56bba..06e858330ba85 100644 --- a/packages/@aws-cdk/assets/lib/fs/copy.ts +++ b/packages/@aws-cdk/assets/lib/fs/copy.ts @@ -2,11 +2,10 @@ import fs = require('fs'); import path = require('path'); import { CopyOptions } from './copy-options'; import { FollowMode } from './follow-mode'; -import { shouldExclude, shouldFollow } from './utils'; +import { listFilesRecursively, shouldFollow } from './utils'; export function copyDirectory(srcDir: string, destDir: string, options: CopyOptions = { }, rootDir?: string) { const follow = options.follow !== undefined ? options.follow : FollowMode.EXTERNAL; - let exclude = [...(options.exclude || [])]; rootDir = rootDir || srcDir; @@ -14,68 +13,34 @@ export function copyDirectory(srcDir: string, destDir: string, options: CopyOpti throw new Error(`${srcDir} is not a directory`); } - const files = fs.readdirSync(srcDir); - for (const file of files) { - const sourceFilePath = path.join(srcDir, file); - const destFilePath = path.join(destDir, file); + for (const sourceFilePath of listFilesRecursively(srcDir, {...options, follow}, rootDir)) { const filePath = path.relative(rootDir, sourceFilePath); + const destFilePath = path.join(destDir, filePath); - let stat: fs.Stats | undefined = follow === FollowMode.ALWAYS - ? fs.statSync(sourceFilePath) - : fs.lstatSync(sourceFilePath); + if (follow !== FollowMode.ALWAYS) { + const stat = fs.lstatSync(sourceFilePath); - // we've just discovered that we have a directory - if (stat && stat.isDirectory()) { - // to help future shouldExclude calls, we're changing the exlusion patterns - // by expliciting "dir" exclusions to "dir/*" (same with "!dir" -> "!dir/*") - exclude = exclude.reduce((res, pattern) => { - res.push(pattern); - if (pattern.trim().replace(/^!/, '') === filePath) { - // we add the pattern immediately after to preserve the exclusion order - res.push(`${pattern}/*`); - } - - return res; - }, []); - } - - const isExcluded = shouldExclude(exclude, filePath); - if (isExcluded) { - if (!stat || !stat.isDirectory()) { - continue; - } - } + if (stat && stat.isSymbolicLink()) { + const target = fs.readlinkSync(sourceFilePath); + const targetPath = path.normalize(path.resolve(srcDir, target)); + if (!shouldFollow(follow, rootDir, targetPath)) { + fs.symlinkSync(target, destFilePath); - if (stat && stat.isSymbolicLink()) { - const target = fs.readlinkSync(sourceFilePath); - - // determine if this is an external link (i.e. the target's absolute path - // is outside of the root directory). - const targetPath = path.normalize(path.resolve(srcDir, target)); - - if (shouldFollow(follow, rootDir, targetPath)) { - stat = fs.statSync(sourceFilePath); - } else { - fs.symlinkSync(target, destFilePath); - stat = undefined; + continue; + } } } - if (stat && stat.isDirectory()) { - fs.mkdirSync(destFilePath); - copyDirectory(sourceFilePath, destFilePath, { ...options, exclude }, rootDir); + { + const destFileDir = path.dirname(destFilePath); - // FIXME kind of ugly - if (isExcluded && !fs.readdirSync(destFilePath).length) { - fs.rmdirSync(destFilePath); + if (!fs.existsSync(destFileDir)) { + // FIXME fs.mkdirpSync(destFileDir); + fs.mkdirSync(destFileDir); } - - stat = undefined; } - if (stat && stat.isFile()) { - fs.copyFileSync(sourceFilePath, destFilePath); - stat = undefined; - } + console.log(sourceFilePath, destFilePath); + fs.copyFileSync(sourceFilePath, destFilePath); } } diff --git a/packages/@aws-cdk/assets/lib/fs/fingerprint.ts b/packages/@aws-cdk/assets/lib/fs/fingerprint.ts index 750e541b94eaf..b57d3bde127bf 100644 --- a/packages/@aws-cdk/assets/lib/fs/fingerprint.ts +++ b/packages/@aws-cdk/assets/lib/fs/fingerprint.ts @@ -39,7 +39,11 @@ export function fingerprint(fileOrDirectory: string, options: FingerprintOptions ? fileOrDirectory : path.dirname(fileOrDirectory); const exclude = options.exclude || []; - _processFileOrDirectory(fileOrDirectory); + // _processFileOrDirectory(fileOrDirectory); + + for (const file of listFilesRecursively(fileOrDirectory, {...options, follow}, exclude)) { + // TODO process + } return hash.digest('hex'); diff --git a/packages/@aws-cdk/assets/lib/fs/utils.ts b/packages/@aws-cdk/assets/lib/fs/utils.ts index a63717f46e49a..162e3207c8404 100644 --- a/packages/@aws-cdk/assets/lib/fs/utils.ts +++ b/packages/@aws-cdk/assets/lib/fs/utils.ts @@ -1,6 +1,7 @@ import fs = require('fs'); import minimatch = require('minimatch'); import path = require('path'); +import { CopyOptions } from './copy-options'; import { FollowMode } from './follow-mode'; /** @@ -58,3 +59,82 @@ export function shouldFollow(mode: FollowMode, sourceRoot: string, realPath: str return path.resolve(realPath).startsWith(path.resolve(sourceRoot)); } } + +export function listFilesRecursively( + dir: string, + options: CopyOptions = {}, + rootDir?: string, + files: string[] = [] +): string[] { + let exclude = [...(options.exclude || [])]; + rootDir = rootDir || dir; + { + const stat = fs.statSync(dir); + if (!stat) { + return []; + } + + if (!stat.isDirectory()) { + return [dir]; + } + } + + for (const file of fs.readdirSync(dir)) { + let fullFilePath = path.join(dir, file); + + let stat: fs.Stats | undefined = follow === FollowMode.ALWAYS + ? fs.statSync(fullFilePath) + : fs.lstatSync(fullFilePath); + + if (!stat) { + continue; + } + + if (stat.isSymbolicLink()) { + const target = fs.readlinkSync(fullFilePath); + + // determine if this is an external link (i.e. the target's absolute path + // is outside of the root directory). + const targetPath = path.normalize(path.resolve(dir, target)); + + if (shouldFollow(follow, rootDir, targetPath)) { + stat = fs.statSync(fullFilePath); + if (!stat) { + continue; + } + + fullFilePath = target; + } + } + + const relativeFilePath = path.relative(rootDir, fullFilePath); + + // we've just discovered that we have a directory + if (stat.isDirectory()) { + // to help future shouldExclude calls, we're changing the exlusion patterns + // by expliciting "dir" exclusions to "dir/*" (same with "!dir" -> "!dir/*") + exclude = exclude.reduce((res, pattern) => { + res.push(pattern); + if (pattern.trim().replace(/^!/, '') === relativeFilePath) { + // we add the pattern immediately after to preserve the exclusion order + res.push(`${pattern}/*`); + } + + return res; + }, []); + } + + const isExcluded = shouldExclude(exclude, relativeFilePath); + if (isExcluded && !stat.isDirectory()) { + continue; + } + + if (stat.isFile()) { + files.push(fullFilePath); + } else if (stat.isDirectory()) { + files.push(...listFilesRecursively(fullFilePath, { ...options, exclude }, rootDir)); + } + } + + return files; +} \ No newline at end of file From 89c792ffe6d8e1f7a37215421c8c3a8ff1c420cf Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Fri, 11 Oct 2019 21:29:21 +0200 Subject: [PATCH 05/35] fix: finish refactoring listFilesRecursively --- packages/@aws-cdk/assets/lib/fs/copy.ts | 4 +-- .../@aws-cdk/assets/lib/fs/fingerprint.ts | 31 +++++-------------- packages/@aws-cdk/assets/lib/fs/utils.ts | 13 +++----- 3 files changed, 15 insertions(+), 33 deletions(-) diff --git a/packages/@aws-cdk/assets/lib/fs/copy.ts b/packages/@aws-cdk/assets/lib/fs/copy.ts index 06e858330ba85..b5766ea335e21 100644 --- a/packages/@aws-cdk/assets/lib/fs/copy.ts +++ b/packages/@aws-cdk/assets/lib/fs/copy.ts @@ -35,12 +35,12 @@ export function copyDirectory(srcDir: string, destDir: string, options: CopyOpti const destFileDir = path.dirname(destFilePath); if (!fs.existsSync(destFileDir)) { + // {recursive: true} is node 10+ // FIXME fs.mkdirpSync(destFileDir); - fs.mkdirSync(destFileDir); + fs.mkdirSync(destFileDir, {recursive: true}); } } - console.log(sourceFilePath, destFilePath); fs.copyFileSync(sourceFilePath, destFilePath); } } diff --git a/packages/@aws-cdk/assets/lib/fs/fingerprint.ts b/packages/@aws-cdk/assets/lib/fs/fingerprint.ts index b57d3bde127bf..197ecadc5ca42 100644 --- a/packages/@aws-cdk/assets/lib/fs/fingerprint.ts +++ b/packages/@aws-cdk/assets/lib/fs/fingerprint.ts @@ -3,7 +3,7 @@ import fs = require('fs'); import path = require('path'); import { CopyOptions } from './copy-options'; import { FollowMode } from './follow-mode'; -import { shouldExclude, shouldFollow } from './utils'; +import { listFilesRecursively, shouldFollow } from './utils'; const BUFFER_SIZE = 8 * 1024; const CTRL_SOH = '\x01'; @@ -38,41 +38,26 @@ export function fingerprint(fileOrDirectory: string, options: FingerprintOptions const rootDirectory = fs.statSync(fileOrDirectory).isDirectory() ? fileOrDirectory : path.dirname(fileOrDirectory); - const exclude = options.exclude || []; - // _processFileOrDirectory(fileOrDirectory); - - for (const file of listFilesRecursively(fileOrDirectory, {...options, follow}, exclude)) { - // TODO process - } - - return hash.digest('hex'); - - function _processFileOrDirectory(symbolicPath: string, realPath = symbolicPath) { - if (shouldExclude(exclude, symbolicPath)) { - return; - } + const exclude = [...(options.exclude || [])]; + for (const realPath of listFilesRecursively(fileOrDirectory, {...options, follow, exclude}, rootDirectory)) { const stat = fs.lstatSync(realPath); - const relativePath = path.relative(fileOrDirectory, symbolicPath); + const relativePath = path.relative(fileOrDirectory, realPath); if (stat.isSymbolicLink()) { const linkTarget = fs.readlinkSync(realPath); const resolvedLinkTarget = path.resolve(path.dirname(realPath), linkTarget); - if (shouldFollow(follow, rootDirectory, resolvedLinkTarget)) { - _processFileOrDirectory(symbolicPath, resolvedLinkTarget); - } else { + if (!shouldFollow(follow, rootDirectory, resolvedLinkTarget)) { _hashField(hash, `link:${relativePath}`, linkTarget); } } else if (stat.isFile()) { _hashField(hash, `file:${relativePath}`, _contentFingerprint(realPath, stat)); - } else if (stat.isDirectory()) { - for (const item of fs.readdirSync(realPath).sort()) { - _processFileOrDirectory(path.join(symbolicPath, item), path.join(realPath, item)); - } } else { - throw new Error(`Unable to hash ${symbolicPath}: it is neither a file nor a directory`); + throw new Error(`Unable to hash ${realPath}: it is neither a file nor a directory`); } } + + return hash.digest('hex'); } function _contentFingerprint(file: string, stat: fs.Stats): string { diff --git a/packages/@aws-cdk/assets/lib/fs/utils.ts b/packages/@aws-cdk/assets/lib/fs/utils.ts index 162e3207c8404..7ddda381aa403 100644 --- a/packages/@aws-cdk/assets/lib/fs/utils.ts +++ b/packages/@aws-cdk/assets/lib/fs/utils.ts @@ -60,12 +60,9 @@ export function shouldFollow(mode: FollowMode, sourceRoot: string, realPath: str } } -export function listFilesRecursively( - dir: string, - options: CopyOptions = {}, - rootDir?: string, - files: string[] = [] -): string[] { +export function listFilesRecursively(dir: string, options: CopyOptions & Required>, rootDir?: string): string[] { + const files = []; + let exclude = [...(options.exclude || [])]; rootDir = rootDir || dir; { @@ -82,7 +79,7 @@ export function listFilesRecursively( for (const file of fs.readdirSync(dir)) { let fullFilePath = path.join(dir, file); - let stat: fs.Stats | undefined = follow === FollowMode.ALWAYS + let stat: fs.Stats | undefined = options.follow === FollowMode.ALWAYS ? fs.statSync(fullFilePath) : fs.lstatSync(fullFilePath); @@ -97,7 +94,7 @@ export function listFilesRecursively( // is outside of the root directory). const targetPath = path.normalize(path.resolve(dir, target)); - if (shouldFollow(follow, rootDir, targetPath)) { + if (shouldFollow(options.follow, rootDir, targetPath)) { stat = fs.statSync(fullFilePath); if (!stat) { continue; From 13602a512765d69da57f91dea3bec9328db23119 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Mon, 21 Oct 2019 12:14:46 +0200 Subject: [PATCH 06/35] fix: implement mkdirpSync --- packages/@aws-cdk/assets/lib/fs/copy.ts | 5 +- packages/@aws-cdk/assets/lib/fs/mkdirpSync.ts | 67 +++++++++++++++++++ 2 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 packages/@aws-cdk/assets/lib/fs/mkdirpSync.ts diff --git a/packages/@aws-cdk/assets/lib/fs/copy.ts b/packages/@aws-cdk/assets/lib/fs/copy.ts index b5766ea335e21..1b1970edb919a 100644 --- a/packages/@aws-cdk/assets/lib/fs/copy.ts +++ b/packages/@aws-cdk/assets/lib/fs/copy.ts @@ -2,6 +2,7 @@ import fs = require('fs'); import path = require('path'); import { CopyOptions } from './copy-options'; import { FollowMode } from './follow-mode'; +import { mkdirpSync } from './mkdirpSync'; import { listFilesRecursively, shouldFollow } from './utils'; export function copyDirectory(srcDir: string, destDir: string, options: CopyOptions = { }, rootDir?: string) { @@ -35,9 +36,7 @@ export function copyDirectory(srcDir: string, destDir: string, options: CopyOpti const destFileDir = path.dirname(destFilePath); if (!fs.existsSync(destFileDir)) { - // {recursive: true} is node 10+ - // FIXME fs.mkdirpSync(destFileDir); - fs.mkdirSync(destFileDir, {recursive: true}); + mkdirpSync(destFileDir); } } diff --git a/packages/@aws-cdk/assets/lib/fs/mkdirpSync.ts b/packages/@aws-cdk/assets/lib/fs/mkdirpSync.ts new file mode 100644 index 0000000000000..94f68354e23b4 --- /dev/null +++ b/packages/@aws-cdk/assets/lib/fs/mkdirpSync.ts @@ -0,0 +1,67 @@ +// Slightly refactored version of fs-extra mkdirpSync +// https://github.com/jprichardson/node-fs-extra/blob/d1a01e735e81688e08688557d7a254fa8297d98e/lib/mkdirs/mkdirs.js + +import fs = require('fs'); +import path = require('path'); + +const INVALID_PATH_CHARS = /[<>:"|?*]/; +const o777 = parseInt('0777', 8); + +function getRootPath(p: string) { + const paths = path.normalize(path.resolve(p)).split(path.sep); + if (paths.length > 0) { return paths[0]; } + return null; +} + +function invalidWin32Path(p: string) { + const rp = getRootPath(p); + p = p.replace(rp || '', ''); + return INVALID_PATH_CHARS.test(p); +} + +export function mkdirpSync(p: string, opts?: any, made?: any) { + if (!opts || typeof opts !== 'object') { + opts = { mode: opts }; + } + + let mode = opts.mode; + const xfs = opts.fs || fs; + + if (process.platform === 'win32' && invalidWin32Path(p)) { + const errInval = new Error(p + ' contains invalid WIN32 path characters.'); + // @ts-ignore + errInval.code = 'EINVAL'; + throw errInval; + } + + if (mode === undefined) { + // tslint:disable-next-line: no-bitwise + mode = o777 & (~process.umask()); + } + if (!made) { made = null; } + + p = path.resolve(p); + + try { + xfs.mkdirSync(p, mode); + made = made || p; + } catch (err0) { + if (err0.code === 'ENOENT') { + if (path.dirname(p) === p) { throw err0; } + made = mkdirpSync(path.dirname(p), opts, made); + mkdirpSync(p, opts, made); + } else { + // In the case of any other error, just see if there's a dir there + // already. If so, then hooray! If not, then something is borked. + let stat; + try { + stat = xfs.statSync(p); + } catch (err1) { + throw err0; + } + if (!stat.isDirectory()) { throw err0; } + } + } + + return made; +} \ No newline at end of file From 65a3f966a2f18b1c11065ecb6b19eb2786570a90 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Mon, 21 Oct 2019 13:25:37 +0200 Subject: [PATCH 07/35] fix: symlink discovery --- packages/@aws-cdk/assets/lib/fs/utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/assets/lib/fs/utils.ts b/packages/@aws-cdk/assets/lib/fs/utils.ts index 7ddda381aa403..4f84ca3db2574 100644 --- a/packages/@aws-cdk/assets/lib/fs/utils.ts +++ b/packages/@aws-cdk/assets/lib/fs/utils.ts @@ -100,7 +100,7 @@ export function listFilesRecursively(dir: string, options: CopyOptions & Require continue; } - fullFilePath = target; + fullFilePath = path.resolve(dir, target); } } @@ -126,7 +126,7 @@ export function listFilesRecursively(dir: string, options: CopyOptions & Require continue; } - if (stat.isFile()) { + if (stat.isFile() || stat.isSymbolicLink()) { files.push(fullFilePath); } else if (stat.isDirectory()) { files.push(...listFilesRecursively(fullFilePath, { ...options, exclude }, rootDir)); From 6afead387a15b5550617c91b48b907c0650e2fd2 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Mon, 21 Oct 2019 13:34:59 +0200 Subject: [PATCH 08/35] fix: don't follow symlinks early --- packages/@aws-cdk/assets/lib/fs/utils.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/@aws-cdk/assets/lib/fs/utils.ts b/packages/@aws-cdk/assets/lib/fs/utils.ts index 4f84ca3db2574..2648b0ffe6df5 100644 --- a/packages/@aws-cdk/assets/lib/fs/utils.ts +++ b/packages/@aws-cdk/assets/lib/fs/utils.ts @@ -99,8 +99,6 @@ export function listFilesRecursively(dir: string, options: CopyOptions & Require if (!stat) { continue; } - - fullFilePath = path.resolve(dir, target); } } From 3b745bdcd920cb7a151e18f164455f3f7fe86f08 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Mon, 21 Oct 2019 13:41:53 +0200 Subject: [PATCH 09/35] fix: create empty directories --- packages/@aws-cdk/assets/lib/fs/copy.ts | 6 +++++- packages/@aws-cdk/assets/lib/fs/utils.ts | 8 +++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/assets/lib/fs/copy.ts b/packages/@aws-cdk/assets/lib/fs/copy.ts index 1b1970edb919a..0af53d8f006e7 100644 --- a/packages/@aws-cdk/assets/lib/fs/copy.ts +++ b/packages/@aws-cdk/assets/lib/fs/copy.ts @@ -40,6 +40,10 @@ export function copyDirectory(srcDir: string, destDir: string, options: CopyOpti } } - fs.copyFileSync(sourceFilePath, destFilePath); + if (!sourceFilePath.endsWith('/')) { + fs.copyFileSync(sourceFilePath, destFilePath); + } else { + mkdirpSync(destFilePath); + } } } diff --git a/packages/@aws-cdk/assets/lib/fs/utils.ts b/packages/@aws-cdk/assets/lib/fs/utils.ts index 2648b0ffe6df5..dfe0f9da22281 100644 --- a/packages/@aws-cdk/assets/lib/fs/utils.ts +++ b/packages/@aws-cdk/assets/lib/fs/utils.ts @@ -127,7 +127,13 @@ export function listFilesRecursively(dir: string, options: CopyOptions & Require if (stat.isFile() || stat.isSymbolicLink()) { files.push(fullFilePath); } else if (stat.isDirectory()) { - files.push(...listFilesRecursively(fullFilePath, { ...options, exclude }, rootDir)); + const dirFiles = listFilesRecursively(fullFilePath, { ...options, exclude }, rootDir); + + if (dirFiles.length) { + files.push(...dirFiles); + } else if (!isExcluded) { + files.push(fullFilePath + '/'); + } } } From 14625669f52570abab57a9594389a82131bf0cb0 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Mon, 21 Oct 2019 13:42:09 +0200 Subject: [PATCH 10/35] chore: remove useless let --- packages/@aws-cdk/assets/lib/fs/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/assets/lib/fs/utils.ts b/packages/@aws-cdk/assets/lib/fs/utils.ts index dfe0f9da22281..83411553b07db 100644 --- a/packages/@aws-cdk/assets/lib/fs/utils.ts +++ b/packages/@aws-cdk/assets/lib/fs/utils.ts @@ -77,7 +77,7 @@ export function listFilesRecursively(dir: string, options: CopyOptions & Require } for (const file of fs.readdirSync(dir)) { - let fullFilePath = path.join(dir, file); + const fullFilePath = path.join(dir, file); let stat: fs.Stats | undefined = options.follow === FollowMode.ALWAYS ? fs.statSync(fullFilePath) From 5abcf669f2f096601f31b4e08f12d76258da7fe8 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Mon, 21 Oct 2019 14:03:32 +0200 Subject: [PATCH 11/35] fix: symlink fingerprinting --- packages/@aws-cdk/assets/lib/fs/fingerprint.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/@aws-cdk/assets/lib/fs/fingerprint.ts b/packages/@aws-cdk/assets/lib/fs/fingerprint.ts index 197ecadc5ca42..eeae3e786d265 100644 --- a/packages/@aws-cdk/assets/lib/fs/fingerprint.ts +++ b/packages/@aws-cdk/assets/lib/fs/fingerprint.ts @@ -49,6 +49,8 @@ export function fingerprint(fileOrDirectory: string, options: FingerprintOptions const resolvedLinkTarget = path.resolve(path.dirname(realPath), linkTarget); if (!shouldFollow(follow, rootDirectory, resolvedLinkTarget)) { _hashField(hash, `link:${relativePath}`, linkTarget); + } else { + _hashField(hash, `file:${relativePath}`, _contentFingerprint(realPath, fs.statSync(realPath))); } } else if (stat.isFile()) { _hashField(hash, `file:${relativePath}`, _contentFingerprint(realPath, stat)); From 12d896e55789dfed891a81b9545db15bc9b3179e Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Mon, 21 Oct 2019 14:05:01 +0200 Subject: [PATCH 12/35] fix: don't throw when fingerprinting directories --- packages/@aws-cdk/assets/lib/fs/fingerprint.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/assets/lib/fs/fingerprint.ts b/packages/@aws-cdk/assets/lib/fs/fingerprint.ts index eeae3e786d265..6b6f3a8627d79 100644 --- a/packages/@aws-cdk/assets/lib/fs/fingerprint.ts +++ b/packages/@aws-cdk/assets/lib/fs/fingerprint.ts @@ -54,7 +54,7 @@ export function fingerprint(fileOrDirectory: string, options: FingerprintOptions } } else if (stat.isFile()) { _hashField(hash, `file:${relativePath}`, _contentFingerprint(realPath, stat)); - } else { + } else if (!stat.isDirectory()) { throw new Error(`Unable to hash ${realPath}: it is neither a file nor a directory`); } } From 21727931275067abf700a5b6cc199603026464a5 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Mon, 21 Oct 2019 14:07:55 +0200 Subject: [PATCH 13/35] chore: remove unneeded 'exclude' cloning --- packages/@aws-cdk/assets/lib/fs/fingerprint.ts | 3 +-- packages/@aws-cdk/assets/lib/fs/utils.ts | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/assets/lib/fs/fingerprint.ts b/packages/@aws-cdk/assets/lib/fs/fingerprint.ts index 6b6f3a8627d79..651c8cdc99c51 100644 --- a/packages/@aws-cdk/assets/lib/fs/fingerprint.ts +++ b/packages/@aws-cdk/assets/lib/fs/fingerprint.ts @@ -38,9 +38,8 @@ export function fingerprint(fileOrDirectory: string, options: FingerprintOptions const rootDirectory = fs.statSync(fileOrDirectory).isDirectory() ? fileOrDirectory : path.dirname(fileOrDirectory); - const exclude = [...(options.exclude || [])]; - for (const realPath of listFilesRecursively(fileOrDirectory, {...options, follow, exclude}, rootDirectory)) { + for (const realPath of listFilesRecursively(fileOrDirectory, {...options, follow}, rootDirectory)) { const stat = fs.lstatSync(realPath); const relativePath = path.relative(fileOrDirectory, realPath); diff --git a/packages/@aws-cdk/assets/lib/fs/utils.ts b/packages/@aws-cdk/assets/lib/fs/utils.ts index 83411553b07db..1b8e76f519722 100644 --- a/packages/@aws-cdk/assets/lib/fs/utils.ts +++ b/packages/@aws-cdk/assets/lib/fs/utils.ts @@ -63,7 +63,7 @@ export function shouldFollow(mode: FollowMode, sourceRoot: string, realPath: str export function listFilesRecursively(dir: string, options: CopyOptions & Required>, rootDir?: string): string[] { const files = []; - let exclude = [...(options.exclude || [])]; + let exclude = options.exclude || []; rootDir = rootDir || dir; { const stat = fs.statSync(dir); From 47bc251dc8c2103e35557ce60769e67e7188e063 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Mon, 21 Oct 2019 14:10:09 +0200 Subject: [PATCH 14/35] chore: refactor mkdirp calls --- packages/@aws-cdk/assets/lib/fs/copy.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/packages/@aws-cdk/assets/lib/fs/copy.ts b/packages/@aws-cdk/assets/lib/fs/copy.ts index 0af53d8f006e7..32aafab3970b3 100644 --- a/packages/@aws-cdk/assets/lib/fs/copy.ts +++ b/packages/@aws-cdk/assets/lib/fs/copy.ts @@ -32,15 +32,8 @@ export function copyDirectory(srcDir: string, destDir: string, options: CopyOpti } } - { - const destFileDir = path.dirname(destFilePath); - - if (!fs.existsSync(destFileDir)) { - mkdirpSync(destFileDir); - } - } - if (!sourceFilePath.endsWith('/')) { + mkdirpSync(path.dirname(destFilePath)); fs.copyFileSync(sourceFilePath, destFilePath); } else { mkdirpSync(destFilePath); From e5bf4850212947d128f2c914a623fd7bddc6ecba Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Mon, 21 Oct 2019 14:58:49 +0200 Subject: [PATCH 15/35] chore: refactor AssetFile --- packages/@aws-cdk/assets/lib/fs/copy.ts | 17 +++----- .../@aws-cdk/assets/lib/fs/fingerprint.ts | 26 ++++++----- packages/@aws-cdk/assets/lib/fs/utils.ts | 43 +++++++++++++++---- 3 files changed, 54 insertions(+), 32 deletions(-) diff --git a/packages/@aws-cdk/assets/lib/fs/copy.ts b/packages/@aws-cdk/assets/lib/fs/copy.ts index 32aafab3970b3..0796ca5698c96 100644 --- a/packages/@aws-cdk/assets/lib/fs/copy.ts +++ b/packages/@aws-cdk/assets/lib/fs/copy.ts @@ -14,27 +14,24 @@ export function copyDirectory(srcDir: string, destDir: string, options: CopyOpti throw new Error(`${srcDir} is not a directory`); } - for (const sourceFilePath of listFilesRecursively(srcDir, {...options, follow}, rootDir)) { - const filePath = path.relative(rootDir, sourceFilePath); + for (const assetFile of listFilesRecursively(srcDir, {...options, follow}, rootDir)) { + const filePath = assetFile.relativePath; const destFilePath = path.join(destDir, filePath); if (follow !== FollowMode.ALWAYS) { - const stat = fs.lstatSync(sourceFilePath); - - if (stat && stat.isSymbolicLink()) { - const target = fs.readlinkSync(sourceFilePath); - const targetPath = path.normalize(path.resolve(srcDir, target)); + if (assetFile.isSymbolicLink) { + const targetPath = path.normalize(path.resolve(srcDir, assetFile.symlinkTarget)); if (!shouldFollow(follow, rootDir, targetPath)) { - fs.symlinkSync(target, destFilePath); + fs.symlinkSync(assetFile.symlinkTarget, destFilePath); continue; } } } - if (!sourceFilePath.endsWith('/')) { + if (!assetFile.isDirectory) { mkdirpSync(path.dirname(destFilePath)); - fs.copyFileSync(sourceFilePath, destFilePath); + fs.copyFileSync(assetFile.absolutePath, destFilePath); } else { mkdirpSync(destFilePath); } diff --git a/packages/@aws-cdk/assets/lib/fs/fingerprint.ts b/packages/@aws-cdk/assets/lib/fs/fingerprint.ts index 651c8cdc99c51..b660a9213ab2a 100644 --- a/packages/@aws-cdk/assets/lib/fs/fingerprint.ts +++ b/packages/@aws-cdk/assets/lib/fs/fingerprint.ts @@ -39,29 +39,27 @@ export function fingerprint(fileOrDirectory: string, options: FingerprintOptions ? fileOrDirectory : path.dirname(fileOrDirectory); - for (const realPath of listFilesRecursively(fileOrDirectory, {...options, follow}, rootDirectory)) { - const stat = fs.lstatSync(realPath); - const relativePath = path.relative(fileOrDirectory, realPath); + for (const assetFile of listFilesRecursively(fileOrDirectory, {...options, follow}, rootDirectory)) { + const relativePath = path.relative(fileOrDirectory, assetFile.absolutePath); - if (stat.isSymbolicLink()) { - const linkTarget = fs.readlinkSync(realPath); - const resolvedLinkTarget = path.resolve(path.dirname(realPath), linkTarget); + if (assetFile.isSymbolicLink) { + const resolvedLinkTarget = path.resolve(path.dirname(assetFile.absolutePath), assetFile.symlinkTarget); if (!shouldFollow(follow, rootDirectory, resolvedLinkTarget)) { - _hashField(hash, `link:${relativePath}`, linkTarget); + _hashField(hash, `link:${relativePath}`, assetFile.symlinkTarget); } else { - _hashField(hash, `file:${relativePath}`, _contentFingerprint(realPath, fs.statSync(realPath))); + _hashField(hash, `file:${relativePath}`, _contentFingerprint(assetFile.absolutePath, assetFile.size)); } - } else if (stat.isFile()) { - _hashField(hash, `file:${relativePath}`, _contentFingerprint(realPath, stat)); - } else if (!stat.isDirectory()) { - throw new Error(`Unable to hash ${realPath}: it is neither a file nor a directory`); + } else if (assetFile.isFile) { + _hashField(hash, `file:${relativePath}`, _contentFingerprint(assetFile.absolutePath, assetFile.size)); + } else if (!assetFile.isDirectory) { + throw new Error(`Unable to hash ${assetFile.absolutePath}: it is neither a file nor a directory`); } } return hash.digest('hex'); } -function _contentFingerprint(file: string, stat: fs.Stats): string { +function _contentFingerprint(file: string, size: number): string { const hash = crypto.createHash('sha256'); const buffer = Buffer.alloc(BUFFER_SIZE); // tslint:disable-next-line: no-bitwise @@ -75,7 +73,7 @@ function _contentFingerprint(file: string, stat: fs.Stats): string { } finally { fs.closeSync(fd); } - return `${stat.size}:${hash.digest('hex')}`; + return `${size}:${hash.digest('hex')}`; } function _hashField(hash: crypto.Hash, header: string, value: string | Buffer | DataView) { diff --git a/packages/@aws-cdk/assets/lib/fs/utils.ts b/packages/@aws-cdk/assets/lib/fs/utils.ts index 1b8e76f519722..f137256b44564 100644 --- a/packages/@aws-cdk/assets/lib/fs/utils.ts +++ b/packages/@aws-cdk/assets/lib/fs/utils.ts @@ -60,8 +60,31 @@ export function shouldFollow(mode: FollowMode, sourceRoot: string, realPath: str } } -export function listFilesRecursively(dir: string, options: CopyOptions & Required>, rootDir?: string): string[] { - const files = []; +type AssetFile = { + absolutePath: string; + relativePath: string; + isFile: boolean; + isDirectory: boolean; + size: number; +} & ({ isSymbolicLink: false } | { isSymbolicLink: true; symlinkTarget: string}); + +const generateAssetFile = (rootDir: string, fullFilePath: string, stat: fs.Stats): AssetFile => ({ + absolutePath: fullFilePath, + relativePath: path.relative(rootDir, fullFilePath), + isFile: stat.isFile(), + isDirectory: stat.isDirectory(), + size: stat.size, + isSymbolicLink: false, +}); + +const generateAssetSymlinkFile = (rootDir: string, fullFilePath: string, stat: fs.Stats, symlinkTarget: string): AssetFile => ({ + ...generateAssetFile(rootDir, fullFilePath, stat), + isSymbolicLink: true, + symlinkTarget, +}); + +export function listFilesRecursively(dir: string, options: CopyOptions & Required>, rootDir?: string): AssetFile[] { + const files: AssetFile[] = []; let exclude = options.exclude || []; rootDir = rootDir || dir; @@ -72,7 +95,7 @@ export function listFilesRecursively(dir: string, options: CopyOptions & Require } if (!stat.isDirectory()) { - return [dir]; + return [generateAssetFile(rootDir, dir, stat)]; } } @@ -87,8 +110,9 @@ export function listFilesRecursively(dir: string, options: CopyOptions & Require continue; } + let target = ''; if (stat.isSymbolicLink()) { - const target = fs.readlinkSync(fullFilePath); + target = fs.readlinkSync(fullFilePath); // determine if this is an external link (i.e. the target's absolute path // is outside of the root directory). @@ -124,15 +148,18 @@ export function listFilesRecursively(dir: string, options: CopyOptions & Require continue; } - if (stat.isFile() || stat.isSymbolicLink()) { - files.push(fullFilePath); - } else if (stat.isDirectory()) { + if (stat.isFile()) { + files.push(generateAssetFile(rootDir, fullFilePath, stat)); + } else if (stat.isSymbolicLink()) { + files.push(generateAssetSymlinkFile(rootDir, fullFilePath, stat, target)); + } else if (stat.isDirectory()) { const dirFiles = listFilesRecursively(fullFilePath, { ...options, exclude }, rootDir); if (dirFiles.length) { files.push(...dirFiles); } else if (!isExcluded) { - files.push(fullFilePath + '/'); + // helps "copy" create an empty directory + files.push(generateAssetFile(rootDir, fullFilePath, stat)); } } } From 4966740a66f63dfbbcfbab4233a3e23a064c74e4 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Mon, 21 Oct 2019 15:07:47 +0200 Subject: [PATCH 16/35] chore: refactor recursion --- packages/@aws-cdk/assets/lib/fs/utils.ts | 128 ++++++++++++----------- 1 file changed, 65 insertions(+), 63 deletions(-) diff --git a/packages/@aws-cdk/assets/lib/fs/utils.ts b/packages/@aws-cdk/assets/lib/fs/utils.ts index f137256b44564..287a968a1c0c6 100644 --- a/packages/@aws-cdk/assets/lib/fs/utils.ts +++ b/packages/@aws-cdk/assets/lib/fs/utils.ts @@ -66,7 +66,7 @@ type AssetFile = { isFile: boolean; isDirectory: boolean; size: number; -} & ({ isSymbolicLink: false } | { isSymbolicLink: true; symlinkTarget: string}); +} & ({ isSymbolicLink: false } | { isSymbolicLink: true; symlinkTarget: string }); const generateAssetFile = (rootDir: string, fullFilePath: string, stat: fs.Stats): AssetFile => ({ absolutePath: fullFilePath, @@ -83,83 +83,85 @@ const generateAssetSymlinkFile = (rootDir: string, fullFilePath: string, stat: f symlinkTarget, }); -export function listFilesRecursively(dir: string, options: CopyOptions & Required>, rootDir?: string): AssetFile[] { +export function listFilesRecursively(dir: string, options: CopyOptions & Required>, _rootDir?: string): AssetFile[] { const files: AssetFile[] = []; - let exclude = options.exclude || []; - rootDir = rootDir || dir; - { - const stat = fs.statSync(dir); - if (!stat) { - return []; - } - - if (!stat.isDirectory()) { - return [generateAssetFile(rootDir, dir, stat)]; - } - } + const rootDir = _rootDir || dir; + const followStatsFn = options.follow === FollowMode.ALWAYS ? fs.statSync : fs.lstatSync; - for (const file of fs.readdirSync(dir)) { - const fullFilePath = path.join(dir, file); + recurse(dir); - let stat: fs.Stats | undefined = options.follow === FollowMode.ALWAYS - ? fs.statSync(fullFilePath) - : fs.lstatSync(fullFilePath); + function recurse(currentPath: string): void { + { + const stat = fs.statSync(currentPath); + if (!stat) { + return; + } - if (!stat) { - continue; + if (!stat.isDirectory()) { + files.push(generateAssetFile(rootDir, currentPath, stat)); + return; + } } - let target = ''; - if (stat.isSymbolicLink()) { - target = fs.readlinkSync(fullFilePath); + for (const file of fs.readdirSync(currentPath)) { + const fullFilePath = path.join(currentPath, file); - // determine if this is an external link (i.e. the target's absolute path - // is outside of the root directory). - const targetPath = path.normalize(path.resolve(dir, target)); - - if (shouldFollow(options.follow, rootDir, targetPath)) { - stat = fs.statSync(fullFilePath); - if (!stat) { - continue; - } + let stat: fs.Stats | undefined = followStatsFn(fullFilePath); + if (!stat) { + continue; } - } - const relativeFilePath = path.relative(rootDir, fullFilePath); - - // we've just discovered that we have a directory - if (stat.isDirectory()) { - // to help future shouldExclude calls, we're changing the exlusion patterns - // by expliciting "dir" exclusions to "dir/*" (same with "!dir" -> "!dir/*") - exclude = exclude.reduce((res, pattern) => { - res.push(pattern); - if (pattern.trim().replace(/^!/, '') === relativeFilePath) { - // we add the pattern immediately after to preserve the exclusion order - res.push(`${pattern}/*`); + let target = ''; + if (stat.isSymbolicLink()) { + target = fs.readlinkSync(fullFilePath); + + // determine if this is an external link (i.e. the target's absolute path + // is outside of the root directory). + const targetPath = path.normalize(path.resolve(currentPath, target)); + + if (shouldFollow(options.follow, rootDir, targetPath)) { + stat = fs.statSync(fullFilePath); + if (!stat) { + continue; + } } + } - return res; - }, []); - } + const relativeFilePath = path.relative(rootDir, fullFilePath); + + // we've just discovered that we have a directory + if (stat.isDirectory()) { + // to help future shouldExclude calls, we're changing the exlusion patterns + // by expliciting "dir" exclusions to "dir/*" (same with "!dir" -> "!dir/*") + exclude = exclude.reduce((res, pattern) => { + res.push(pattern); + if (pattern.trim().replace(/^!/, '') === relativeFilePath) { + // we add the pattern immediately after to preserve the exclusion order + res.push(`${pattern}/*`); + } + + return res; + }, []); + } - const isExcluded = shouldExclude(exclude, relativeFilePath); - if (isExcluded && !stat.isDirectory()) { - continue; - } + const isExcluded = shouldExclude(exclude, relativeFilePath); + if (isExcluded && !stat.isDirectory()) { + continue; + } - if (stat.isFile()) { - files.push(generateAssetFile(rootDir, fullFilePath, stat)); - } else if (stat.isSymbolicLink()) { - files.push(generateAssetSymlinkFile(rootDir, fullFilePath, stat, target)); - } else if (stat.isDirectory()) { - const dirFiles = listFilesRecursively(fullFilePath, { ...options, exclude }, rootDir); - - if (dirFiles.length) { - files.push(...dirFiles); - } else if (!isExcluded) { - // helps "copy" create an empty directory + if (stat.isFile()) { files.push(generateAssetFile(rootDir, fullFilePath, stat)); + } else if (stat.isSymbolicLink()) { + files.push(generateAssetSymlinkFile(rootDir, fullFilePath, stat, target)); + } else if (stat.isDirectory()) { + const previousLength = files.length; + recurse(fullFilePath); + + if (files.length === previousLength && !isExcluded) { + // helps "copy" create an empty directory + files.push(generateAssetFile(rootDir, fullFilePath, stat)); + } } } } From 6059be04d8a9f4cfc1eb5664f6da83a97557f0de Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Mon, 21 Oct 2019 15:12:51 +0200 Subject: [PATCH 17/35] chore: prevent unneeded stats call --- packages/@aws-cdk/assets/lib/fs/utils.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/assets/lib/fs/utils.ts b/packages/@aws-cdk/assets/lib/fs/utils.ts index 287a968a1c0c6..d53bc856a7e10 100644 --- a/packages/@aws-cdk/assets/lib/fs/utils.ts +++ b/packages/@aws-cdk/assets/lib/fs/utils.ts @@ -91,9 +91,9 @@ export function listFilesRecursively(dir: string, options: CopyOptions & Require recurse(dir); - function recurse(currentPath: string): void { + function recurse(currentPath: string, currentStat?: fs.Stats): void { { - const stat = fs.statSync(currentPath); + const stat = currentStat || fs.statSync(currentPath); if (!stat) { return; } @@ -156,7 +156,7 @@ export function listFilesRecursively(dir: string, options: CopyOptions & Require files.push(generateAssetSymlinkFile(rootDir, fullFilePath, stat, target)); } else if (stat.isDirectory()) { const previousLength = files.length; - recurse(fullFilePath); + recurse(fullFilePath, stat); if (files.length === previousLength && !isExcluded) { // helps "copy" create an empty directory From 4a1d7b64a9adc41a7a8fa74bfd791514380515a3 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Tue, 22 Oct 2019 11:32:09 +0200 Subject: [PATCH 18/35] chore: createFsStructureFromTree, remove empty files --- .../dockerignore-image-advanced/.dockerignore | 22 --- .../dockerignore-image-advanced/Dockerfile | 5 - .../config/config-prod.txt | 0 .../config/config-test.txt | 0 .../config/config.txt | 0 .../deep/dir/struct/qux.txt | 0 .../deep/include_me/sub/dir/quuz.txt | 0 .../test/dockerignore-image-advanced/foo.txt | 0 .../dockerignore-image-advanced/foobar.txt | 0 .../test/dockerignore-image-advanced/index.py | 33 ---- .../subdirectory/baz.txt | 0 .../subdirectory/quux.txt | 0 .../dockerignore-image-negative/.dockerignore | 8 - .../dockerignore-image-negative/Dockerfile | 5 - .../deep/dir/struct/qux.txt | 0 .../dockerignore-image-negative/foobar.txt | 0 .../test/dockerignore-image-negative/index.py | 33 ---- .../subdirectory/baz.txt | 0 .../subdirectory/foo.txt | 0 .../test/dockerignore-image/.dockerignore | 1 - .../test/dockerignore-image/Dockerfile | 5 - .../test/dockerignore-image/foobar.txt | 0 .../test/dockerignore-image/index.py | 33 ---- .../dockerignore-image/subdirectory/baz.txt | 0 .../aws-ecr-assets/test/test.image-asset.ts | 159 ++++++++++++++++-- 25 files changed, 148 insertions(+), 156 deletions(-) delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/.dockerignore delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/Dockerfile delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/config/config-prod.txt delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/config/config-test.txt delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/config/config.txt delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/deep/dir/struct/qux.txt delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/deep/include_me/sub/dir/quuz.txt delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/foo.txt delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/foobar.txt delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/index.py delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/subdirectory/baz.txt delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/subdirectory/quux.txt delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/.dockerignore delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/Dockerfile delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/deep/dir/struct/qux.txt delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/foobar.txt delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/index.py delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/subdirectory/baz.txt delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/subdirectory/foo.txt delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/.dockerignore delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/Dockerfile delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/foobar.txt delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/index.py delete mode 100644 packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/subdirectory/baz.txt diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/.dockerignore b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/.dockerignore deleted file mode 100644 index 6cae4e26f057c..0000000000000 --- a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/.dockerignore +++ /dev/null @@ -1,22 +0,0 @@ -# This a comment, followed by an empty line - -# The following line should be ignored -#index.py - -# This shouldn't ignore foo.txt -foo.? -# This shoul ignore foobar.txt -foobar.??? -# This should catch qux.txt -deep/**/*.txt -# but quuz should be added back -!deep/include_me/** - -# baz and quux should be ignored -subdirectory/** -# but baz should be added back -!subdirectory/baz* - -config/config*.txt -!config/config-*.txt -config/config-test.txt \ No newline at end of file diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/Dockerfile b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/Dockerfile deleted file mode 100644 index 123b5670febc8..0000000000000 --- a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM python:3.6 -EXPOSE 8000 -WORKDIR /src -ADD . /src -CMD python3 index.py diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/config/config-prod.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/config/config-prod.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/config/config-test.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/config/config-test.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/config/config.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/config/config.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/deep/dir/struct/qux.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/deep/dir/struct/qux.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/deep/include_me/sub/dir/quuz.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/deep/include_me/sub/dir/quuz.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/foo.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/foo.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/foobar.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/foobar.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/index.py b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/index.py deleted file mode 100644 index 2ccedfce3ab76..0000000000000 --- a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/index.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/python -import sys -import textwrap -import http.server -import socketserver - -PORT = 8000 - - -class Handler(http.server.SimpleHTTPRequestHandler): - def do_GET(self): - self.send_response(200) - self.send_header('Content-Type', 'text/html') - self.end_headers() - self.wfile.write(textwrap.dedent('''\ - - It works - -

Hello from the integ test container

-

This container got built and started as part of the integ test.

- - - ''').encode('utf-8')) - - -def main(): - httpd = http.server.HTTPServer(("", PORT), Handler) - print("serving at port", PORT) - httpd.serve_forever() - - -if __name__ == '__main__': - main() diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/subdirectory/baz.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/subdirectory/baz.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/subdirectory/quux.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-advanced/subdirectory/quux.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/.dockerignore b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/.dockerignore deleted file mode 100644 index d8b3df3b4c01a..0000000000000 --- a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/.dockerignore +++ /dev/null @@ -1,8 +0,0 @@ -# Comment - -* -!index.py -!subdirectory -subdirectory/foo.txt - -# Dockerfile isn't explicitly included, but we'll add it anyway to build the image \ No newline at end of file diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/Dockerfile b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/Dockerfile deleted file mode 100644 index 123b5670febc8..0000000000000 --- a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM python:3.6 -EXPOSE 8000 -WORKDIR /src -ADD . /src -CMD python3 index.py diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/deep/dir/struct/qux.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/deep/dir/struct/qux.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/foobar.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/foobar.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/index.py b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/index.py deleted file mode 100644 index 2ccedfce3ab76..0000000000000 --- a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/index.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/python -import sys -import textwrap -import http.server -import socketserver - -PORT = 8000 - - -class Handler(http.server.SimpleHTTPRequestHandler): - def do_GET(self): - self.send_response(200) - self.send_header('Content-Type', 'text/html') - self.end_headers() - self.wfile.write(textwrap.dedent('''\ - - It works - -

Hello from the integ test container

-

This container got built and started as part of the integ test.

- - - ''').encode('utf-8')) - - -def main(): - httpd = http.server.HTTPServer(("", PORT), Handler) - print("serving at port", PORT) - httpd.serve_forever() - - -if __name__ == '__main__': - main() diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/subdirectory/baz.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/subdirectory/baz.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/subdirectory/foo.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image-negative/subdirectory/foo.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/.dockerignore b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/.dockerignore deleted file mode 100644 index b7c7139ddb1a4..0000000000000 --- a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -foobar.txt diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/Dockerfile b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/Dockerfile deleted file mode 100644 index 123b5670febc8..0000000000000 --- a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM python:3.6 -EXPOSE 8000 -WORKDIR /src -ADD . /src -CMD python3 index.py diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/foobar.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/foobar.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/index.py b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/index.py deleted file mode 100644 index 2ccedfce3ab76..0000000000000 --- a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/index.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/python -import sys -import textwrap -import http.server -import socketserver - -PORT = 8000 - - -class Handler(http.server.SimpleHTTPRequestHandler): - def do_GET(self): - self.send_response(200) - self.send_header('Content-Type', 'text/html') - self.end_headers() - self.wfile.write(textwrap.dedent('''\ - - It works - -

Hello from the integ test container

-

This container got built and started as part of the integ test.

- - - ''').encode('utf-8')) - - -def main(): - httpd = http.server.HTTPServer(("", PORT), Handler) - print("serving at port", PORT) - httpd.serve_forever() - - -if __name__ == '__main__': - main() diff --git a/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/subdirectory/baz.txt b/packages/@aws-cdk/aws-ecr-assets/test/dockerignore-image/subdirectory/baz.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts b/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts index 85f9dfe588dc6..37950b5585f12 100644 --- a/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts +++ b/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts @@ -3,6 +3,7 @@ import iam = require('@aws-cdk/aws-iam'); import { App, Lazy, Stack } from '@aws-cdk/core'; import fs = require('fs'); import { Test } from 'nodeunit'; +import os = require('os'); import path = require('path'); import { DockerImageAsset } from '../lib'; @@ -212,9 +213,17 @@ export = { const app = new App(); const stack = new Stack(app, 'stack'); - const image = new DockerImageAsset(stack, 'MyAsset', { - directory: path.join(__dirname, 'dockerignore-image') - }); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dockerignore-image')); + createFsStructureFromTree(directory, ` + ├── Dockerfile + ├── .dockerignore + ├── foobar.txt + ├── index.py + └── subdirectory + └── baz.txt`); + fs.writeFileSync(path.join(directory, '.dockerignore'), 'foobar.txt'); + + const image = new DockerImageAsset(stack, 'MyAsset', { directory }); const session = app.synth(); @@ -244,8 +253,18 @@ export = { const app = new App(); const stack = new Stack(app, 'stack'); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dockerignore-image')); + createFsStructureFromTree(directory, ` + ├── Dockerfile + ├── .dockerignore + ├── foobar.txt + ├── index.py + └── subdirectory + └── baz.txt`); + fs.writeFileSync(path.join(directory, '.dockerignore'), 'foobar.txt'); + const image = new DockerImageAsset(stack, 'MyAsset', { - directory: path.join(__dirname, 'dockerignore-image'), + directory, exclude: ['subdirectory'] }); @@ -276,14 +295,62 @@ export = { const app = new App(); const stack = new Stack(app, 'stack'); - const image = new DockerImageAsset(stack, 'MyAsset', { - directory: path.join(__dirname, 'dockerignore-image-advanced') - }); - + // GIVEN + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dockerignore-image-advanced')); + createFsStructureFromTree(directory, ` + ├── config + │   ├── config-prod.txt + │   ├── config-test.txt + │   └── config.txt + ├── deep + │   ├── dir + │   │   └── struct + │   │   └── qux.txt + │   └── include_me + │   └── sub + │   └── dir + │   └── quuz.txt + ├── foobar.txt + ├── foo.txt + ├── Dockerfile + ├── index.py + ├── .hidden-file + └── empty-directory (D) + └── subdirectory + ├── baz.txt + └── quux.txt`); + + fs.writeFileSync(path.join(directory, '.dockerignore'), ` + # This a comment, followed by an empty line + + # The following line should be ignored + #index.py + + # This shouldn't ignore foo.txt + foo.? + # This shoul ignore foobar.txt + foobar.??? + # This should catch qux.txt + deep/**/*.txt + # but quuz should be added back + !deep/include_me/** + + # baz and quux should be ignored + subdirectory/** + # but baz should be added back + !subdirectory/baz* + + config/config*.txt + !config/config-*.txt + config/config-test.txt + `.split('\n').map(line => line.trim()).join('\n')); + + const image = new DockerImageAsset(stack, 'MyAsset', { directory }); const session = app.synth(); const expectedFiles = [ '.dockerignore', + '.hidden-file', 'Dockerfile', 'index.py', 'foo.txt', @@ -313,9 +380,32 @@ export = { const app = new App(); const stack = new Stack(app, 'stack'); - const image = new DockerImageAsset(stack, 'MyAsset', { - directory: path.join(__dirname, 'dockerignore-image-negative') - }); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dockerignore-image-advanced')); + createFsStructureFromTree(directory, ` + ├── deep + │   └── dir + │   └── struct + │   └── qux.txt + ├── Dockerfile + ├── .dockerignore + ├── foobar.txt + ├── index.py + └── subdirectory + ├── baz.txt + └── foo.txt`); + + fs.writeFileSync(path.join(directory, '.dockerignore'), ` + # Comment + + * + !index.py + !subdirectory + subdirectory/foo.txt + + # Dockerfile isn't explicitly included, but we'll add it anyway to build the image + `.split('\n').map(line => line.trim()).join('\n')); + + const image = new DockerImageAsset(stack, 'MyAsset', { directory }); const session = app.synth(); @@ -377,3 +467,50 @@ export = { test.done(); } }; + +const INDENT_CHARACTERS_REGEX = /^[\s├─│└]+/; +const TRAILING_CHARACTERS_REGEX = /(\/|\(D\))\s*$/i; +const IS_DIRECTORY_REGEX = /\(D\)\s*$/i; + +const createFsStructureFromTree = (parentDir: string, tree: string): void => { + const directories: string[] = []; + const files: string[] = []; + + // we push an element at the end because we push the files/directories during the previous iteration + const lines = [...tree.replace(/^\n/, '').split('\n'), '']; + const initialIndentLevel = (lines[0].match(/^\s*/) || [''])[0].length; + + lines.reduce<[string, number, boolean]>(([previousDir, previousIndentLevel, wasDirectory], line) => { + const indentCharacters = (line.match(INDENT_CHARACTERS_REGEX) || [''])[0]; + const indentLevel = (indentCharacters.length - initialIndentLevel) / 4; + + const fileName = line.slice(indentCharacters.length).replace(TRAILING_CHARACTERS_REGEX, ''); + + const current = indentLevel <= previousIndentLevel ? + path.join( + ...previousDir.split(path.sep) + .slice(0, indentLevel - 1), + // .slice(0, indentLevel === previousIndentLevel ? indentLevel - 1 : indentLevel), + fileName + ) : + path.join(previousDir, fileName); + + if (previousDir) { + if (indentLevel > previousIndentLevel || wasDirectory) { + directories.push(previousDir); + } else { + files.push(previousDir); + } + } + + return [current, indentLevel, IS_DIRECTORY_REGEX.test(line)]; + }, ['', 0, false]); + + for (const directory of directories) { + fs.mkdirSync(path.join(parentDir, directory)); + } + + for (const file of files) { + fs.writeFileSync(path.join(parentDir, file), 'content'); + } +}; \ No newline at end of file From 7f4096205c7db9a066ca9c7cde0b133d0aa42d6c Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Tue, 22 Oct 2019 11:34:44 +0200 Subject: [PATCH 19/35] chore: cleanup --- .../@aws-cdk/aws-ecr-assets/test/test.image-asset.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts b/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts index 37950b5585f12..f4e5f7705b775 100644 --- a/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts +++ b/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts @@ -487,12 +487,7 @@ const createFsStructureFromTree = (parentDir: string, tree: string): void => { const fileName = line.slice(indentCharacters.length).replace(TRAILING_CHARACTERS_REGEX, ''); const current = indentLevel <= previousIndentLevel ? - path.join( - ...previousDir.split(path.sep) - .slice(0, indentLevel - 1), - // .slice(0, indentLevel === previousIndentLevel ? indentLevel - 1 : indentLevel), - fileName - ) : + path.join(...previousDir.split(path.sep).slice(0, indentLevel - 1), fileName) : path.join(previousDir, fileName); if (previousDir) { @@ -513,4 +508,4 @@ const createFsStructureFromTree = (parentDir: string, tree: string): void => { for (const file of files) { fs.writeFileSync(path.join(parentDir, file), 'content'); } -}; \ No newline at end of file +}; From baa431539a0b024d9f065cfed8a27e24771ce6cc Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Tue, 22 Oct 2019 11:37:23 +0200 Subject: [PATCH 20/35] fix: empty-directory test --- packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts b/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts index f4e5f7705b775..64855e9a0004e 100644 --- a/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts +++ b/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts @@ -354,6 +354,7 @@ export = { 'Dockerfile', 'index.py', 'foo.txt', + 'empty-directory', path.join('subdirectory', 'baz.txt'), path.join('deep', 'include_me', 'sub', 'dir', 'quuz.txt'), path.join('config', 'config-prod.txt'), @@ -469,7 +470,7 @@ export = { }; const INDENT_CHARACTERS_REGEX = /^[\s├─│└]+/; -const TRAILING_CHARACTERS_REGEX = /(\/|\(D\))\s*$/i; +const TRAILING_CHARACTERS_REGEX = /\/|\(D\)$/i; const IS_DIRECTORY_REGEX = /\(D\)\s*$/i; const createFsStructureFromTree = (parentDir: string, tree: string): void => { @@ -484,7 +485,7 @@ const createFsStructureFromTree = (parentDir: string, tree: string): void => { const indentCharacters = (line.match(INDENT_CHARACTERS_REGEX) || [''])[0]; const indentLevel = (indentCharacters.length - initialIndentLevel) / 4; - const fileName = line.slice(indentCharacters.length).replace(TRAILING_CHARACTERS_REGEX, ''); + const fileName = line.slice(indentCharacters.length).replace(TRAILING_CHARACTERS_REGEX, '').trimRight(); const current = indentLevel <= previousIndentLevel ? path.join(...previousDir.split(path.sep).slice(0, indentLevel - 1), fileName) : From 273d3f51904825e33e96b0d38662644f0d09b7cb Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Wed, 23 Oct 2019 10:44:06 +0200 Subject: [PATCH 21/35] feat: shouldExcludeDeep --- packages/@aws-cdk/assets/lib/fs/utils.ts | 42 ++++++++-- .../@aws-cdk/assets/test/fs/test.utils.ts | 83 +++++++++++++++++++ 2 files changed, 118 insertions(+), 7 deletions(-) diff --git a/packages/@aws-cdk/assets/lib/fs/utils.ts b/packages/@aws-cdk/assets/lib/fs/utils.ts index d53bc856a7e10..674967d84f79e 100644 --- a/packages/@aws-cdk/assets/lib/fs/utils.ts +++ b/packages/@aws-cdk/assets/lib/fs/utils.ts @@ -14,22 +14,50 @@ import { FollowMode } from './follow-mode'; * @returns `true` if the file should be excluded */ export function shouldExclude(exclude: string[], filePath: string): boolean { - let excludeOutput = false; + const [_shouldExclude] = shouldExcludePriority(exclude, filePath); + return _shouldExclude; +} - for (const pattern of exclude) { +/** + * Determines whether a given file should be excluded or not based on given + * exclusion glob patterns. + * + * @param exclude exclusion patterns + * @param filePath file apth to be assessed against the pattern + * + * @returns `true` if the file should be excluded, followed by the index of the rule applied + */ +export function shouldExcludePriority(exclude: string[], filePath: string): [boolean, number] { + return exclude.reduce<[boolean, number]>((res, pattern, patternIndex) => { const negate = pattern.startsWith('!'); const match = minimatch(filePath, pattern, { matchBase: true, flipNegate: true }); if (!negate && match) { - excludeOutput = true; + res = [true, patternIndex]; } if (negate && match) { - excludeOutput = false; + res = [false, patternIndex]; } - } - return excludeOutput; + return res; + }, [false, -1]); +} + +export function shouldExcludeDeep(exclude: string[], relativePath: string): boolean { + const [_shouldExclude] = relativePath.split(path.sep).reduce<[boolean, number, string]>( + ([accExclude, accPriority, pathIterator], pathComponent) => { + pathIterator = path.join(pathIterator, pathComponent); + + const [shouldExcludeIt, priorityIt] = shouldExcludePriority(exclude, pathIterator); + if (priorityIt > accPriority) { + return [shouldExcludeIt, priorityIt, pathIterator]; + } + + return [accExclude, accPriority, pathIterator]; + }, [false, -1, '']); + + return _shouldExclude; } /** @@ -167,4 +195,4 @@ export function listFilesRecursively(dir: string, options: CopyOptions & Require } return files; -} \ No newline at end of file +} diff --git a/packages/@aws-cdk/assets/test/fs/test.utils.ts b/packages/@aws-cdk/assets/test/fs/test.utils.ts index d0d4cde73dcea..d6aa7849707e1 100644 --- a/packages/@aws-cdk/assets/test/fs/test.utils.ts +++ b/packages/@aws-cdk/assets/test/fs/test.utils.ts @@ -192,4 +192,87 @@ export = { }, } }, + + shouldExcludeDeep: { + 'basic usage'(test: Test) { + testShouldExcludeDeep(test, ['foo.txt'], [ + 'foo.txt', + 'foo.txt/file', + 'dir/foo.txt', + ], [ + 'bar.txt', + 'foo', + 'foo.txt.old', + ]); + + test.done(); + }, + 'contridactory'(test: Test) { + testShouldExcludeDeep(test, ['foo.txt', '!foo.txt'], [], ['foo.txt']); + + test.done(); + }, + 'dir single wildcard'(test: Test) { + testShouldExcludeDeep(test, ['d?r'], [ + 'dir', + 'dir/exclude', + 'dir/exclude/file', + ], [ + 'door', + 'door/file', + ]); + + test.done(); + }, + 'dir wildcard'(test: Test) { + testShouldExcludeDeep(test, ['d*r'], [ + 'dir', + 'dir/file', + 'door', + 'door/file', + ], [ + 'dog', + 'dog/file', + ]); + + test.done(); + }, + 'deep structure'(test: Test) { + testShouldExcludeDeep(test, ['deep/exclude'], [ + 'deep/exclude', + 'deep/exclude/file', + ], [ + 'deep', + 'deep/include', + 'deep/include/file', + ]); + + test.done(); + }, + 'inverted pattern'(test: Test) { + testShouldExcludeDeep(test, ['*', '!foo.txt', '!d?r', 'dir/exclude'], [ + 'bar.txt', + 'dir/exclude', + 'dir/exclude/file', + ], [ + '.hidden-file', + 'foo.txt', + 'dir', + 'dir/include', + 'dir/include/subdir', + 'exclude/foo.txt', + ]); + + test.done(); + }, + }, +}; + +const testShouldExcludeDeep = (test: Test, pattern: string[], expectExclude: string[], expectInclude: string[]) => { + for (const include of expectExclude) { + test.ok(util.shouldExcludeDeep(pattern, include), `${include} should have been included, but wasn't`); + } + for (const exclude of expectInclude) { + test.ok(!util.shouldExcludeDeep(pattern, exclude), `${exclude} should have been excluded, but wasn't`); + } }; From 10e62b576d3fa834cf6208cc9a681b8c797fb65c Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Fri, 25 Oct 2019 10:34:57 +0200 Subject: [PATCH 22/35] chore: fromTree in @/assert, cleanup fn, test --- packages/@aws-cdk/assert/lib/fs-utils.ts | 88 +++++++++++++++++++ packages/@aws-cdk/assert/lib/index.ts | 1 + .../@aws-cdk/assert/test/test.fs-utils.ts | 53 +++++++++++ .../aws-ecr-assets/test/test.image-asset.ts | 64 +++----------- 4 files changed, 153 insertions(+), 53 deletions(-) create mode 100644 packages/@aws-cdk/assert/lib/fs-utils.ts create mode 100644 packages/@aws-cdk/assert/test/test.fs-utils.ts diff --git a/packages/@aws-cdk/assert/lib/fs-utils.ts b/packages/@aws-cdk/assert/lib/fs-utils.ts new file mode 100644 index 0000000000000..fb98bf0b62a56 --- /dev/null +++ b/packages/@aws-cdk/assert/lib/fs-utils.ts @@ -0,0 +1,88 @@ +import fs = require('fs'); +import os = require('os'); +import path = require('path'); + +interface FromTreeOutput { + /** + * Absolute path of the created temporary directory, containing the generated structure + */ + readonly directory: string; + /** + * Cleanup function that will remove the generated files once called + */ + readonly cleanup: () => void; +} + +/** + * Collection of file-system utility methods + */ +export class FsUtils { + /** + * Generates a filesystem structure from a string, + * formatted like the output of a `tree` shell command + * + * @param tmpPrefix temp directory prefix, used by {@link fs.mkdtemp} + * @param tree + * @param content the content + * + * @returns an array containing the absolute path of the created directory, + * and a cleanup function that will remove the generated files when called + */ + public static fromTree(tmpPrefix: string, tree: string, content = 'content'): FromTreeOutput { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), tmpPrefix)); + + const directories: string[] = []; + const files: string[] = []; + + // we push an element at the end because we push the files/directories during the previous iteration + const lines = [...tree.replace(/^\n/, '').split('\n'), '']; + const initialIndentLevel = (lines[0].match(/^\s*/) || [''])[0].length; + + lines.reduce<[string, number, boolean]>(([previousDir, previousIndentLevel, wasDirectory], line) => { + const indentCharacters = (line.match(FsUtils.INDENT_CHARACTERS_REGEX) || [''])[0]; + const indentLevel = (indentCharacters.length - initialIndentLevel) / 4; + + const fileName = line.slice(indentCharacters.length).replace(FsUtils.TRAILING_CHARACTERS_REGEX, '').trimRight(); + + const current = indentLevel <= previousIndentLevel ? + path.join(...previousDir.split(path.sep).slice(0, indentLevel - 1), fileName) : + path.join(previousDir, fileName); + + if (previousDir) { + if (indentLevel > previousIndentLevel || wasDirectory) { + directories.push(previousDir); + } else { + files.push(previousDir); + } + } + + return [current, indentLevel, FsUtils.IS_DIRECTORY_REGEX.test(line)]; + }, ['', 0, false]); + + for (const _directory of directories) { + fs.mkdirSync(path.join(directory, _directory)); + } + for (const file of files) { + fs.writeFileSync(path.join(directory, file), content); + } + + return { + directory, + cleanup: () => { + for (const file of files) { + fs.unlinkSync(path.join(directory, file)); + } + for (const _directory of directories.reverse()) { + fs.rmdirSync(path.join(directory, _directory)); + } + + fs.rmdirSync(directory); + } + }; + } + + private static readonly INDENT_CHARACTERS_REGEX = /^[\s├─│└]+/; + private static readonly TRAILING_CHARACTERS_REGEX = /\/|\(D\)$/i; + private static readonly IS_DIRECTORY_REGEX = /\(D\)\s*$/i; + +} \ No newline at end of file diff --git a/packages/@aws-cdk/assert/lib/index.ts b/packages/@aws-cdk/assert/lib/index.ts index b79d93592affc..a7aa6078b3bef 100644 --- a/packages/@aws-cdk/assert/lib/index.ts +++ b/packages/@aws-cdk/assert/lib/index.ts @@ -1,5 +1,6 @@ export * from './assertion'; export * from './expect'; +export * from './fs-utils'; export * from './inspector'; export * from './synth-utils'; diff --git a/packages/@aws-cdk/assert/test/test.fs-utils.ts b/packages/@aws-cdk/assert/test/test.fs-utils.ts new file mode 100644 index 0000000000000..b286fed340d7a --- /dev/null +++ b/packages/@aws-cdk/assert/test/test.fs-utils.ts @@ -0,0 +1,53 @@ +import fs = require('fs'); +import { Test } from 'nodeunit'; +import path = require('path'); +import { FsUtils } from '../lib'; + +export = { + fromTree: { + 'basic usage'(test: Test) { + // GIVEN + const tree = ` + ├── foo + └── dir + └── subdir + └── bar.txt`; + + // THEN + const {directory, cleanup} = FsUtils.fromTree('basic-usage', tree); + + test.ok(fs.existsSync(path.join(directory, 'foo'))); + test.ok(fs.existsSync(path.join(directory, 'dir', 'subdir', 'bar.txt'))); + + cleanup(); + + test.ok(!fs.existsSync(path.join(directory, 'foo'))); + test.ok(!fs.existsSync(path.join(directory, 'dir', 'subdir', 'bar.txt'))); + test.ok(!fs.existsSync(directory)); + + test.done(); + }, + 'works with any indent'(test: Test) { + // GIVEN + const tree = ` + ├── foo + └── dir + └── subdir + └── bar.txt`; + + // THEN + const {directory, cleanup} = FsUtils.fromTree('basic-usage', tree); + + test.ok(fs.existsSync(path.join(directory, 'foo'))); + test.ok(fs.existsSync(path.join(directory, 'dir', 'subdir', 'bar.txt'))); + + cleanup(); + + test.ok(!fs.existsSync(path.join(directory, 'foo'))); + test.ok(!fs.existsSync(path.join(directory, 'dir', 'subdir', 'bar.txt'))); + test.ok(!fs.existsSync(directory)); + + test.done(); + }, + }, +}; \ No newline at end of file diff --git a/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts b/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts index ac28ff73b83e3..1559adf56eef5 100644 --- a/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts +++ b/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts @@ -1,10 +1,9 @@ -import { expect, haveResource, SynthUtils } from '@aws-cdk/assert'; +import { expect, FsUtils, haveResource, SynthUtils } from '@aws-cdk/assert'; import iam = require('@aws-cdk/aws-iam'); import { App, Construct, Lazy, Resource, Stack } from '@aws-cdk/core'; import { ASSET_METADATA } from '@aws-cdk/cx-api'; import fs = require('fs'); import { Test } from 'nodeunit'; -import os = require('os'); import path = require('path'); import { DockerImageAsset } from '../lib'; @@ -235,8 +234,7 @@ export = { const app = new App(); const stack = new Stack(app, 'stack'); - const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dockerignore-image')); - createFsStructureFromTree(directory, ` + const {directory, cleanup} = FsUtils.fromTree('dockerignore-image', ` ├── Dockerfile ├── .dockerignore ├── foobar.txt @@ -268,6 +266,7 @@ export = { test.ok(!fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, unexpectedFile)), unexpectedFile); } + cleanup(); test.done(); }, @@ -275,8 +274,7 @@ export = { const app = new App(); const stack = new Stack(app, 'stack'); - const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dockerignore-image')); - createFsStructureFromTree(directory, ` + const {directory, cleanup} = FsUtils.fromTree('dockerignore-image', ` ├── Dockerfile ├── .dockerignore ├── foobar.txt @@ -310,6 +308,7 @@ export = { test.ok(!fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, unexpectedFile)), unexpectedFile); } + cleanup(); test.done(); }, @@ -318,8 +317,7 @@ export = { const stack = new Stack(app, 'stack'); // GIVEN - const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dockerignore-image-advanced')); - createFsStructureFromTree(directory, ` + const {directory, cleanup} = FsUtils.fromTree('dockerignore-image-advanced', ` ├── config │   ├── config-prod.txt │   ├── config-test.txt @@ -334,6 +332,7 @@ export = { │   └── quuz.txt ├── foobar.txt ├── foo.txt + ├── .dockerignore ├── Dockerfile ├── index.py ├── .hidden-file @@ -396,15 +395,15 @@ export = { test.ok(!fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, unexpectedFile)), unexpectedFile); } + cleanup(); test.done(); }, - 'negagive .dockerignore test case'(test: Test) { + 'negative .dockerignore test case'(test: Test) { const app = new App(); const stack = new Stack(app, 'stack'); - const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dockerignore-image-advanced')); - createFsStructureFromTree(directory, ` + const {directory, cleanup} = FsUtils.fromTree('dockerignore-image-advanced', ` ├── deep │   └── dir │   └── struct @@ -453,6 +452,7 @@ export = { test.ok(!fs.existsSync(path.join(session.directory, `asset.${image.sourceHash}`, unexpectedFile)), unexpectedFile); } + cleanup(); test.done(); }, @@ -490,45 +490,3 @@ export = { test.done(); } }; - -const INDENT_CHARACTERS_REGEX = /^[\s├─│└]+/; -const TRAILING_CHARACTERS_REGEX = /\/|\(D\)$/i; -const IS_DIRECTORY_REGEX = /\(D\)\s*$/i; - -const createFsStructureFromTree = (parentDir: string, tree: string): void => { - const directories: string[] = []; - const files: string[] = []; - - // we push an element at the end because we push the files/directories during the previous iteration - const lines = [...tree.replace(/^\n/, '').split('\n'), '']; - const initialIndentLevel = (lines[0].match(/^\s*/) || [''])[0].length; - - lines.reduce<[string, number, boolean]>(([previousDir, previousIndentLevel, wasDirectory], line) => { - const indentCharacters = (line.match(INDENT_CHARACTERS_REGEX) || [''])[0]; - const indentLevel = (indentCharacters.length - initialIndentLevel) / 4; - - const fileName = line.slice(indentCharacters.length).replace(TRAILING_CHARACTERS_REGEX, '').trimRight(); - - const current = indentLevel <= previousIndentLevel ? - path.join(...previousDir.split(path.sep).slice(0, indentLevel - 1), fileName) : - path.join(previousDir, fileName); - - if (previousDir) { - if (indentLevel > previousIndentLevel || wasDirectory) { - directories.push(previousDir); - } else { - files.push(previousDir); - } - } - - return [current, indentLevel, IS_DIRECTORY_REGEX.test(line)]; - }, ['', 0, false]); - - for (const directory of directories) { - fs.mkdirSync(path.join(parentDir, directory)); - } - - for (const file of files) { - fs.writeFileSync(path.join(parentDir, file), 'content'); - } -}; From c426803378b86bf3d7023c64f6a1f387c5f782f4 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Fri, 25 Oct 2019 13:12:28 +0200 Subject: [PATCH 23/35] feat: shouldExcludeDirectory --- packages/@aws-cdk/assets/lib/fs/utils.ts | 49 ++++++++++- .../@aws-cdk/assets/test/fs/test.utils.ts | 88 +++++++++++++++++-- 2 files changed, 130 insertions(+), 7 deletions(-) diff --git a/packages/@aws-cdk/assets/lib/fs/utils.ts b/packages/@aws-cdk/assets/lib/fs/utils.ts index 674967d84f79e..f367791f51b5b 100644 --- a/packages/@aws-cdk/assets/lib/fs/utils.ts +++ b/packages/@aws-cdk/assets/lib/fs/utils.ts @@ -9,7 +9,7 @@ import { FollowMode } from './follow-mode'; * exclusion glob patterns. * * @param exclude exclusion patterns - * @param filePath file apth to be assessed against the pattern + * @param filePath file path to be assessed against the pattern * * @returns `true` if the file should be excluded */ @@ -23,7 +23,7 @@ export function shouldExclude(exclude: string[], filePath: string): boolean { * exclusion glob patterns. * * @param exclude exclusion patterns - * @param filePath file apth to be assessed against the pattern + * @param filePath file patg to be assessed against the pattern * * @returns `true` if the file should be excluded, followed by the index of the rule applied */ @@ -44,6 +44,12 @@ export function shouldExcludePriority(exclude: string[], filePath: string): [boo }, [false, -1]); } +/** + * Determines whether a given file should be excluded,taking into account deep file structures + * + * @param exclude exclusion patterns + * @param filePath file path to be assessed against the pattern + */ export function shouldExcludeDeep(exclude: string[], relativePath: string): boolean { const [_shouldExclude] = relativePath.split(path.sep).reduce<[boolean, number, string]>( ([accExclude, accPriority, pathIterator], pathComponent) => { @@ -60,6 +66,44 @@ export function shouldExcludeDeep(exclude: string[], relativePath: string): bool return _shouldExclude; } +/** + * Determines whether a given directory should be excluded and not explored further + * This might be true even if the directory is explicitly excluded, + * but one of its children might be inclunded + * + * @param exclude exclusion patterns + * @param directoryPath directory path to be assessed against the pattern + */ +export function shouldExcludeDirectory(exclude: string[], directoryPath: string): boolean { + const splitPatterns = exclude.map((exc) => exc.split(path.sep)); + const patternLength = splitPatterns.map(({length}) => length); + const maxPatternLength = Math.max(...patternLength); + + const [_shouldExclude] = directoryPath.split(path.sep).reduce<[boolean, string]>( + ([accExclude, pathIterator], pathComponent) => { + pathIterator = path.join(pathIterator, pathComponent); + + for (let pattenItLength = 1; pattenItLength <= maxPatternLength; ++pattenItLength) { + const excludeSliced = splitPatterns.map((pattern) => pattern.slice(0, pattenItLength).join(path.sep)); + const [shouldExcludeIt, patternIndex] = shouldExcludePriority(excludeSliced, pathIterator); + + if (shouldExcludeIt || patternIndex < 0) { + continue; + } + + if (pattenItLength < patternLength[patternIndex]) { + accExclude = shouldExcludeIt; + } else if (!excludeSliced[patternIndex].includes('**')) { + accExclude = true; + } + } + + return [accExclude, pathIterator]; + }, [true, '']); + + return _shouldExclude; +} + /** * Determines whether a symlink should be followed or not, based on a FollowMode. * @@ -127,6 +171,7 @@ export function listFilesRecursively(dir: string, options: CopyOptions & Require } if (!stat.isDirectory()) { + // FIXME not checking wether the file was ignored or not files.push(generateAssetFile(rootDir, currentPath, stat)); return; } diff --git a/packages/@aws-cdk/assets/test/fs/test.utils.ts b/packages/@aws-cdk/assets/test/fs/test.utils.ts index d6aa7849707e1..2b829a95317cd 100644 --- a/packages/@aws-cdk/assets/test/fs/test.utils.ts +++ b/packages/@aws-cdk/assets/test/fs/test.utils.ts @@ -237,6 +237,20 @@ export = { test.done(); }, + 'dir deep wildcard'(test: Test) { + testShouldExcludeDeep(test, ['dir/**/*', '!dir/include/**/*'], [ + 'dir/deep', + 'dir/deep/file', + 'dir/deep/deeper/file', + 'dir/include', + ], [ + 'dir', + 'dir/include/deep', + 'dir/include/deep/deeper', + ]); + + test.done(); + }, 'deep structure'(test: Test) { testShouldExcludeDeep(test, ['deep/exclude'], [ 'deep/exclude', @@ -266,13 +280,77 @@ export = { test.done(); }, }, + + shouldExcludeDirectory: { + 'basic usage'(test: Test) { + const pattern = ['dir', '!dir/*', 'other_dir']; + + testShouldExcludeDeep(test, pattern, ['dir', 'other_dir'], ['dir/file']); + testShouldExcludeDirectory(test, pattern, ['dir/deep', 'other_dir'], ['dir']); + + test.done(); + }, + 'deep structure'(test: Test) { + const pattern = ['dir', '!dir/subdir/?', 'other_dir', 'really/deep/structure/of/files/and/dirs']; + + testShouldExcludeDeep(test, pattern, + ['dir', 'dir/subdir', 'other_dir'], + ['dir/subdir/a'] + ); + testShouldExcludeDirectory(test, pattern, + ['other_dir', 'dir/subdir/d'], + ['dir', 'dir/subdir'] + ); + + test.done(); + }, + 'wildcard pattern'(test: Test) { + const pattern = ['dir', '!dir/*/*', 'other_dir']; + + testShouldExcludeDeep(test, pattern, + ['dir', 'other_dir', 'dir/file'], + ['dir/file/deep'] + ); + testShouldExcludeDirectory(test, pattern, + ['other_dir', 'dir/deep/struct'], + ['dir', 'dir/deep', 'dir/deep'] + ); + + test.done(); + }, + 'deep wildcard'(test: Test) { + const pattern = ['dir', '!dir/**/*', 'other_dir']; + + testShouldExcludeDeep(test, pattern, + ['dir', 'other_dir'], + ['dir/file', 'dir/file/deep'] + ); + testShouldExcludeDirectory(test, pattern, + ['other_dir'], + ['dir', 'dir/deep', 'dir/deep/struct', 'dir/really/really/really/really/deep'] + ); + + test.done(); + }, + }, }; -const testShouldExcludeDeep = (test: Test, pattern: string[], expectExclude: string[], expectInclude: string[]) => { - for (const include of expectExclude) { - test.ok(util.shouldExcludeDeep(pattern, include), `${include} should have been included, but wasn't`); +const testShouldExclude = ( + test: Test, + pattern: string[], + expectExclude: string[], + expectInclude: string[], + shouldExcludeMethod: (pattern: string[], path: string) => boolean) => { + for (const exclude of expectExclude) { + test.ok(shouldExcludeMethod(pattern, exclude), `${exclude} should have been excluded, but wasn't`); } - for (const exclude of expectInclude) { - test.ok(!util.shouldExcludeDeep(pattern, exclude), `${exclude} should have been excluded, but wasn't`); + for (const include of expectInclude) { + test.ok(!shouldExcludeMethod(pattern, include), `${include} should have been included, but wasn't`); } }; + +const testShouldExcludeDeep = (test: Test, pattern: string[], expectExclude: string[], expectInclude: string[]) => + testShouldExclude(test, pattern, expectExclude, expectInclude, util.shouldExcludeDeep); + +const testShouldExcludeDirectory = (test: Test, pattern: string[], expectExclude: string[], expectInclude: string[]) => + testShouldExclude(test, pattern, expectExclude, expectInclude, util.shouldExcludeDirectory); \ No newline at end of file From 0c528dbaec03fda589962ee214ce0968eff0a1d8 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Fri, 25 Oct 2019 14:27:39 +0200 Subject: [PATCH 24/35] chore: refactor listFiles with new methods (missing symlinks) --- packages/@aws-cdk/assets/lib/fs/utils.ts | 78 ++++++++++++++++--- .../@aws-cdk/assets/test/fs/test.utils.ts | 71 +++++++++++++++++ 2 files changed, 138 insertions(+), 11 deletions(-) diff --git a/packages/@aws-cdk/assets/lib/fs/utils.ts b/packages/@aws-cdk/assets/lib/fs/utils.ts index f367791f51b5b..f04e8e6007e3f 100644 --- a/packages/@aws-cdk/assets/lib/fs/utils.ts +++ b/packages/@aws-cdk/assets/lib/fs/utils.ts @@ -76,18 +76,24 @@ export function shouldExcludeDeep(exclude: string[], relativePath: string): bool */ export function shouldExcludeDirectory(exclude: string[], directoryPath: string): boolean { const splitPatterns = exclude.map((exc) => exc.split(path.sep)); - const patternLength = splitPatterns.map(({length}) => length); + const patternLength = splitPatterns.map(({ length }) => length); const maxPatternLength = Math.max(...patternLength); - const [_shouldExclude] = directoryPath.split(path.sep).reduce<[boolean, string]>( + const [_shouldExclude] = directoryPath.split(path.sep).reduce<[boolean | null, string]>( ([accExclude, pathIterator], pathComponent) => { pathIterator = path.join(pathIterator, pathComponent); for (let pattenItLength = 1; pattenItLength <= maxPatternLength; ++pattenItLength) { const excludeSliced = splitPatterns.map((pattern) => pattern.slice(0, pattenItLength).join(path.sep)); const [shouldExcludeIt, patternIndex] = shouldExcludePriority(excludeSliced, pathIterator); + if (patternIndex < 0) { + continue; + } - if (shouldExcludeIt || patternIndex < 0) { + if (shouldExcludeIt) { + if (accExclude == null) { + accExclude = true; + } continue; } @@ -99,9 +105,9 @@ export function shouldExcludeDirectory(exclude: string[], directoryPath: string) } return [accExclude, pathIterator]; - }, [true, '']); + }, [null, '']); - return _shouldExclude; + return _shouldExclude || false; } /** @@ -142,7 +148,7 @@ type AssetFile = { const generateAssetFile = (rootDir: string, fullFilePath: string, stat: fs.Stats): AssetFile => ({ absolutePath: fullFilePath, - relativePath: path.relative(rootDir, fullFilePath), + relativePath: path.relative(rootDir, fullFilePath) || path.relative(path.dirname(rootDir), fullFilePath), isFile: stat.isFile(), isDirectory: stat.isDirectory(), size: stat.size, @@ -155,13 +161,58 @@ const generateAssetSymlinkFile = (rootDir: string, fullFilePath: string, stat: f symlinkTarget, }); -export function listFilesRecursively(dir: string, options: CopyOptions & Required>, _rootDir?: string): AssetFile[] { +export function listFilesRecursively( + dirOrFile: string, + options: CopyOptions & Required>, _rootDir?: string +): AssetFile[] { + const files: AssetFile[] = []; + const exclude = options.exclude || []; + const rootDir = _rootDir || dirOrFile; + const followStatsFn = options.follow === FollowMode.ALWAYS ? fs.statSync : fs.lstatSync; + + recurse(dirOrFile); + + function recurse(currentPath: string, _currentStat?: fs.Stats): void { + const currentStat = _currentStat || fs.statSync(currentPath); + if (!currentStat) { + return; + } + + for (const file of currentStat.isDirectory() ? fs.readdirSync(currentPath) : ['']) { + const fullFilePath = path.join(currentPath, file); + const relativeFilePath = path.relative(rootDir, fullFilePath); + + let stat: fs.Stats | undefined = followStatsFn(fullFilePath); + if (!stat) { + continue; + } + + const excluded = shouldExcludeDeep(exclude, relativeFilePath); + if (!excluded) { + if (stat.isFile()) { + files.push(generateAssetFile(rootDir, fullFilePath, stat)); + } + } + + if (stat.isDirectory() && (!excluded || !shouldExcludeDirectory(exclude, relativeFilePath))) { + recurse(fullFilePath, stat); + } + } + } + + return files; +} + +export function listFilesRecursivelyOld( + dirOrFile: string, + options: CopyOptions & Required>, _rootDir?: string +): AssetFile[] { const files: AssetFile[] = []; let exclude = options.exclude || []; - const rootDir = _rootDir || dir; + const rootDir = _rootDir || dirOrFile; const followStatsFn = options.follow === FollowMode.ALWAYS ? fs.statSync : fs.lstatSync; - recurse(dir); + recurse(dirOrFile); function recurse(currentPath: string, currentStat?: fs.Stats): void { { @@ -171,8 +222,13 @@ export function listFilesRecursively(dir: string, options: CopyOptions & Require } if (!stat.isDirectory()) { - // FIXME not checking wether the file was ignored or not - files.push(generateAssetFile(rootDir, currentPath, stat)); + const relativePath = + path.relative(rootDir, currentPath) || + path.relative(path.dirname(rootDir), currentPath); + + if (!shouldExclude(exclude, relativePath)) { + files.push(generateAssetFile(rootDir, currentPath, stat)); + } return; } } diff --git a/packages/@aws-cdk/assets/test/fs/test.utils.ts b/packages/@aws-cdk/assets/test/fs/test.utils.ts index 2b829a95317cd..55c17e36432e8 100644 --- a/packages/@aws-cdk/assets/test/fs/test.utils.ts +++ b/packages/@aws-cdk/assets/test/fs/test.utils.ts @@ -1,3 +1,4 @@ +import { FsUtils } from '@aws-cdk/assert'; import fs = require('fs'); import { Test } from 'nodeunit'; import path = require('path'); @@ -194,6 +195,13 @@ export = { }, shouldExcludeDeep: { + 'without pattern'(test: Test) { + testShouldExcludeDeep(test, [], [], ['foo.txt']); + testShouldExcludeDeep(test, [''], [], ['foo.txt']); + testShouldExcludeDeep(test, ['# comment'], [], ['foo.txt']); + + test.done(); + }, 'basic usage'(test: Test) { testShouldExcludeDeep(test, ['foo.txt'], [ 'foo.txt', @@ -282,6 +290,13 @@ export = { }, shouldExcludeDirectory: { + 'without pattern'(test: Test) { + testShouldExcludeDirectory(test, [], [], ['dir']); + testShouldExcludeDirectory(test, [''], [], ['dir']); + testShouldExcludeDirectory(test, ['# comment'], [], ['dir']); + + test.done(); + }, 'basic usage'(test: Test) { const pattern = ['dir', '!dir/*', 'other_dir']; @@ -333,6 +348,62 @@ export = { test.done(); }, }, + + listFilesRecursively: { + 'basic usage'(test: Test) { + const exclude = ['']; + const follow = FollowMode.ALWAYS; + const tree = ` + ├── directory + │   ├── foo.txt + │   └── bar.txt + ├── deep + │   ├── dir + │   │   └── struct + │   │   └── qux.txt + ├── foobar.txt`; + + const { directory, cleanup } = FsUtils.fromTree('basic', tree); + const paths = util.listFilesRecursively(directory, { exclude, follow }).map(({ relativePath }) => relativePath); + + test.deepEqual(paths, [ + 'deep/dir/struct/qux.txt', + 'directory/bar.txt', + 'directory/foo.txt', + 'foobar.txt', + ]); + + cleanup(); + test.done(); + }, + 'exclude'(test: Test) { + const exclude = ['foobar.txt', 'deep', '!deep/foo.txt']; + const follow = FollowMode.ALWAYS; + const tree = ` + ├── directory + │   ├── foo.txt + │   └── bar.txt + ├── deep + │   ├── dir + │   │   └── struct + │   │   └── qux.txt + │   ├── foo.txt + │   └── bar.txt + ├── foobar.txt`; + + const { directory, cleanup } = FsUtils.fromTree('exclude', tree); + const paths = util.listFilesRecursively(directory, { exclude, follow }).map(({ relativePath }) => relativePath); + + test.deepEqual(paths, [ + 'deep/foo.txt', + 'directory/bar.txt', + 'directory/foo.txt', + ]); + + cleanup(); + test.done(); + }, + }, }; const testShouldExclude = ( From c07eb60f034b059cbe1bfc4a91763c6320e1563e Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Fri, 25 Oct 2019 14:41:42 +0200 Subject: [PATCH 25/35] feat: add symlink support to fromTree --- packages/@aws-cdk/assert/lib/fs-utils.ts | 12 ++++- .../@aws-cdk/assert/test/test.fs-utils.ts | 49 ++++++++++++++++++- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/assert/lib/fs-utils.ts b/packages/@aws-cdk/assert/lib/fs-utils.ts index fb98bf0b62a56..055141c0feec5 100644 --- a/packages/@aws-cdk/assert/lib/fs-utils.ts +++ b/packages/@aws-cdk/assert/lib/fs-utils.ts @@ -33,9 +33,10 @@ export class FsUtils { const directories: string[] = []; const files: string[] = []; + const symlinks: Array<[string, string]> = []; // we push an element at the end because we push the files/directories during the previous iteration - const lines = [...tree.replace(/^\n/, '').split('\n'), '']; + const lines = [...tree.replace(/^\n+/, '').trimRight().split('\n'), '']; const initialIndentLevel = (lines[0].match(/^\s*/) || [''])[0].length; lines.reduce<[string, number, boolean]>(([previousDir, previousIndentLevel, wasDirectory], line) => { @@ -51,6 +52,9 @@ export class FsUtils { if (previousDir) { if (indentLevel > previousIndentLevel || wasDirectory) { directories.push(previousDir); + } else if (previousDir.includes('->')) { + const [link, target] = previousDir.split(/\s*->\s*/); + symlinks.push([link, target]); } else { files.push(previousDir); } @@ -65,10 +69,16 @@ export class FsUtils { for (const file of files) { fs.writeFileSync(path.join(directory, file), content); } + for (const [link, target] of symlinks) { + fs.symlinkSync(target, path.join(directory, link)); + } return { directory, cleanup: () => { + for (const [link] of symlinks) { + fs.unlinkSync(path.join(directory, link)); + } for (const file of files) { fs.unlinkSync(path.join(directory, file)); } diff --git a/packages/@aws-cdk/assert/test/test.fs-utils.ts b/packages/@aws-cdk/assert/test/test.fs-utils.ts index b286fed340d7a..d51874c5448a7 100644 --- a/packages/@aws-cdk/assert/test/test.fs-utils.ts +++ b/packages/@aws-cdk/assert/test/test.fs-utils.ts @@ -27,16 +27,61 @@ export = { test.done(); }, + 'symlinks'(test: Test) { + // GIVEN + const tree = ` + ├── link -> target + ├── target + └── foo.txt`; + + // THEN + const {directory, cleanup} = FsUtils.fromTree('empty-directory', tree); + + test.ok(fs.existsSync(path.join(directory, 'target', 'foo.txt'))); + test.ok(fs.existsSync(path.join(directory, 'link', 'foo.txt'))); + test.equal(fs.readlinkSync(path.join(directory, 'link')), 'target'); + + cleanup(); + + test.ok(!fs.existsSync(path.join(directory, 'target'))); + test.ok(!fs.existsSync(path.join(directory, 'link'))); + test.ok(!fs.existsSync(directory)); + + test.done(); + }, + 'empty directory'(test: Test) { + // GIVEN + const tree = ` + ├── dir (D)`; + + // THEN + const {directory, cleanup} = FsUtils.fromTree('empty-directory', tree); + + test.ok(fs.existsSync(path.join(directory, 'dir'))); + + cleanup(); + + test.ok(!fs.existsSync(path.join(directory, 'dir'))); + test.ok(!fs.existsSync(directory)); + + test.done(); + }, 'works with any indent'(test: Test) { // GIVEN const tree = ` + + + ├── foo └── dir └── subdir - └── bar.txt`; + └── bar.txt + + + `; // THEN - const {directory, cleanup} = FsUtils.fromTree('basic-usage', tree); + const {directory, cleanup} = FsUtils.fromTree('any-indent', tree); test.ok(fs.existsSync(path.join(directory, 'foo'))); test.ok(fs.existsSync(path.join(directory, 'dir', 'subdir', 'bar.txt'))); From 5de1e6d57a06e097bf3975e6e87d915c52f88615 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Fri, 25 Oct 2019 15:14:33 +0200 Subject: [PATCH 26/35] fix: fromTree external directory symlink --- packages/@aws-cdk/assert/lib/fs-utils.ts | 2 +- .../@aws-cdk/assert/test/test.fs-utils.ts | 39 ++++++++++++++++--- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/packages/@aws-cdk/assert/lib/fs-utils.ts b/packages/@aws-cdk/assert/lib/fs-utils.ts index 055141c0feec5..dec82ffac97e2 100644 --- a/packages/@aws-cdk/assert/lib/fs-utils.ts +++ b/packages/@aws-cdk/assert/lib/fs-utils.ts @@ -92,7 +92,7 @@ export class FsUtils { } private static readonly INDENT_CHARACTERS_REGEX = /^[\s├─│└]+/; - private static readonly TRAILING_CHARACTERS_REGEX = /\/|\(D\)$/i; + private static readonly TRAILING_CHARACTERS_REGEX = /(\/|\(D\))$/i; private static readonly IS_DIRECTORY_REGEX = /\(D\)\s*$/i; } \ No newline at end of file diff --git a/packages/@aws-cdk/assert/test/test.fs-utils.ts b/packages/@aws-cdk/assert/test/test.fs-utils.ts index d51874c5448a7..b621dcc0b7b73 100644 --- a/packages/@aws-cdk/assert/test/test.fs-utils.ts +++ b/packages/@aws-cdk/assert/test/test.fs-utils.ts @@ -1,5 +1,6 @@ import fs = require('fs'); import { Test } from 'nodeunit'; +import os = require('os'); import path = require('path'); import { FsUtils } from '../lib'; @@ -10,11 +11,11 @@ export = { const tree = ` ├── foo └── dir - └── subdir + └── subdir/ └── bar.txt`; // THEN - const {directory, cleanup} = FsUtils.fromTree('basic-usage', tree); + const { directory, cleanup } = FsUtils.fromTree('basic-usage', tree); test.ok(fs.existsSync(path.join(directory, 'foo'))); test.ok(fs.existsSync(path.join(directory, 'dir', 'subdir', 'bar.txt'))); @@ -35,7 +36,7 @@ export = { └── foo.txt`; // THEN - const {directory, cleanup} = FsUtils.fromTree('empty-directory', tree); + const { directory, cleanup } = FsUtils.fromTree('symlink', tree); test.ok(fs.existsSync(path.join(directory, 'target', 'foo.txt'))); test.ok(fs.existsSync(path.join(directory, 'link', 'foo.txt'))); @@ -49,13 +50,41 @@ export = { test.done(); }, + 'external smylinks'(test: Test) { + // GIVEN + const externalTree = FsUtils.fromTree('external', ` + ├── external_dir + │   ├── foobar.txt`); + + // THEN + + const externalRelativeDirectory = path.relative(os.tmpdir(), externalTree.directory); + const externalLink = `../${externalRelativeDirectory}/external_dir`; + + const internalTree = FsUtils.fromTree('internal', ` + ├── external_link -> ${externalLink}`); + + test.ok(fs.existsSync(path.join(externalTree.directory, 'external_dir', 'foobar.txt'))); + test.ok(fs.existsSync(path.join(internalTree.directory, 'external_link', 'foobar.txt'))); + test.equal(fs.readlinkSync(path.join(internalTree.directory, 'external_link')), externalLink); + + externalTree.cleanup(); + internalTree.cleanup(); + + test.ok(!fs.existsSync(path.join(externalTree.directory, 'external_dir'))); + test.ok(!fs.existsSync(path.join(internalTree.directory, 'external_link'))); + test.ok(!fs.existsSync(internalTree.directory)); + test.ok(!fs.existsSync(externalTree.directory)); + + test.done(); + }, 'empty directory'(test: Test) { // GIVEN const tree = ` ├── dir (D)`; // THEN - const {directory, cleanup} = FsUtils.fromTree('empty-directory', tree); + const { directory, cleanup } = FsUtils.fromTree('empty-directory', tree); test.ok(fs.existsSync(path.join(directory, 'dir'))); @@ -81,7 +110,7 @@ export = { `; // THEN - const {directory, cleanup} = FsUtils.fromTree('any-indent', tree); + const { directory, cleanup } = FsUtils.fromTree('any-indent', tree); test.ok(fs.existsSync(path.join(directory, 'foo'))); test.ok(fs.existsSync(path.join(directory, 'dir', 'subdir', 'bar.txt'))); From 1e9476de4f687a40abc4d4c0bfc8dc850166a6b6 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Fri, 25 Oct 2019 15:52:24 +0200 Subject: [PATCH 27/35] fix: listFiles symlink support --- packages/@aws-cdk/assets/lib/fs/utils.ts | 111 +++--------------- .../@aws-cdk/assets/test/fs/test.utils.ts | 95 +++++++++++++++ 2 files changed, 114 insertions(+), 92 deletions(-) diff --git a/packages/@aws-cdk/assets/lib/fs/utils.ts b/packages/@aws-cdk/assets/lib/fs/utils.ts index f04e8e6007e3f..5ffb774a7805f 100644 --- a/packages/@aws-cdk/assets/lib/fs/utils.ts +++ b/packages/@aws-cdk/assets/lib/fs/utils.ts @@ -187,103 +187,30 @@ export function listFilesRecursively( continue; } - const excluded = shouldExcludeDeep(exclude, relativeFilePath); - if (!excluded) { - if (stat.isFile()) { - files.push(generateAssetFile(rootDir, fullFilePath, stat)); - } - } - - if (stat.isDirectory() && (!excluded || !shouldExcludeDirectory(exclude, relativeFilePath))) { - recurse(fullFilePath, stat); - } - } - } - - return files; -} - -export function listFilesRecursivelyOld( - dirOrFile: string, - options: CopyOptions & Required>, _rootDir?: string -): AssetFile[] { - const files: AssetFile[] = []; - let exclude = options.exclude || []; - const rootDir = _rootDir || dirOrFile; - const followStatsFn = options.follow === FollowMode.ALWAYS ? fs.statSync : fs.lstatSync; - - recurse(dirOrFile); - - function recurse(currentPath: string, currentStat?: fs.Stats): void { - { - const stat = currentStat || fs.statSync(currentPath); - if (!stat) { - return; - } - - if (!stat.isDirectory()) { - const relativePath = - path.relative(rootDir, currentPath) || - path.relative(path.dirname(rootDir), currentPath); - - if (!shouldExclude(exclude, relativePath)) { - files.push(generateAssetFile(rootDir, currentPath, stat)); - } - return; - } - } - - for (const file of fs.readdirSync(currentPath)) { - const fullFilePath = path.join(currentPath, file); - - let stat: fs.Stats | undefined = followStatsFn(fullFilePath); - if (!stat) { - continue; - } - - let target = ''; - if (stat.isSymbolicLink()) { - target = fs.readlinkSync(fullFilePath); - - // determine if this is an external link (i.e. the target's absolute path - // is outside of the root directory). - const targetPath = path.normalize(path.resolve(currentPath, target)); - - if (shouldFollow(options.follow, rootDir, targetPath)) { - stat = fs.statSync(fullFilePath); - if (!stat) { - continue; + const isExcluded = shouldExcludeDeep(exclude, relativeFilePath); + if (!isExcluded) { + let target = ''; + if (stat.isSymbolicLink()) { + target = fs.readlinkSync(fullFilePath); + + // determine if this is an external link (i.e. the target's absolute path is outside of the root directory). + const targetPath = path.normalize(path.resolve(currentPath, target)); + if (shouldFollow(options.follow, rootDir, targetPath)) { + stat = fs.statSync(fullFilePath); + if (!stat) { + continue; + } } } - } - - const relativeFilePath = path.relative(rootDir, fullFilePath); - - // we've just discovered that we have a directory - if (stat.isDirectory()) { - // to help future shouldExclude calls, we're changing the exlusion patterns - // by expliciting "dir" exclusions to "dir/*" (same with "!dir" -> "!dir/*") - exclude = exclude.reduce((res, pattern) => { - res.push(pattern); - if (pattern.trim().replace(/^!/, '') === relativeFilePath) { - // we add the pattern immediately after to preserve the exclusion order - res.push(`${pattern}/*`); - } - return res; - }, []); - } - - const isExcluded = shouldExclude(exclude, relativeFilePath); - if (isExcluded && !stat.isDirectory()) { - continue; + if (stat.isFile()) { + files.push(generateAssetFile(rootDir, fullFilePath, stat)); + } else if (stat.isSymbolicLink()) { + files.push(generateAssetSymlinkFile(rootDir, fullFilePath, stat, target)); + } } - if (stat.isFile()) { - files.push(generateAssetFile(rootDir, fullFilePath, stat)); - } else if (stat.isSymbolicLink()) { - files.push(generateAssetSymlinkFile(rootDir, fullFilePath, stat, target)); - } else if (stat.isDirectory()) { + if (stat.isDirectory() && (!isExcluded || !shouldExcludeDirectory(exclude, relativeFilePath))) { const previousLength = files.length; recurse(fullFilePath, stat); diff --git a/packages/@aws-cdk/assets/test/fs/test.utils.ts b/packages/@aws-cdk/assets/test/fs/test.utils.ts index 55c17e36432e8..210bbe20b6cbe 100644 --- a/packages/@aws-cdk/assets/test/fs/test.utils.ts +++ b/packages/@aws-cdk/assets/test/fs/test.utils.ts @@ -1,6 +1,7 @@ import { FsUtils } from '@aws-cdk/assert'; import fs = require('fs'); import { Test } from 'nodeunit'; +import os = require('os'); import path = require('path'); import { ImportMock } from 'ts-mock-imports'; import { FollowMode } from '../../lib/fs'; @@ -357,6 +358,7 @@ export = { ├── directory │   ├── foo.txt │   └── bar.txt + ├── empty-dir (D) ├── deep │   ├── dir │   │   └── struct @@ -370,6 +372,7 @@ export = { 'deep/dir/struct/qux.txt', 'directory/bar.txt', 'directory/foo.txt', + 'empty-dir', 'foobar.txt', ]); @@ -403,6 +406,98 @@ export = { cleanup(); test.done(); }, + 'symlinks': { + 'without exclusion'(test: Test) { + const exclude = ['']; + const externalTree = FsUtils.fromTree('exclude', ` + ├── external_dir + │   ├── foobar.txt`); + + const internalTree = FsUtils.fromTree('exclude', ` + ├── directory + │   ├── foo.txt + ├── internal_link -> directory + ├── external_link -> ../${path.relative(os.tmpdir(), externalTree.directory)}/external_dir`); + + const expected: { [followMode in FollowMode]: string[] } = { + [FollowMode.NEVER]: [ + 'directory/foo.txt', + 'external_link', + 'internal_link', + ], + [FollowMode.ALWAYS]: [ + 'directory/foo.txt', + 'external_link/foobar.txt', + 'internal_link/foo.txt', + ], + [FollowMode.EXTERNAL]: [ + 'directory/foo.txt', + 'external_link/foobar.txt', + 'internal_link' + ], + [FollowMode.BLOCK_EXTERNAL]: [ + 'directory/foo.txt', + 'external_link', + 'internal_link/foo.txt', + ], + }; + + for (const follow of Object.values(FollowMode)) { + const paths = util.listFilesRecursively(internalTree.directory, { exclude, follow }).map(({ relativePath }) => relativePath); + + test.deepEqual(paths, expected[follow], follow); + } + + internalTree.cleanup(); + externalTree.cleanup(); + test.done(); + }, + }, + 'exclude targets'(test: Test) { + const exclude = ['external_dir', 'directory']; + const follow = FollowMode.ALWAYS; + const externalTree = FsUtils.fromTree('exclude', ` + ├── external_dir + │   ├── foobar.txt`); + + const internalTree = FsUtils.fromTree('exclude', ` + ├── directory + │   ├── foo.txt + ├── internal_link -> directory + ├── external_link -> ../${path.relative(os.tmpdir(), externalTree.directory)}/external_dir`); + const paths = util.listFilesRecursively(internalTree.directory, { exclude, follow }).map(({ relativePath }) => relativePath); + + test.deepEqual(paths, [ + 'external_link/foobar.txt', + 'internal_link/foo.txt' + ]); + + internalTree.cleanup(); + externalTree.cleanup(); + test.done(); + }, + 'exclude links'(test: Test) { + const exclude = ['internal_link', 'external_link']; + const follow = FollowMode.ALWAYS; + const externalTree = FsUtils.fromTree('exclude', ` + ├── external_dir + │   ├── foobar.txt`); + + const internalTree = FsUtils.fromTree('exclude', ` + ├── directory + │   ├── foo.txt + ├── internal_link -> directory + ├── external_link -> ../${path.relative(os.tmpdir(), externalTree.directory)}/external_dir`); + const paths = util.listFilesRecursively(internalTree.directory, { exclude, follow }).map(({ relativePath }) => relativePath); + + test.deepEqual(paths, [ + 'directory/foo.txt' + ]); + + internalTree.cleanup(); + externalTree.cleanup(); + test.done(); + }, }, }; From b49f64f17497bc04eb8db40a7fea5cab54ddc1f6 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Tue, 29 Oct 2019 11:59:14 +0100 Subject: [PATCH 28/35] chore: additional contridactory test --- packages/@aws-cdk/assets/test/fs/test.utils.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/assets/test/fs/test.utils.ts b/packages/@aws-cdk/assets/test/fs/test.utils.ts index 210bbe20b6cbe..a0d6e960ac6c1 100644 --- a/packages/@aws-cdk/assets/test/fs/test.utils.ts +++ b/packages/@aws-cdk/assets/test/fs/test.utils.ts @@ -216,11 +216,16 @@ export = { test.done(); }, - 'contridactory'(test: Test) { + 'negative contridactory'(test: Test) { testShouldExcludeDeep(test, ['foo.txt', '!foo.txt'], [], ['foo.txt']); test.done(); }, + 'positive contridactory'(test: Test) { + testShouldExcludeDeep(test, ['!foo.txt', 'foo.txt'], ['foo.txt'], []); + + test.done(); + }, 'dir single wildcard'(test: Test) { testShouldExcludeDeep(test, ['d?r'], [ 'dir', From 44f77576894f22f700aa5c8931f5cc7d746776ae Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Tue, 29 Oct 2019 12:00:03 +0100 Subject: [PATCH 29/35] chore: fix docs --- packages/@aws-cdk/assets/lib/fs/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/assets/lib/fs/utils.ts b/packages/@aws-cdk/assets/lib/fs/utils.ts index 5ffb774a7805f..5affbf6f6615d 100644 --- a/packages/@aws-cdk/assets/lib/fs/utils.ts +++ b/packages/@aws-cdk/assets/lib/fs/utils.ts @@ -68,7 +68,7 @@ export function shouldExcludeDeep(exclude: string[], relativePath: string): bool /** * Determines whether a given directory should be excluded and not explored further - * This might be true even if the directory is explicitly excluded, + * This might be `false` even if the directory is explicitly excluded, * but one of its children might be inclunded * * @param exclude exclusion patterns From ab92e3f3cf9b46d45cea32e0fc98366d7786ff92 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Tue, 29 Oct 2019 13:32:12 +0100 Subject: [PATCH 30/35] chore: ExcludeRules class refactor --- packages/@aws-cdk/assets/lib/fs/utils.ts | 167 ++++++++++-------- .../@aws-cdk/assets/test/fs/test.utils.ts | 10 +- 2 files changed, 98 insertions(+), 79 deletions(-) diff --git a/packages/@aws-cdk/assets/lib/fs/utils.ts b/packages/@aws-cdk/assets/lib/fs/utils.ts index 5affbf6f6615d..3ea2cdf803e1a 100644 --- a/packages/@aws-cdk/assets/lib/fs/utils.ts +++ b/packages/@aws-cdk/assets/lib/fs/utils.ts @@ -12,102 +12,113 @@ import { FollowMode } from './follow-mode'; * @param filePath file path to be assessed against the pattern * * @returns `true` if the file should be excluded + * + * @deprecated see {@link ExcludeRules.excludeFile} */ export function shouldExclude(exclude: string[], filePath: string): boolean { - const [_shouldExclude] = shouldExcludePriority(exclude, filePath); + const [_shouldExclude] = ExcludeRules.evaluateFile(exclude, filePath); return _shouldExclude; } /** - * Determines whether a given file should be excluded or not based on given - * exclusion glob patterns. - * - * @param exclude exclusion patterns - * @param filePath file patg to be assessed against the pattern - * - * @returns `true` if the file should be excluded, followed by the index of the rule applied + * Set of exclusion evaluation methods */ -export function shouldExcludePriority(exclude: string[], filePath: string): [boolean, number] { - return exclude.reduce<[boolean, number]>((res, pattern, patternIndex) => { - const negate = pattern.startsWith('!'); - const match = minimatch(filePath, pattern, { matchBase: true, flipNegate: true }); +export class ExcludeRules { + /** + * Determines whether a given file should be excluded or not based on given + * exclusion glob patterns. + * + * @param exclude exclusion patterns + * @param filePath file path to be assessed against the pattern + * + * @returns `true` if the file should be excluded, followed by the index of the rule applied + */ + public static evaluateFile(exclude: string[], filePath: string): [boolean, number] { + return exclude.reduce<[boolean, number]>((res, pattern, patternIndex) => { + const negate = pattern.startsWith('!'); + const match = minimatch(filePath, pattern, { matchBase: true, flipNegate: true }); + + if (!negate && match) { + res = [true, patternIndex]; + } - if (!negate && match) { - res = [true, patternIndex]; - } + if (negate && match) { + res = [false, patternIndex]; + } - if (negate && match) { - res = [false, patternIndex]; - } + return res; + }, [false, -1]); + } - return res; - }, [false, -1]); -} + private static getComponents = (value: string): string[] => value.split(path.sep); -/** - * Determines whether a given file should be excluded,taking into account deep file structures - * - * @param exclude exclusion patterns - * @param filePath file path to be assessed against the pattern - */ -export function shouldExcludeDeep(exclude: string[], relativePath: string): boolean { - const [_shouldExclude] = relativePath.split(path.sep).reduce<[boolean, number, string]>( - ([accExclude, accPriority, pathIterator], pathComponent) => { - pathIterator = path.join(pathIterator, pathComponent); - - const [shouldExcludeIt, priorityIt] = shouldExcludePriority(exclude, pathIterator); - if (priorityIt > accPriority) { - return [shouldExcludeIt, priorityIt, pathIterator]; - } + private readonly patternComponents: string[][] = this.patterns.map(ExcludeRules.getComponents); + public constructor(private readonly patterns: string[]) { + } - return [accExclude, accPriority, pathIterator]; - }, [false, -1, '']); + /** + * Determines whether a given file should be excluded,taking into account deep file structures + * + * @param filePath file path to be assessed against the pattern + */ + public excludeFile(relativePath: string): boolean { + const [_shouldExclude] = ExcludeRules.getComponents(relativePath).reduce<[boolean, number, string]>( + ([accExclude, accPriority, pathIterator], pathComponent) => { + pathIterator = path.join(pathIterator, pathComponent); + + const [shouldExcludeIt, priorityIt] = ExcludeRules.evaluateFile(this.patterns, pathIterator); + if (priorityIt > accPriority) { + return [shouldExcludeIt, priorityIt, pathIterator]; + } - return _shouldExclude; -} + return [accExclude, accPriority, pathIterator]; + }, [false, -1, '']); -/** - * Determines whether a given directory should be excluded and not explored further - * This might be `false` even if the directory is explicitly excluded, - * but one of its children might be inclunded - * - * @param exclude exclusion patterns - * @param directoryPath directory path to be assessed against the pattern - */ -export function shouldExcludeDirectory(exclude: string[], directoryPath: string): boolean { - const splitPatterns = exclude.map((exc) => exc.split(path.sep)); - const patternLength = splitPatterns.map(({ length }) => length); - const maxPatternLength = Math.max(...patternLength); - - const [_shouldExclude] = directoryPath.split(path.sep).reduce<[boolean | null, string]>( - ([accExclude, pathIterator], pathComponent) => { - pathIterator = path.join(pathIterator, pathComponent); - - for (let pattenItLength = 1; pattenItLength <= maxPatternLength; ++pattenItLength) { - const excludeSliced = splitPatterns.map((pattern) => pattern.slice(0, pattenItLength).join(path.sep)); - const [shouldExcludeIt, patternIndex] = shouldExcludePriority(excludeSliced, pathIterator); - if (patternIndex < 0) { - continue; - } + return _shouldExclude; + } - if (shouldExcludeIt) { - if (accExclude == null) { + /** + * Determines whether a given directory should be excluded and not explored further + * This might be `true` even if the directory is explicitly excluded, + * but one of its children might be inclunded + * + * @param directoryPath directory path to be assessed against the pattern + */ + public excludeDirectory(directoryPath: string): boolean { + const patternLength = this.patternComponents.map(({ length }) => length); + const maxPatternLength = Math.max(...patternLength); + + const [_shouldExclude] = directoryPath.split(path.sep).reduce<[boolean | null, string]>( + ([accExclude, pathIterator], pathComponent) => { + pathIterator = path.join(pathIterator, pathComponent); + + for (let pattenItLength = 1; pattenItLength <= maxPatternLength; ++pattenItLength) { + const excludeSliced = this.patternComponents.map((pattern) => pattern.slice(0, pattenItLength).join(path.sep)); + const [shouldExcludeIt, patternIndex] = ExcludeRules.evaluateFile(excludeSliced, pathIterator); + if (patternIndex < 0) { + continue; + } + + if (shouldExcludeIt) { + if (accExclude == null) { + accExclude = true; + } + continue; + } + + if (pattenItLength < patternLength[patternIndex]) { + accExclude = shouldExcludeIt; + } else if (!excludeSliced[patternIndex].includes('**')) { accExclude = true; } - continue; } - if (pattenItLength < patternLength[patternIndex]) { - accExclude = shouldExcludeIt; - } else if (!excludeSliced[patternIndex].includes('**')) { - accExclude = true; - } - } + return [accExclude, pathIterator]; + }, [null, '']); - return [accExclude, pathIterator]; - }, [null, '']); + return _shouldExclude || false; + } - return _shouldExclude || false; } /** @@ -170,6 +181,8 @@ export function listFilesRecursively( const rootDir = _rootDir || dirOrFile; const followStatsFn = options.follow === FollowMode.ALWAYS ? fs.statSync : fs.lstatSync; + const excludeRules = new ExcludeRules(exclude); + recurse(dirOrFile); function recurse(currentPath: string, _currentStat?: fs.Stats): void { @@ -187,7 +200,7 @@ export function listFilesRecursively( continue; } - const isExcluded = shouldExcludeDeep(exclude, relativeFilePath); + const isExcluded = excludeRules.excludeFile(relativeFilePath); if (!isExcluded) { let target = ''; if (stat.isSymbolicLink()) { @@ -210,7 +223,7 @@ export function listFilesRecursively( } } - if (stat.isDirectory() && (!isExcluded || !shouldExcludeDirectory(exclude, relativeFilePath))) { + if (stat.isDirectory() && (!isExcluded || !excludeRules.excludeDirectory(relativeFilePath))) { const previousLength = files.length; recurse(fullFilePath, stat); diff --git a/packages/@aws-cdk/assets/test/fs/test.utils.ts b/packages/@aws-cdk/assets/test/fs/test.utils.ts index a0d6e960ac6c1..8f93d8a2c77eb 100644 --- a/packages/@aws-cdk/assets/test/fs/test.utils.ts +++ b/packages/@aws-cdk/assets/test/fs/test.utils.ts @@ -521,7 +521,13 @@ const testShouldExclude = ( }; const testShouldExcludeDeep = (test: Test, pattern: string[], expectExclude: string[], expectInclude: string[]) => - testShouldExclude(test, pattern, expectExclude, expectInclude, util.shouldExcludeDeep); + testShouldExclude(test, pattern, expectExclude, expectInclude, (patterns: string[], _path: string) => { + const excludeRules = new util.ExcludeRules(patterns); + return excludeRules.excludeFile(_path); + }); const testShouldExcludeDirectory = (test: Test, pattern: string[], expectExclude: string[], expectInclude: string[]) => - testShouldExclude(test, pattern, expectExclude, expectInclude, util.shouldExcludeDirectory); \ No newline at end of file + testShouldExclude(test, pattern, expectExclude, expectInclude, (patterns: string[], _path: string) => { + const excludeRules = new util.ExcludeRules(patterns); + return excludeRules.excludeDirectory(_path); + }); \ No newline at end of file From af2098b77c0428c46b8311a2169e9477c8adb7b0 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Tue, 29 Oct 2019 13:38:55 +0100 Subject: [PATCH 31/35] chore: evaluateFile refactor --- packages/@aws-cdk/assets/lib/fs/utils.ts | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/packages/@aws-cdk/assets/lib/fs/utils.ts b/packages/@aws-cdk/assets/lib/fs/utils.ts index 3ea2cdf803e1a..281695470cab6 100644 --- a/packages/@aws-cdk/assets/lib/fs/utils.ts +++ b/packages/@aws-cdk/assets/lib/fs/utils.ts @@ -28,26 +28,22 @@ export class ExcludeRules { * Determines whether a given file should be excluded or not based on given * exclusion glob patterns. * - * @param exclude exclusion patterns + * @param patterns exclusion patterns * @param filePath file path to be assessed against the pattern * * @returns `true` if the file should be excluded, followed by the index of the rule applied */ - public static evaluateFile(exclude: string[], filePath: string): [boolean, number] { - return exclude.reduce<[boolean, number]>((res, pattern, patternIndex) => { - const negate = pattern.startsWith('!'); - const match = minimatch(filePath, pattern, { matchBase: true, flipNegate: true }); + public static evaluateFile(patterns: string[], filePath: string): [boolean, number] { + let _shouldExclude = false; + let exclusionIndex = -1; - if (!negate && match) { - res = [true, patternIndex]; + patterns.map((pattern, patternIndex) => { + if (minimatch(filePath, pattern, { matchBase: true, flipNegate: true })) { + [_shouldExclude, exclusionIndex] = [!pattern.startsWith('!'), patternIndex]; } + }); - if (negate && match) { - res = [false, patternIndex]; - } - - return res; - }, [false, -1]); + return [_shouldExclude, exclusionIndex]; } private static getComponents = (value: string): string[] => value.split(path.sep); From 9cf39b0beb6b6a7c7ad904f0b8e58e2a1b3bd6fd Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Tue, 29 Oct 2019 13:57:41 +0100 Subject: [PATCH 32/35] chore: further evaluateFile refactor --- packages/@aws-cdk/assets/lib/fs/utils.ts | 54 ++++++++++++++++++------ 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/packages/@aws-cdk/assets/lib/fs/utils.ts b/packages/@aws-cdk/assets/lib/fs/utils.ts index 281695470cab6..350aa854572d7 100644 --- a/packages/@aws-cdk/assets/lib/fs/utils.ts +++ b/packages/@aws-cdk/assets/lib/fs/utils.ts @@ -46,31 +46,57 @@ export class ExcludeRules { return [_shouldExclude, exclusionIndex]; } - private static getComponents = (value: string): string[] => value.split(path.sep); + /** + * Splits a file or directory path in an array of elements + * containing each path component (directories and file) + * + * @param filePath the path to split + * @returns an array containing each path component + * + * @example ExcludeRules.getPathComponents('a/b/c') = ['a', 'b', 'c'] + */ + private static getPathComponents = (filePath: string): string[] => filePath.split(path.sep); - private readonly patternComponents: string[][] = this.patterns.map(ExcludeRules.getComponents); - public constructor(private readonly patterns: string[]) { + /** + * Splits a file or directory path in an array of elements + * containing each partial path up to that point + * + * @param filePath the path to split + * @returns an array containing each path component + * + * @example ExcludeRules.getAccumulatedPathComponents('a/b/c') = ['a', 'a/b', 'a/b/c'] + */ + private static getAccumulatedPathComponents(filePath: string): string[] { + const accComponents: string[] = []; + for (const component of ExcludeRules.getPathComponents(filePath)) { + accComponents.push(accComponents.length ? + [accComponents[accComponents.length - 1], component].join(path.sep) : + component + ); + } + return accComponents; } + private readonly patternComponents: string[][] = this.patterns.map(ExcludeRules.getPathComponents); + public constructor(private readonly patterns: string[]) { } + /** * Determines whether a given file should be excluded,taking into account deep file structures * * @param filePath file path to be assessed against the pattern */ public excludeFile(relativePath: string): boolean { - const [_shouldExclude] = ExcludeRules.getComponents(relativePath).reduce<[boolean, number, string]>( - ([accExclude, accPriority, pathIterator], pathComponent) => { - pathIterator = path.join(pathIterator, pathComponent); + let accExclude = false; + let accPriority = -1; - const [shouldExcludeIt, priorityIt] = ExcludeRules.evaluateFile(this.patterns, pathIterator); - if (priorityIt > accPriority) { - return [shouldExcludeIt, priorityIt, pathIterator]; - } - - return [accExclude, accPriority, pathIterator]; - }, [false, -1, '']); + for (const pathComponent of ExcludeRules.getAccumulatedPathComponents(relativePath)) { + const [shouldExcludeIt, priorityIt] = ExcludeRules.evaluateFile(this.patterns, pathComponent); + if (priorityIt > accPriority) { + [accExclude, accPriority] = [shouldExcludeIt, priorityIt]; + } + } - return _shouldExclude; + return accExclude; } /** From 02a8d0efc4b73dc3e841727b1145e18187b7ad88 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Tue, 29 Oct 2019 14:17:15 +0100 Subject: [PATCH 33/35] chore: evaluateDirectory refactor --- packages/@aws-cdk/assets/lib/fs/utils.ts | 58 ++++++++++++------------ 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/packages/@aws-cdk/assets/lib/fs/utils.ts b/packages/@aws-cdk/assets/lib/fs/utils.ts index 350aa854572d7..1bff5c4729e33 100644 --- a/packages/@aws-cdk/assets/lib/fs/utils.ts +++ b/packages/@aws-cdk/assets/lib/fs/utils.ts @@ -78,6 +78,18 @@ export class ExcludeRules { } private readonly patternComponents: string[][] = this.patterns.map(ExcludeRules.getPathComponents); + private get accumulatedPatternComponents(): string[][] { + const patternComponentsLength = this.patternComponents.map(({ length }) => length); + const maxPatternLength = Math.max(...patternComponentsLength); + + const accPatternComponents: string[][] = []; + for (let pattenComponentsLength = 1; pattenComponentsLength <= maxPatternLength; ++pattenComponentsLength) { + accPatternComponents.push(this.patternComponents.map((pattern) => pattern.slice(0, pattenComponentsLength).join(path.sep))); + } + + return accPatternComponents; + } + public constructor(private readonly patterns: string[]) { } /** @@ -89,8 +101,8 @@ export class ExcludeRules { let accExclude = false; let accPriority = -1; - for (const pathComponent of ExcludeRules.getAccumulatedPathComponents(relativePath)) { - const [shouldExcludeIt, priorityIt] = ExcludeRules.evaluateFile(this.patterns, pathComponent); + for (const accPath of ExcludeRules.getAccumulatedPathComponents(relativePath)) { + const [shouldExcludeIt, priorityIt] = ExcludeRules.evaluateFile(this.patterns, accPath); if (priorityIt > accPriority) { [accExclude, accPriority] = [shouldExcludeIt, priorityIt]; } @@ -107,36 +119,26 @@ export class ExcludeRules { * @param directoryPath directory path to be assessed against the pattern */ public excludeDirectory(directoryPath: string): boolean { - const patternLength = this.patternComponents.map(({ length }) => length); - const maxPatternLength = Math.max(...patternLength); - - const [_shouldExclude] = directoryPath.split(path.sep).reduce<[boolean | null, string]>( - ([accExclude, pathIterator], pathComponent) => { - pathIterator = path.join(pathIterator, pathComponent); - - for (let pattenItLength = 1; pattenItLength <= maxPatternLength; ++pattenItLength) { - const excludeSliced = this.patternComponents.map((pattern) => pattern.slice(0, pattenItLength).join(path.sep)); - const [shouldExcludeIt, patternIndex] = ExcludeRules.evaluateFile(excludeSliced, pathIterator); - if (patternIndex < 0) { - continue; - } + let _shouldExclude: boolean | null = null; - if (shouldExcludeIt) { - if (accExclude == null) { - accExclude = true; - } - continue; - } + for (const accPath of ExcludeRules.getAccumulatedPathComponents(directoryPath)) { + this.accumulatedPatternComponents.map((accumulatedPatterns, accumulatedIndex) => { + const [shouldExcludeIt, patternIndex] = ExcludeRules.evaluateFile(accumulatedPatterns, accPath); + if (patternIndex < 0) { + return; + } - if (pattenItLength < patternLength[patternIndex]) { - accExclude = shouldExcludeIt; - } else if (!excludeSliced[patternIndex].includes('**')) { - accExclude = true; + if (shouldExcludeIt) { + if (_shouldExclude === null) { + _shouldExclude = true; } + } else if (accumulatedIndex < this.patternComponents[patternIndex].length - 1) { + _shouldExclude = shouldExcludeIt; + } else if (!accumulatedPatterns[patternIndex].includes('**')) { + _shouldExclude = true; } - - return [accExclude, pathIterator]; - }, [null, '']); + }); + } return _shouldExclude || false; } From 23331a3845c2bba0191f28044ea1f2aaea345371 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Tue, 29 Oct 2019 14:43:08 +0100 Subject: [PATCH 34/35] chore: move FsUtils to assets --- packages/@aws-cdk/assets/test/fs/fs-utils.ts | 119 ++++++++++++++++ .../@aws-cdk/assets/test/fs/test.fs-utils.ts | 131 ++++++++++++++++++ .../@aws-cdk/assets/test/fs/test.utils.ts | 2 +- 3 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 packages/@aws-cdk/assets/test/fs/fs-utils.ts create mode 100644 packages/@aws-cdk/assets/test/fs/test.fs-utils.ts diff --git a/packages/@aws-cdk/assets/test/fs/fs-utils.ts b/packages/@aws-cdk/assets/test/fs/fs-utils.ts new file mode 100644 index 0000000000000..222af8a5e22ba --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fs-utils.ts @@ -0,0 +1,119 @@ +import fs = require('fs'); +import os = require('os'); +import path = require('path'); + +interface FromTreeOutput { + /** + * Absolute path of the created temporary directory, containing the generated structure + */ + readonly directory: string; + /** + * Cleanup function that will remove the generated files once called + */ + readonly cleanup: () => void; +} + +/** + * Collection of file-system utility methods + */ +export class FsUtils { + /** + * Generates a filesystem structure from a string, + * formatted like the output of a `tree` shell command + * + * @param tmpPrefix temp directory prefix, used by {@link fs.mkdtemp} + * @param tree + * @param content the content + * + * @returns an array containing the absolute path of the created directory, + * and a cleanup function that will remove the generated files on invocation + */ + public static fromTree(tmpPrefix: string, tree: string, content = 'content'): FromTreeOutput { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), tmpPrefix)); + + const directories: string[] = []; + const files: string[] = []; + const symlinks: Array<[string, string]> = []; + + // we push an element at the end because we push the files/directories during the previous iteration + const lines = [...tree.replace(/^\n+/, '').trimRight().split('\n'), '']; + const initialIndentLevel = (lines[0].match(/^\s*/) || [''])[0].length; + + lines.reduce<[string, number, boolean]>(([previousDir, previousIndentLevel, wasDirectory], line) => { + const indentCharacters = (line.match(FsUtils.INDENT_CHARACTERS_REGEX) || [''])[0]; + const indentLevel = (indentCharacters.length - initialIndentLevel) / 4; + + const fileName = line.slice(indentCharacters.length).replace(FsUtils.TRAILING_CHARACTERS_REGEX, '').trimRight(); + + const current = indentLevel <= previousIndentLevel ? + path.join(...previousDir.split(path.sep).slice(0, indentLevel - 1), fileName) : + path.join(previousDir, fileName); + + if (previousDir) { + // Because of the structure of a tree output, we need the next line + // to tell whether the current one is a directory or not. + // If the indentation as increased (or it was forcefully marked as a directory by ending with "(D)") + // then we know the previous file is a directory + if (indentLevel > previousIndentLevel || wasDirectory) { + directories.push(previousDir); + } else if (FsUtils.SYMBOLIC_LINK_REGEX.test(previousDir)) { + const [link, target] = previousDir.split(FsUtils.SYMBOLIC_LINK_REGEX); + symlinks.push([link, target]); + } else { + files.push(previousDir); + } + } + + return [current, indentLevel, FsUtils.IS_DIRECTORY_REGEX.test(line)]; + }, ['', 0, false]); + + // we create the directories first, as they're needed to store the files + for (const _directory of directories) { + fs.mkdirSync(path.join(directory, _directory)); + } + // we create the files first, as they're needed to create the symlinks + for (const file of files) { + fs.writeFileSync(path.join(directory, file), content); + } + for (const [link, target] of symlinks) { + fs.symlinkSync(target, path.join(directory, link)); + } + + return { + directory, + cleanup: () => { + // reverse order of the creation, we need to empty the directories before rmdir + for (const [link] of symlinks) { + fs.unlinkSync(path.join(directory, link)); + } + for (const file of files) { + fs.unlinkSync(path.join(directory, file)); + } + for (const _directory of directories.reverse()) { + fs.rmdirSync(path.join(directory, _directory)); + } + + // finally, we delete the directory created by mkdtempSync + fs.rmdirSync(directory); + } + }; + } + + /** + * RegExp matching characters used to indent the tree, indicating the line depth + */ + private static readonly INDENT_CHARACTERS_REGEX = /^[\s├─│└]+/; + /** + * RegExp matching characters trailing a tree line + */ + private static readonly TRAILING_CHARACTERS_REGEX = /(\/|\(D\))$/i; + /** + * RegExp determining whether a given line is an explicit directory + */ + private static readonly IS_DIRECTORY_REGEX = /\(D\)\s*$/i; + /** + * RegExp determining whether a given line is a symblic link + */ + private static readonly SYMBOLIC_LINK_REGEX = /\s*[=-]>\s*/; + +} \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/test.fs-utils.ts b/packages/@aws-cdk/assets/test/fs/test.fs-utils.ts new file mode 100644 index 0000000000000..a284281765b3a --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/test.fs-utils.ts @@ -0,0 +1,131 @@ +import fs = require('fs'); +import { Test } from 'nodeunit'; +import os = require('os'); +import path = require('path'); +import { FsUtils } from './fs-utils'; + +export = { + fromTree: { + 'basic usage'(test: Test) { + // GIVEN + const tree = ` + ├── foo + └── dir + └── subdir/ + └── bar.txt`; + + // THEN + const { directory, cleanup } = FsUtils.fromTree('basic-usage', tree); + + test.ok(fs.existsSync(path.join(directory, 'foo'))); + test.ok(fs.existsSync(path.join(directory, 'dir', 'subdir', 'bar.txt'))); + + cleanup(); + + test.ok(!fs.existsSync(path.join(directory, 'foo'))); + test.ok(!fs.existsSync(path.join(directory, 'dir', 'subdir', 'bar.txt'))); + test.ok(!fs.existsSync(directory)); + + test.done(); + }, + 'symlinks'(test: Test) { + // GIVEN + const tree = ` + ├── link -> target + ├── other_link=>target + ├── target + └── foo.txt`; + + // THEN + const { directory, cleanup } = FsUtils.fromTree('symlink', tree); + + test.ok(fs.existsSync(path.join(directory, 'target', 'foo.txt'))); + test.ok(fs.existsSync(path.join(directory, 'link', 'foo.txt'))); + test.ok(fs.existsSync(path.join(directory, 'other_link', 'foo.txt'))); + test.equal(fs.readlinkSync(path.join(directory, 'link')), 'target'); + test.equal(fs.readlinkSync(path.join(directory, 'other_link')), 'target'); + + cleanup(); + + test.ok(!fs.existsSync(path.join(directory, 'target'))); + test.ok(!fs.existsSync(path.join(directory, 'link'))); + test.ok(!fs.existsSync(path.join(directory, 'other_link'))); + test.ok(!fs.existsSync(directory)); + + test.done(); + }, + 'external smylinks'(test: Test) { + // GIVEN + const externalTree = FsUtils.fromTree('external', ` + ├── external_dir + │   ├── foobar.txt`); + + // THEN + + const externalRelativeDirectory = path.relative(os.tmpdir(), externalTree.directory); + const externalLink = `../${externalRelativeDirectory}/external_dir`; + + const internalTree = FsUtils.fromTree('internal', ` + ├── external_link -> ${externalLink}`); + + test.ok(fs.existsSync(path.join(externalTree.directory, 'external_dir', 'foobar.txt'))); + test.ok(fs.existsSync(path.join(internalTree.directory, 'external_link', 'foobar.txt'))); + test.equal(fs.readlinkSync(path.join(internalTree.directory, 'external_link')), externalLink); + + externalTree.cleanup(); + internalTree.cleanup(); + + test.ok(!fs.existsSync(path.join(externalTree.directory, 'external_dir'))); + test.ok(!fs.existsSync(path.join(internalTree.directory, 'external_link'))); + test.ok(!fs.existsSync(internalTree.directory)); + test.ok(!fs.existsSync(externalTree.directory)); + + test.done(); + }, + 'empty directory'(test: Test) { + // GIVEN + const tree = ` + ├── dir (D)`; + + // THEN + const { directory, cleanup } = FsUtils.fromTree('empty-directory', tree); + + test.ok(fs.existsSync(path.join(directory, 'dir'))); + + cleanup(); + + test.ok(!fs.existsSync(path.join(directory, 'dir'))); + test.ok(!fs.existsSync(directory)); + + test.done(); + }, + 'works with any indent'(test: Test) { + // GIVEN + const tree = ` + + + + ├── foo + └── dir + └── subdir + └── bar.txt + + + `; + + // THEN + const { directory, cleanup } = FsUtils.fromTree('any-indent', tree); + + test.ok(fs.existsSync(path.join(directory, 'foo'))); + test.ok(fs.existsSync(path.join(directory, 'dir', 'subdir', 'bar.txt'))); + + cleanup(); + + test.ok(!fs.existsSync(path.join(directory, 'foo'))); + test.ok(!fs.existsSync(path.join(directory, 'dir', 'subdir', 'bar.txt'))); + test.ok(!fs.existsSync(directory)); + + test.done(); + }, + }, +}; \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/test.utils.ts b/packages/@aws-cdk/assets/test/fs/test.utils.ts index 8f93d8a2c77eb..51f83f611e3ae 100644 --- a/packages/@aws-cdk/assets/test/fs/test.utils.ts +++ b/packages/@aws-cdk/assets/test/fs/test.utils.ts @@ -1,4 +1,3 @@ -import { FsUtils } from '@aws-cdk/assert'; import fs = require('fs'); import { Test } from 'nodeunit'; import os = require('os'); @@ -6,6 +5,7 @@ import path = require('path'); import { ImportMock } from 'ts-mock-imports'; import { FollowMode } from '../../lib/fs'; import util = require('../../lib/fs/utils'); +import { FsUtils } from './fs-utils'; export = { shouldExclude: { From fc95caa64e99084a5f50a07d7f424a5fb1fa00f3 Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Tue, 29 Oct 2019 14:43:50 +0100 Subject: [PATCH 35/35] chore: move FsUtils to assets (unstaged files) --- packages/@aws-cdk/assert/lib/fs-utils.ts | 98 -------------- packages/@aws-cdk/assert/lib/index.ts | 1 - .../@aws-cdk/assert/test/test.fs-utils.ts | 127 ------------------ .../aws-ecr-assets/test/test.image-asset.ts | 3 +- 4 files changed, 2 insertions(+), 227 deletions(-) delete mode 100644 packages/@aws-cdk/assert/lib/fs-utils.ts delete mode 100644 packages/@aws-cdk/assert/test/test.fs-utils.ts diff --git a/packages/@aws-cdk/assert/lib/fs-utils.ts b/packages/@aws-cdk/assert/lib/fs-utils.ts deleted file mode 100644 index dec82ffac97e2..0000000000000 --- a/packages/@aws-cdk/assert/lib/fs-utils.ts +++ /dev/null @@ -1,98 +0,0 @@ -import fs = require('fs'); -import os = require('os'); -import path = require('path'); - -interface FromTreeOutput { - /** - * Absolute path of the created temporary directory, containing the generated structure - */ - readonly directory: string; - /** - * Cleanup function that will remove the generated files once called - */ - readonly cleanup: () => void; -} - -/** - * Collection of file-system utility methods - */ -export class FsUtils { - /** - * Generates a filesystem structure from a string, - * formatted like the output of a `tree` shell command - * - * @param tmpPrefix temp directory prefix, used by {@link fs.mkdtemp} - * @param tree - * @param content the content - * - * @returns an array containing the absolute path of the created directory, - * and a cleanup function that will remove the generated files when called - */ - public static fromTree(tmpPrefix: string, tree: string, content = 'content'): FromTreeOutput { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), tmpPrefix)); - - const directories: string[] = []; - const files: string[] = []; - const symlinks: Array<[string, string]> = []; - - // we push an element at the end because we push the files/directories during the previous iteration - const lines = [...tree.replace(/^\n+/, '').trimRight().split('\n'), '']; - const initialIndentLevel = (lines[0].match(/^\s*/) || [''])[0].length; - - lines.reduce<[string, number, boolean]>(([previousDir, previousIndentLevel, wasDirectory], line) => { - const indentCharacters = (line.match(FsUtils.INDENT_CHARACTERS_REGEX) || [''])[0]; - const indentLevel = (indentCharacters.length - initialIndentLevel) / 4; - - const fileName = line.slice(indentCharacters.length).replace(FsUtils.TRAILING_CHARACTERS_REGEX, '').trimRight(); - - const current = indentLevel <= previousIndentLevel ? - path.join(...previousDir.split(path.sep).slice(0, indentLevel - 1), fileName) : - path.join(previousDir, fileName); - - if (previousDir) { - if (indentLevel > previousIndentLevel || wasDirectory) { - directories.push(previousDir); - } else if (previousDir.includes('->')) { - const [link, target] = previousDir.split(/\s*->\s*/); - symlinks.push([link, target]); - } else { - files.push(previousDir); - } - } - - return [current, indentLevel, FsUtils.IS_DIRECTORY_REGEX.test(line)]; - }, ['', 0, false]); - - for (const _directory of directories) { - fs.mkdirSync(path.join(directory, _directory)); - } - for (const file of files) { - fs.writeFileSync(path.join(directory, file), content); - } - for (const [link, target] of symlinks) { - fs.symlinkSync(target, path.join(directory, link)); - } - - return { - directory, - cleanup: () => { - for (const [link] of symlinks) { - fs.unlinkSync(path.join(directory, link)); - } - for (const file of files) { - fs.unlinkSync(path.join(directory, file)); - } - for (const _directory of directories.reverse()) { - fs.rmdirSync(path.join(directory, _directory)); - } - - fs.rmdirSync(directory); - } - }; - } - - private static readonly INDENT_CHARACTERS_REGEX = /^[\s├─│└]+/; - private static readonly TRAILING_CHARACTERS_REGEX = /(\/|\(D\))$/i; - private static readonly IS_DIRECTORY_REGEX = /\(D\)\s*$/i; - -} \ No newline at end of file diff --git a/packages/@aws-cdk/assert/lib/index.ts b/packages/@aws-cdk/assert/lib/index.ts index a7aa6078b3bef..b79d93592affc 100644 --- a/packages/@aws-cdk/assert/lib/index.ts +++ b/packages/@aws-cdk/assert/lib/index.ts @@ -1,6 +1,5 @@ export * from './assertion'; export * from './expect'; -export * from './fs-utils'; export * from './inspector'; export * from './synth-utils'; diff --git a/packages/@aws-cdk/assert/test/test.fs-utils.ts b/packages/@aws-cdk/assert/test/test.fs-utils.ts deleted file mode 100644 index b621dcc0b7b73..0000000000000 --- a/packages/@aws-cdk/assert/test/test.fs-utils.ts +++ /dev/null @@ -1,127 +0,0 @@ -import fs = require('fs'); -import { Test } from 'nodeunit'; -import os = require('os'); -import path = require('path'); -import { FsUtils } from '../lib'; - -export = { - fromTree: { - 'basic usage'(test: Test) { - // GIVEN - const tree = ` - ├── foo - └── dir - └── subdir/ - └── bar.txt`; - - // THEN - const { directory, cleanup } = FsUtils.fromTree('basic-usage', tree); - - test.ok(fs.existsSync(path.join(directory, 'foo'))); - test.ok(fs.existsSync(path.join(directory, 'dir', 'subdir', 'bar.txt'))); - - cleanup(); - - test.ok(!fs.existsSync(path.join(directory, 'foo'))); - test.ok(!fs.existsSync(path.join(directory, 'dir', 'subdir', 'bar.txt'))); - test.ok(!fs.existsSync(directory)); - - test.done(); - }, - 'symlinks'(test: Test) { - // GIVEN - const tree = ` - ├── link -> target - ├── target - └── foo.txt`; - - // THEN - const { directory, cleanup } = FsUtils.fromTree('symlink', tree); - - test.ok(fs.existsSync(path.join(directory, 'target', 'foo.txt'))); - test.ok(fs.existsSync(path.join(directory, 'link', 'foo.txt'))); - test.equal(fs.readlinkSync(path.join(directory, 'link')), 'target'); - - cleanup(); - - test.ok(!fs.existsSync(path.join(directory, 'target'))); - test.ok(!fs.existsSync(path.join(directory, 'link'))); - test.ok(!fs.existsSync(directory)); - - test.done(); - }, - 'external smylinks'(test: Test) { - // GIVEN - const externalTree = FsUtils.fromTree('external', ` - ├── external_dir - │   ├── foobar.txt`); - - // THEN - - const externalRelativeDirectory = path.relative(os.tmpdir(), externalTree.directory); - const externalLink = `../${externalRelativeDirectory}/external_dir`; - - const internalTree = FsUtils.fromTree('internal', ` - ├── external_link -> ${externalLink}`); - - test.ok(fs.existsSync(path.join(externalTree.directory, 'external_dir', 'foobar.txt'))); - test.ok(fs.existsSync(path.join(internalTree.directory, 'external_link', 'foobar.txt'))); - test.equal(fs.readlinkSync(path.join(internalTree.directory, 'external_link')), externalLink); - - externalTree.cleanup(); - internalTree.cleanup(); - - test.ok(!fs.existsSync(path.join(externalTree.directory, 'external_dir'))); - test.ok(!fs.existsSync(path.join(internalTree.directory, 'external_link'))); - test.ok(!fs.existsSync(internalTree.directory)); - test.ok(!fs.existsSync(externalTree.directory)); - - test.done(); - }, - 'empty directory'(test: Test) { - // GIVEN - const tree = ` - ├── dir (D)`; - - // THEN - const { directory, cleanup } = FsUtils.fromTree('empty-directory', tree); - - test.ok(fs.existsSync(path.join(directory, 'dir'))); - - cleanup(); - - test.ok(!fs.existsSync(path.join(directory, 'dir'))); - test.ok(!fs.existsSync(directory)); - - test.done(); - }, - 'works with any indent'(test: Test) { - // GIVEN - const tree = ` - - - - ├── foo - └── dir - └── subdir - └── bar.txt - - - `; - - // THEN - const { directory, cleanup } = FsUtils.fromTree('any-indent', tree); - - test.ok(fs.existsSync(path.join(directory, 'foo'))); - test.ok(fs.existsSync(path.join(directory, 'dir', 'subdir', 'bar.txt'))); - - cleanup(); - - test.ok(!fs.existsSync(path.join(directory, 'foo'))); - test.ok(!fs.existsSync(path.join(directory, 'dir', 'subdir', 'bar.txt'))); - test.ok(!fs.existsSync(directory)); - - test.done(); - }, - }, -}; \ No newline at end of file diff --git a/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts b/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts index 1559adf56eef5..e8d7b47f9345e 100644 --- a/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts +++ b/packages/@aws-cdk/aws-ecr-assets/test/test.image-asset.ts @@ -1,4 +1,5 @@ -import { expect, FsUtils, haveResource, SynthUtils } from '@aws-cdk/assert'; +import { expect, haveResource, SynthUtils } from '@aws-cdk/assert'; +import { FsUtils } from '@aws-cdk/assets/test/fs/fs-utils'; import iam = require('@aws-cdk/aws-iam'); import { App, Construct, Lazy, Resource, Stack } from '@aws-cdk/core'; import { ASSET_METADATA } from '@aws-cdk/cx-api';