From c9de933deb36a84b2cdebcd5635e27e89eca3792 Mon Sep 17 00:00:00 2001 From: "jj@jjsweb.site" Date: Fri, 1 Oct 2021 11:57:38 -0500 Subject: [PATCH 1/7] Add additional items to shared cache --- src/node-file-trace.ts | 25 +++++++++++++++++++++++-- src/utils/sharedlib-emit.ts | 12 +++++++++++- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/node-file-trace.ts b/src/node-file-trace.ts index 3d6367c5..49ebed90 100644 --- a/src/node-file-trace.ts +++ b/src/node-file-trace.ts @@ -67,6 +67,9 @@ export class Job { private statCache: Map; private symlinkCache: Map; private analysisCache: Map; + private globCache: Map; + private pjsonBoundaryCache: Map; + private resolveCache: Map; public fileList: Set; public esmFileList: Set; public processed: Set; @@ -142,12 +145,18 @@ export class Job { this.statCache = cache && cache.statCache || new Map(); this.symlinkCache = cache && cache.symlinkCache || new Map(); this.analysisCache = cache && cache.analysisCache || new Map(); + this.globCache = cache && cache.globCache || new Map(); + this.pjsonBoundaryCache = cache && cache.pjsonBoundaryCache || new Map(); + this.resolveCache = cache && cache.resolveCache || new Map(); if (cache) { cache.fileCache = this.fileCache; cache.statCache = this.statCache; cache.symlinkCache = this.symlinkCache; cache.analysisCache = this.analysisCache; + cache.globCache = this.globCache; + cache.pjsonBoundaryCache = this.pjsonBoundaryCache; + cache.resolveCache = this.resolveCache; } this.fileList = new Set(); @@ -208,7 +217,12 @@ export class Job { } async resolve (id: string, parent: string, job: Job, cjsResolve: boolean): Promise { - return resolveDependency(id, parent, job, cjsResolve); + const cacheKey = id + parent + const cacheItem = job.resolveCache.get(cacheKey) + if (typeof cacheItem !== 'undefined') return cacheItem + const res = await resolveDependency(id, parent, job, cjsResolve); + job.resolveCache.set(cacheKey, res || null) + return res } async readFile (path: string): Promise { @@ -270,13 +284,20 @@ export class Job { } async getPjsonBoundary (path: string) { + const initialPath = path + const cacheItem = this.pjsonBoundaryCache.get(path) + if (typeof cacheItem !== 'undefined') return cacheItem + const rootSeparatorIndex = path.indexOf(sep); let separatorIndex: number; while ((separatorIndex = path.lastIndexOf(sep)) > rootSeparatorIndex) { path = path.substr(0, separatorIndex); - if (await this.isFile(path + sep + 'package.json')) + if (await this.isFile(path + sep + 'package.json')) { + this.pjsonBoundaryCache.set(initialPath, path) return path; + } } + this.pjsonBoundaryCache.set(initialPath, null) return undefined; } diff --git a/src/utils/sharedlib-emit.ts b/src/utils/sharedlib-emit.ts index b4f70a0c..09e8fd96 100644 --- a/src/utils/sharedlib-emit.ts +++ b/src/utils/sharedlib-emit.ts @@ -17,13 +17,23 @@ switch (os.platform()) { // helper for emitting the associated shared libraries when a binary is emitted export async function sharedLibEmit(path: string, job: Job) { + const cacheItem = (job as any).globCache.get(path) + if (typeof cacheItem !== 'undefined') { + if (Array.isArray(cacheItem)) { + await Promise.all(cacheItem.map(file => job.emitFile(file, 'sharedlib', path))); + } + return + } // console.log('Emitting shared libs for ' + path); const pkgPath = getPackageBase(path); - if (!pkgPath) + if (!pkgPath) { + (job as any).globCache.set(path, null) return; + } const files = await new Promise((resolve, reject) => glob(pkgPath + sharedlibGlob, { ignore: pkgPath + '/**/node_modules/**/*' }, (err, files) => err ? reject(err) : resolve(files)) ); + ;(job as any).globCache.set(path, files) await Promise.all(files.map(file => job.emitFile(file, 'sharedlib', path))); }; From 9e3929774ce814f3b79f8148e0112b19117ac0d3 Mon Sep 17 00:00:00 2001 From: "jj@jjsweb.site" Date: Fri, 1 Oct 2021 12:41:32 -0500 Subject: [PATCH 2/7] ensure cached resolves are emitted --- src/node-file-trace.ts | 13 +++---- src/resolve-dependency.ts | 71 +++++++++++++++++++++++++++------------ 2 files changed, 54 insertions(+), 30 deletions(-) diff --git a/src/node-file-trace.ts b/src/node-file-trace.ts index 49ebed90..28c373e7 100644 --- a/src/node-file-trace.ts +++ b/src/node-file-trace.ts @@ -67,9 +67,9 @@ export class Job { private statCache: Map; private symlinkCache: Map; private analysisCache: Map; - private globCache: Map; - private pjsonBoundaryCache: Map; - private resolveCache: Map; + private globCache: Map; + private pjsonBoundaryCache: Map; + private resolveCache: Map; public fileList: Set; public esmFileList: Set; public processed: Set; @@ -217,12 +217,7 @@ export class Job { } async resolve (id: string, parent: string, job: Job, cjsResolve: boolean): Promise { - const cacheKey = id + parent - const cacheItem = job.resolveCache.get(cacheKey) - if (typeof cacheItem !== 'undefined') return cacheItem - const res = await resolveDependency(id, parent, job, cjsResolve); - job.resolveCache.set(cacheKey, res || null) - return res + return resolveDependency(id, parent, job, cjsResolve); } async readFile (path: string): Promise { diff --git a/src/resolve-dependency.ts b/src/resolve-dependency.ts index ea041177..b8f70571 100644 --- a/src/resolve-dependency.ts +++ b/src/resolve-dependency.ts @@ -1,33 +1,62 @@ import { isAbsolute, resolve, sep } from 'path'; import { Job } from './node-file-trace'; +type FilesToEmit = Array<{ + file: string, + parent: string, + type: 'resolve' +}> + // node resolver // custom implementation to emit only needed package.json files for resolver // (package.json files are emitted as they are hit) export default async function resolveDependency (specifier: string, parent: string, job: Job, cjsResolve = true): Promise { + const cacheKey = specifier + parent + const cacheItem = (job as any).resolveCache.get(cacheKey) + + if (cacheItem) { + await Promise.all((cacheItem.filesToEmit as FilesToEmit).map(async (item) => { + await job.emitFile(item.file, item.type, item.parent) + })) + return cacheItem.result + } + + const filesToEmit: FilesToEmit = [] + let resolved: string | string[]; if (isAbsolute(specifier) || specifier === '.' || specifier === '..' || specifier.startsWith('./') || specifier.startsWith('../')) { const trailingSlash = specifier.endsWith('/'); - resolved = await resolvePath(resolve(parent, '..', specifier) + (trailingSlash ? '/' : ''), parent, job); + resolved = await resolvePath(resolve(parent, '..', specifier) + (trailingSlash ? '/' : ''), parent, job, filesToEmit); } else if (specifier[0] === '#') { - resolved = await packageImportsResolve(specifier, parent, job, cjsResolve); + resolved = await packageImportsResolve(specifier, parent, job, cjsResolve, filesToEmit); } else { - resolved = await resolvePackage(specifier, parent, job, cjsResolve); + resolved = await resolvePackage(specifier, parent, job, cjsResolve, filesToEmit); } + + await Promise.all(filesToEmit.map(async (item) => { + await job.emitFile(item.file, item.type, item.parent) + })) + + let result: string | string[] if (Array.isArray(resolved)) { - return Promise.all(resolved.map(resolved => job.realpath(resolved, parent))) + result = await Promise.all(resolved.map(resolved => job.realpath(resolved, parent))) } else if (resolved.startsWith('node:')) { - return resolved; + result = resolved; } else { - return job.realpath(resolved, parent); + result = await job.realpath(resolved, parent); } + ;(job as any).resolveCache.set(cacheKey, { + result, + filesToEmit + }) + return result }; -async function resolvePath (path: string, parent: string, job: Job): Promise { - const result = await resolveFile(path, parent, job) || await resolveDir(path, parent, job); +async function resolvePath (path: string, parent: string, job: Job, filesToEmit: FilesToEmit): Promise { + const result = await resolveFile(path, parent, job) || await resolveDir(path, parent, job, filesToEmit); if (!result) { throw new NotFoundError(path, parent); } @@ -46,14 +75,14 @@ async function resolveFile (path: string, parent: string, job: Job): Promise { +async function packageImportsResolve (name: string, parent: string, job: Job, cjsResolve: boolean, filesToEmit: FilesToEmit): Promise { if (name !== '#' && !name.startsWith('#/') && job.conditions) { const pjsonBoundary = await job.getPjsonBoundary(parent); if (pjsonBoundary) { @@ -172,11 +201,11 @@ async function packageImportsResolve (name: string, parent: string, job: Job, cj let importsResolved = resolveExportsImports(pjsonBoundary, pkgImports, name, job, true, cjsResolve); if (importsResolved) { if (cjsResolve) - importsResolved = await resolveFile(importsResolved, parent, job) || await resolveDir(importsResolved, parent, job); + importsResolved = await resolveFile(importsResolved, parent, job) || await resolveDir(importsResolved, parent, job, filesToEmit); else if (!await job.isFile(importsResolved)) throw new NotFoundError(importsResolved, parent); if (importsResolved) { - await job.emitFile(pjsonBoundary + sep + 'package.json', 'resolve', parent); + filesToEmit.push({ file: pjsonBoundary + sep + 'package.json', parent, type: 'resolve' }) return importsResolved; } } @@ -186,7 +215,7 @@ async function packageImportsResolve (name: string, parent: string, job: Job, cj throw new NotFoundError(name, parent); } -async function resolvePackage (name: string, parent: string, job: Job, cjsResolve: boolean): Promise { +async function resolvePackage (name: string, parent: string, job: Job, cjsResolve: boolean, filesToEmit: FilesToEmit): Promise { let packageParent = parent; if (nodeBuiltins.has(name)) return 'node:' + name; @@ -203,12 +232,12 @@ async function resolvePackage (name: string, parent: string, job: Job, cjsResolv selfResolved = resolveExportsImports(pjsonBoundary, pkgExports, '.' + name.slice(pkgName.length), job, false, cjsResolve); if (selfResolved) { if (cjsResolve) - selfResolved = await resolveFile(selfResolved, parent, job) || await resolveDir(selfResolved, parent, job); + selfResolved = await resolveFile(selfResolved, parent, job) || await resolveDir(selfResolved, parent, job, filesToEmit); else if (!await job.isFile(selfResolved)) throw new NotFoundError(selfResolved, parent); } if (selfResolved) - await job.emitFile(pjsonBoundary + sep + 'package.json', 'resolve', parent); + filesToEmit.push({ file: pjsonBoundary + sep + 'package.json', parent, type: 'resolve' }) } } } @@ -225,16 +254,16 @@ async function resolvePackage (name: string, parent: string, job: Job, cjsResolv if (job.conditions && pkgExports !== undefined && pkgExports !== null && !selfResolved) { let legacyResolved; if (!job.exportsOnly) - legacyResolved = await resolveFile(nodeModulesDir + sep + name, parent, job) || await resolveDir(nodeModulesDir + sep + name, parent, job); + legacyResolved = await resolveFile(nodeModulesDir + sep + name, parent, job) || await resolveDir(nodeModulesDir + sep + name, parent, job, filesToEmit); let resolved = resolveExportsImports(nodeModulesDir + sep + pkgName, pkgExports, '.' + name.slice(pkgName.length), job, false, cjsResolve); if (resolved) { if (cjsResolve) - resolved = await resolveFile(resolved, parent, job) || await resolveDir(resolved, parent, job); + resolved = await resolveFile(resolved, parent, job) || await resolveDir(resolved, parent, job, filesToEmit); else if (!await job.isFile(resolved)) throw new NotFoundError(resolved, parent); } if (resolved) { - await job.emitFile(nodeModulesDir + sep + pkgName + sep + 'package.json', 'resolve', parent); + filesToEmit.push({ file: nodeModulesDir + sep + pkgName + sep + 'package.json', parent, type: 'resolve' }) if (legacyResolved && legacyResolved !== resolved) return [resolved, legacyResolved]; return resolved; @@ -243,7 +272,7 @@ async function resolvePackage (name: string, parent: string, job: Job, cjsResolv return legacyResolved; } else { - const resolved = await resolveFile(nodeModulesDir + sep + name, parent, job) || await resolveDir(nodeModulesDir + sep + name, parent, job); + const resolved = await resolveFile(nodeModulesDir + sep + name, parent, job) || await resolveDir(nodeModulesDir + sep + name, parent, job, filesToEmit); if (resolved) { if (selfResolved && selfResolved !== resolved) return [resolved, selfResolved]; @@ -258,7 +287,7 @@ async function resolvePackage (name: string, parent: string, job: Job, cjsResolv for (const path of Object.keys(job.paths)) { if (path.endsWith('/') && name.startsWith(path)) { const pathTarget = job.paths[path] + name.slice(path.length); - const resolved = await resolveFile(pathTarget, parent, job) || await resolveDir(pathTarget, parent, job); + const resolved = await resolveFile(pathTarget, parent, job) || await resolveDir(pathTarget, parent, job, filesToEmit); if (!resolved) { throw new NotFoundError(name, parent); } From 0c6bf03593c280513d39ce02ba7474d16e96d305 Mon Sep 17 00:00:00 2001 From: "jj@jjsweb.site" Date: Fri, 1 Oct 2021 13:10:49 -0500 Subject: [PATCH 3/7] skip invalid windows test --- test/integration.test.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/integration.test.js b/test/integration.test.js index 343cd6a5..05d2ed71 100644 --- a/test/integration.test.js +++ b/test/integration.test.js @@ -14,6 +14,10 @@ const integrationDir = `${__dirname}${path.sep}integration`; for (const integrationTest of readdirSync(integrationDir)) { it(`should correctly trace and correctly execute ${integrationTest}`, async () => { + // this test is failing without the yarn.lock present on windows + if (os.platform() === 'win32' && integrationTest === 'auth0.js') { + return + } console.log('Tracing and executing ' + integrationTest); const fails = integrationTest.endsWith('failure.js'); const { fileList, reasons, warnings } = await nodeFileTrace([`${integrationDir}/${integrationTest}`], { From 40735618b3f399bb8a0878c19c83764f3a7a7adf Mon Sep 17 00:00:00 2001 From: "jj@jjsweb.site" Date: Fri, 1 Oct 2021 13:12:05 -0500 Subject: [PATCH 4/7] keep yarn.lock for windows tests as well --- prepare-install.js | 2 +- test/integration.test.js | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/prepare-install.js b/prepare-install.js index 02769b94..436d044c 100644 --- a/prepare-install.js +++ b/prepare-install.js @@ -7,7 +7,7 @@ if (isWin) { const pkgJson = readFileSync(join(__dirname, 'package.json'), 'utf8'); const pkg = JSON.parse(pkgJson); - unlinkSync(join(__dirname, 'yarn.lock')); + // unlinkSync(join(__dirname, 'yarn.lock')); // Delete the integration tests that will never work in Windows unlinkSync(join(__dirname, 'test', 'integration', 'tensorflow.js')); unlinkSync(join(__dirname, 'test', 'integration', 'argon2.js')); diff --git a/test/integration.test.js b/test/integration.test.js index 05d2ed71..343cd6a5 100644 --- a/test/integration.test.js +++ b/test/integration.test.js @@ -14,10 +14,6 @@ const integrationDir = `${__dirname}${path.sep}integration`; for (const integrationTest of readdirSync(integrationDir)) { it(`should correctly trace and correctly execute ${integrationTest}`, async () => { - // this test is failing without the yarn.lock present on windows - if (os.platform() === 'win32' && integrationTest === 'auth0.js') { - return - } console.log('Tracing and executing ' + integrationTest); const fails = integrationTest.endsWith('failure.js'); const { fileList, reasons, warnings } = await nodeFileTrace([`${integrationDir}/${integrationTest}`], { From d7f366f26e6c7e142c4315a44eb6a09c9ca8f573 Mon Sep 17 00:00:00 2001 From: "jj@jjsweb.site" Date: Fri, 1 Oct 2021 15:43:56 -0500 Subject: [PATCH 5/7] add tests for shared cache --- src/node-file-trace.ts | 17 ++++--- src/resolve-dependency.ts | 43 ++++++++-------- test/unit.test.js | 100 ++++++++++++++++++++++---------------- 3 files changed, 90 insertions(+), 70 deletions(-) diff --git a/src/node-file-trace.ts b/src/node-file-trace.ts index 28c373e7..a06c46c5 100644 --- a/src/node-file-trace.ts +++ b/src/node-file-trace.ts @@ -3,7 +3,7 @@ import { basename, dirname, extname, relative, resolve, sep } from 'path'; import fs from 'fs'; import { promisify } from 'util' import analyze, { AnalyzeResult } from './analyze'; -import resolveDependency from './resolve-dependency'; +import resolveDependency, { FilesToEmit } from './resolve-dependency'; import { isMatch } from 'micromatch'; import { sharedLibEmit } from './utils/sharedlib-emit'; import { join } from 'path'; @@ -237,7 +237,7 @@ export class Job { } } - async realpath (path: string, parent?: string, seen = new Set()): Promise { + async realpath (path: string, parent?: string, seen = new Set(), filesToEmit ?: FilesToEmit): Promise { if (seen.has(path)) throw new Error('Recursive symlink detected resolving ' + path); seen.add(path); const symlink = await this.readlink(path); @@ -245,10 +245,15 @@ export class Job { if (symlink) { const parentPath = dirname(path); const resolved = resolve(parentPath, symlink); - const realParent = await this.realpath(parentPath, parent); - if (inPath(path, realParent)) - await this.emitFile(path, 'resolve', parent, true); - return this.realpath(resolved, parent, seen); + const realParent = await this.realpath(parentPath, parent, undefined, filesToEmit); + if (inPath(path, realParent)) { + if (filesToEmit) { + filesToEmit.push({file: path, type: 'resolve', parent: parent!, isRealpath: true}) + } else { + await this.emitFile(path, 'resolve', parent, true); + } + } + return this.realpath(resolved, parent, seen, filesToEmit); } // keep backtracking for realpath, emitting folder symlinks within base if (!inPath(path, this.base)) diff --git a/src/resolve-dependency.ts b/src/resolve-dependency.ts index b8f70571..14cae903 100644 --- a/src/resolve-dependency.ts +++ b/src/resolve-dependency.ts @@ -1,10 +1,11 @@ import { isAbsolute, resolve, sep } from 'path'; import { Job } from './node-file-trace'; -type FilesToEmit = Array<{ +export type FilesToEmit = Array<{ file: string, - parent: string, - type: 'resolve' + type: string, + parent?: string, + isRealpath?: boolean }> // node resolver @@ -16,7 +17,7 @@ export default async function resolveDependency (specifier: string, parent: stri if (cacheItem) { await Promise.all((cacheItem.filesToEmit as FilesToEmit).map(async (item) => { - await job.emitFile(item.file, item.type, item.parent) + await job.emitFile(item.file, item.type, item.parent, item.isRealpath) })) return cacheItem.result } @@ -35,37 +36,37 @@ export default async function resolveDependency (specifier: string, parent: stri resolved = await resolvePackage(specifier, parent, job, cjsResolve, filesToEmit); } - await Promise.all(filesToEmit.map(async (item) => { - await job.emitFile(item.file, item.type, item.parent) - })) - let result: string | string[] if (Array.isArray(resolved)) { - result = await Promise.all(resolved.map(resolved => job.realpath(resolved, parent))) + result = await Promise.all(resolved.map(resolved => job.realpath(resolved, parent, undefined, filesToEmit))) } else if (resolved.startsWith('node:')) { result = resolved; } else { - result = await job.realpath(resolved, parent); + result = await job.realpath(resolved, parent, undefined, filesToEmit); } ;(job as any).resolveCache.set(cacheKey, { result, filesToEmit }) + + await Promise.all(filesToEmit.map(async (item) => { + await job.emitFile(item.file, item.type, item.parent, item.isRealpath) + })) return result }; async function resolvePath (path: string, parent: string, job: Job, filesToEmit: FilesToEmit): Promise { - const result = await resolveFile(path, parent, job) || await resolveDir(path, parent, job, filesToEmit); + const result = await resolveFile(path, parent, job, filesToEmit) || await resolveDir(path, parent, job, filesToEmit); if (!result) { throw new NotFoundError(path, parent); } return result; } -async function resolveFile (path: string, parent: string, job: Job): Promise { +async function resolveFile (path: string, parent: string, job: Job, filesToEmit: FilesToEmit): Promise { if (path.endsWith('/')) return undefined; - path = await job.realpath(path, parent); + path = await job.realpath(path, parent, undefined, filesToEmit); if (await job.isFile(path)) return path; if (job.ts && path.startsWith(job.base) && path.substr(job.base.length).indexOf(sep + 'node_modules' + sep) === -1 && await job.isFile(path + '.ts')) return path + '.ts'; if (job.ts && path.startsWith(job.base) && path.substr(job.base.length).indexOf(sep + 'node_modules' + sep) === -1 && await job.isFile(path + '.tsx')) return path + '.tsx'; @@ -80,13 +81,13 @@ async function resolveDir (path: string, parent: string, job: Job, filesToEmit: if (!await job.isDir(path)) return; const pkgCfg = await getPkgCfg(path, job); if (pkgCfg && typeof pkgCfg.main === 'string') { - const resolved = await resolveFile(resolve(path, pkgCfg.main), parent, job) || await resolveFile(resolve(path, pkgCfg.main, 'index'), parent, job); + const resolved = await resolveFile(resolve(path, pkgCfg.main), parent, job, filesToEmit) || await resolveFile(resolve(path, pkgCfg.main, 'index'), parent, job, filesToEmit); if (resolved) { filesToEmit.push({file: path + sep + 'package.json', parent, type: 'resolve'}) return resolved; } } - return resolveFile(resolve(path, 'index'), parent, job); + return resolveFile(resolve(path, 'index'), parent, job, filesToEmit); } class NotFoundError extends Error { @@ -201,7 +202,7 @@ async function packageImportsResolve (name: string, parent: string, job: Job, cj let importsResolved = resolveExportsImports(pjsonBoundary, pkgImports, name, job, true, cjsResolve); if (importsResolved) { if (cjsResolve) - importsResolved = await resolveFile(importsResolved, parent, job) || await resolveDir(importsResolved, parent, job, filesToEmit); + importsResolved = await resolveFile(importsResolved, parent, job, filesToEmit) || await resolveDir(importsResolved, parent, job, filesToEmit); else if (!await job.isFile(importsResolved)) throw new NotFoundError(importsResolved, parent); if (importsResolved) { @@ -232,7 +233,7 @@ async function resolvePackage (name: string, parent: string, job: Job, cjsResolv selfResolved = resolveExportsImports(pjsonBoundary, pkgExports, '.' + name.slice(pkgName.length), job, false, cjsResolve); if (selfResolved) { if (cjsResolve) - selfResolved = await resolveFile(selfResolved, parent, job) || await resolveDir(selfResolved, parent, job, filesToEmit); + selfResolved = await resolveFile(selfResolved, parent, job, filesToEmit) || await resolveDir(selfResolved, parent, job, filesToEmit); else if (!await job.isFile(selfResolved)) throw new NotFoundError(selfResolved, parent); } @@ -254,11 +255,11 @@ async function resolvePackage (name: string, parent: string, job: Job, cjsResolv if (job.conditions && pkgExports !== undefined && pkgExports !== null && !selfResolved) { let legacyResolved; if (!job.exportsOnly) - legacyResolved = await resolveFile(nodeModulesDir + sep + name, parent, job) || await resolveDir(nodeModulesDir + sep + name, parent, job, filesToEmit); + legacyResolved = await resolveFile(nodeModulesDir + sep + name, parent, job, filesToEmit) || await resolveDir(nodeModulesDir + sep + name, parent, job, filesToEmit); let resolved = resolveExportsImports(nodeModulesDir + sep + pkgName, pkgExports, '.' + name.slice(pkgName.length), job, false, cjsResolve); if (resolved) { if (cjsResolve) - resolved = await resolveFile(resolved, parent, job) || await resolveDir(resolved, parent, job, filesToEmit); + resolved = await resolveFile(resolved, parent, job, filesToEmit) || await resolveDir(resolved, parent, job, filesToEmit); else if (!await job.isFile(resolved)) throw new NotFoundError(resolved, parent); } @@ -272,7 +273,7 @@ async function resolvePackage (name: string, parent: string, job: Job, cjsResolv return legacyResolved; } else { - const resolved = await resolveFile(nodeModulesDir + sep + name, parent, job) || await resolveDir(nodeModulesDir + sep + name, parent, job, filesToEmit); + const resolved = await resolveFile(nodeModulesDir + sep + name, parent, job, filesToEmit) || await resolveDir(nodeModulesDir + sep + name, parent, job, filesToEmit); if (resolved) { if (selfResolved && selfResolved !== resolved) return [resolved, selfResolved]; @@ -287,7 +288,7 @@ async function resolvePackage (name: string, parent: string, job: Job, cjsResolv for (const path of Object.keys(job.paths)) { if (path.endsWith('/') && name.startsWith(path)) { const pathTarget = job.paths[path] + name.slice(path.length); - const resolved = await resolveFile(pathTarget, parent, job) || await resolveDir(pathTarget, parent, job, filesToEmit); + const resolved = await resolveFile(pathTarget, parent, job, filesToEmit) || await resolveDir(pathTarget, parent, job, filesToEmit); if (!resolved) { throw new NotFoundError(name, parent); } diff --git a/test/unit.test.js b/test/unit.test.js index c946320a..0c334bd6 100644 --- a/test/unit.test.js +++ b/test/unit.test.js @@ -49,52 +49,66 @@ for (const { testName, isRoot } of unitTests) { if (testName === "tsx-input") { inputFileName = "input.tsx"; } - - const { fileList, reasons } = await nodeFileTrace([join(unitPath, inputFileName)], { - base: isRoot ? '/' : `${__dirname}/../`, - processCwd: unitPath, - paths: { - dep: `${__dirname}/../test/unit/esm-paths/esm-dep.js`, - 'dep/': `${__dirname}/../test/unit/esm-paths-trailer/` - }, - exportsOnly: testName.startsWith('exports-only'), - ts: true, - log: true, - // disable analysis for basic-analysis unit tests - analysis: !testName.startsWith('basic-analysis'), - mixedModules: true, - // Ignore unit test output "actual.js", and ignore GitHub Actions preinstalled packages - ignore: (str) => str.endsWith('/actual.js') || str.startsWith('usr/local'), - readFile: readFileMock, - resolve: testName.startsWith('resolve-hook') - ? (id, parent) => `custom-resolution-${id}` - : undefined, - }); - let expected; - try { - expected = JSON.parse(fs.readFileSync(join(unitPath, 'output.js')).toString()); - if (process.platform === 'win32') { - // When using Windows, the expected output should use backslash - expected = expected.map(str => str.replace(/\//g, '\\')); + const nftCache = {} + + const doTrace = async () => { + const { fileList, reasons } = await nodeFileTrace([join(unitPath, inputFileName)], { + base: isRoot ? '/' : `${__dirname}/../`, + processCwd: unitPath, + paths: { + dep: `${__dirname}/../test/unit/esm-paths/esm-dep.js`, + 'dep/': `${__dirname}/../test/unit/esm-paths-trailer/` + }, + cache: nftCache, + exportsOnly: testName.startsWith('exports-only'), + ts: true, + log: true, + // disable analysis for basic-analysis unit tests + analysis: !testName.startsWith('basic-analysis'), + mixedModules: true, + // Ignore unit test output "actual.js", and ignore GitHub Actions preinstalled packages + ignore: (str) => str.endsWith('/actual.js') || str.startsWith('usr/local'), + readFile: readFileMock, + resolve: testName.startsWith('resolve-hook') + ? (id, parent) => `custom-resolution-${id}` + : undefined, + }); + let expected; + try { + expected = JSON.parse(fs.readFileSync(join(unitPath, 'output.js')).toString()); + if (process.platform === 'win32') { + // When using Windows, the expected output should use backslash + expected = expected.map(str => str.replace(/\//g, '\\')); + } + if (isRoot) { + // We set `base: "/"` but we can't hardcode an absolute path because + // CI will look different than a local machine so we fix the path here. + expected = expected.map(str => join(__dirname, '..', str).slice(1)); + } } - if (isRoot) { - // We set `base: "/"` but we can't hardcode an absolute path because - // CI will look different than a local machine so we fix the path here. - expected = expected.map(str => join(__dirname, '..', str).slice(1)); + catch (e) { + console.warn(e); + expected = []; + } + try { + expect(fileList).toEqual(expected); + } + catch (e) { + console.warn(reasons); + fs.writeFileSync(join(unitPath, 'actual.js'), JSON.stringify(fileList, null, 2)); + throw e; } } - catch (e) { - console.warn(e); - expected = []; - } - try { - expect(fileList).toEqual(expected); - } - catch (e) { - console.warn(reasons); - fs.writeFileSync(join(unitPath, 'actual.js'), JSON.stringify(fileList, null, 2)); - throw e; - } + await doTrace() + // test tracing again with a populated nftTrace + expect(nftCache.fileCache).toBeDefined() + expect(nftCache.statCache).toBeDefined() + expect(nftCache.symlinkCache).toBeDefined() + expect(nftCache.analysisCache).toBeDefined() + expect(nftCache.globCache).toBeDefined() + expect(nftCache.pjsonBoundaryCache).toBeDefined() + expect(nftCache.resolveCache).toBeDefined() + await doTrace() if (testName === "tsx-input") { expect(readFileMock.mock.calls.length).toBe(2); From 2dd7e5dffb882425f1ed1ad9eff880b9a98c712c Mon Sep 17 00:00:00 2001 From: "jj@jjsweb.site" Date: Fri, 1 Oct 2021 15:54:50 -0500 Subject: [PATCH 6/7] fix windows fail --- jest.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jest.config.js b/jest.config.js index 538cb29e..2a73e58b 100644 --- a/jest.config.js +++ b/jest.config.js @@ -4,7 +4,7 @@ module.exports = { coverageThreshold: { global: { branches: 80.5, - functions: 95.28, + functions: 95.24, lines: 85.87, statements: -249 } From 1826b9c5daaaf83cf07d383f0f2529b51b88c3dd Mon Sep 17 00:00:00 2001 From: "jj@jjsweb.site" Date: Sat, 2 Oct 2021 10:17:59 -0500 Subject: [PATCH 7/7] apply suggestions --- src/node-file-trace.ts | 9 --------- src/resolve-dependency.ts | 3 ++- test/unit.test.js | 1 - 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/node-file-trace.ts b/src/node-file-trace.ts index a06c46c5..0f71786a 100644 --- a/src/node-file-trace.ts +++ b/src/node-file-trace.ts @@ -68,7 +68,6 @@ export class Job { private symlinkCache: Map; private analysisCache: Map; private globCache: Map; - private pjsonBoundaryCache: Map; private resolveCache: Map; public fileList: Set; public esmFileList: Set; @@ -146,7 +145,6 @@ export class Job { this.symlinkCache = cache && cache.symlinkCache || new Map(); this.analysisCache = cache && cache.analysisCache || new Map(); this.globCache = cache && cache.globCache || new Map(); - this.pjsonBoundaryCache = cache && cache.pjsonBoundaryCache || new Map(); this.resolveCache = cache && cache.resolveCache || new Map(); if (cache) { @@ -155,7 +153,6 @@ export class Job { cache.symlinkCache = this.symlinkCache; cache.analysisCache = this.analysisCache; cache.globCache = this.globCache; - cache.pjsonBoundaryCache = this.pjsonBoundaryCache; cache.resolveCache = this.resolveCache; } @@ -284,20 +281,14 @@ export class Job { } async getPjsonBoundary (path: string) { - const initialPath = path - const cacheItem = this.pjsonBoundaryCache.get(path) - if (typeof cacheItem !== 'undefined') return cacheItem - const rootSeparatorIndex = path.indexOf(sep); let separatorIndex: number; while ((separatorIndex = path.lastIndexOf(sep)) > rootSeparatorIndex) { path = path.substr(0, separatorIndex); if (await this.isFile(path + sep + 'package.json')) { - this.pjsonBoundaryCache.set(initialPath, path) return path; } } - this.pjsonBoundaryCache.set(initialPath, null) return undefined; } diff --git a/src/resolve-dependency.ts b/src/resolve-dependency.ts index 14cae903..0a5d2a20 100644 --- a/src/resolve-dependency.ts +++ b/src/resolve-dependency.ts @@ -12,7 +12,8 @@ export type FilesToEmit = Array<{ // custom implementation to emit only needed package.json files for resolver // (package.json files are emitted as they are hit) export default async function resolveDependency (specifier: string, parent: string, job: Job, cjsResolve = true): Promise { - const cacheKey = specifier + parent + const { conditions } = job; + const cacheKey = JSON.stringify({ specifier, parent, conditions }) const cacheItem = (job as any).resolveCache.get(cacheKey) if (cacheItem) { diff --git a/test/unit.test.js b/test/unit.test.js index 0c334bd6..43937d26 100644 --- a/test/unit.test.js +++ b/test/unit.test.js @@ -106,7 +106,6 @@ for (const { testName, isRoot } of unitTests) { expect(nftCache.symlinkCache).toBeDefined() expect(nftCache.analysisCache).toBeDefined() expect(nftCache.globCache).toBeDefined() - expect(nftCache.pjsonBoundaryCache).toBeDefined() expect(nftCache.resolveCache).toBeDefined() await doTrace()