diff --git a/jest.config.js b/jest.config.js index 538cb29e..2a73e58b 100644 --- a/jest.config.js +++ b/jest.config.js @@ -4,7 +4,7 @@ module.exports = { coverageThreshold: { global: { branches: 80.5, - functions: 95.28, + functions: 95.24, lines: 85.87, statements: -249 } diff --git a/prepare-install.js b/prepare-install.js index 02769b94..436d044c 100644 --- a/prepare-install.js +++ b/prepare-install.js @@ -7,7 +7,7 @@ if (isWin) { const pkgJson = readFileSync(join(__dirname, 'package.json'), 'utf8'); const pkg = JSON.parse(pkgJson); - unlinkSync(join(__dirname, 'yarn.lock')); + // unlinkSync(join(__dirname, 'yarn.lock')); // Delete the integration tests that will never work in Windows unlinkSync(join(__dirname, 'test', 'integration', 'tensorflow.js')); unlinkSync(join(__dirname, 'test', 'integration', 'argon2.js')); diff --git a/src/node-file-trace.ts b/src/node-file-trace.ts index 3d6367c5..0f71786a 100644 --- a/src/node-file-trace.ts +++ b/src/node-file-trace.ts @@ -3,7 +3,7 @@ import { basename, dirname, extname, relative, resolve, sep } from 'path'; import fs from 'fs'; import { promisify } from 'util' import analyze, { AnalyzeResult } from './analyze'; -import resolveDependency from './resolve-dependency'; +import resolveDependency, { FilesToEmit } from './resolve-dependency'; import { isMatch } from 'micromatch'; import { sharedLibEmit } from './utils/sharedlib-emit'; import { join } from 'path'; @@ -67,6 +67,8 @@ export class Job { private statCache: Map; private symlinkCache: Map; private analysisCache: Map; + private globCache: Map; + private resolveCache: Map; public fileList: Set; public esmFileList: Set; public processed: Set; @@ -142,12 +144,16 @@ export class Job { this.statCache = cache && cache.statCache || new Map(); this.symlinkCache = cache && cache.symlinkCache || new Map(); this.analysisCache = cache && cache.analysisCache || new Map(); + this.globCache = cache && cache.globCache || new Map(); + this.resolveCache = cache && cache.resolveCache || new Map(); if (cache) { cache.fileCache = this.fileCache; cache.statCache = this.statCache; cache.symlinkCache = this.symlinkCache; cache.analysisCache = this.analysisCache; + cache.globCache = this.globCache; + cache.resolveCache = this.resolveCache; } this.fileList = new Set(); @@ -228,7 +234,7 @@ export class Job { } } - async realpath (path: string, parent?: string, seen = new Set()): Promise { + async realpath (path: string, parent?: string, seen = new Set(), filesToEmit ?: FilesToEmit): Promise { if (seen.has(path)) throw new Error('Recursive symlink detected resolving ' + path); seen.add(path); const symlink = await this.readlink(path); @@ -236,10 +242,15 @@ export class Job { if (symlink) { const parentPath = dirname(path); const resolved = resolve(parentPath, symlink); - const realParent = await this.realpath(parentPath, parent); - if (inPath(path, realParent)) - await this.emitFile(path, 'resolve', parent, true); - return this.realpath(resolved, parent, seen); + const realParent = await this.realpath(parentPath, parent, undefined, filesToEmit); + if (inPath(path, realParent)) { + if (filesToEmit) { + filesToEmit.push({file: path, type: 'resolve', parent: parent!, isRealpath: true}) + } else { + await this.emitFile(path, 'resolve', parent, true); + } + } + return this.realpath(resolved, parent, seen, filesToEmit); } // keep backtracking for realpath, emitting folder symlinks within base if (!inPath(path, this.base)) @@ -274,8 +285,9 @@ export class Job { let separatorIndex: number; while ((separatorIndex = path.lastIndexOf(sep)) > rootSeparatorIndex) { path = path.substr(0, separatorIndex); - if (await this.isFile(path + sep + 'package.json')) + if (await this.isFile(path + sep + 'package.json')) { return path; + } } return undefined; } diff --git a/src/resolve-dependency.ts b/src/resolve-dependency.ts index ea041177..0a5d2a20 100644 --- a/src/resolve-dependency.ts +++ b/src/resolve-dependency.ts @@ -1,42 +1,73 @@ import { isAbsolute, resolve, sep } from 'path'; import { Job } from './node-file-trace'; +export type FilesToEmit = Array<{ + file: string, + type: string, + parent?: string, + isRealpath?: boolean +}> + // node resolver // custom implementation to emit only needed package.json files for resolver // (package.json files are emitted as they are hit) export default async function resolveDependency (specifier: string, parent: string, job: Job, cjsResolve = true): Promise { + const { conditions } = job; + const cacheKey = JSON.stringify({ specifier, parent, conditions }) + const cacheItem = (job as any).resolveCache.get(cacheKey) + + if (cacheItem) { + await Promise.all((cacheItem.filesToEmit as FilesToEmit).map(async (item) => { + await job.emitFile(item.file, item.type, item.parent, item.isRealpath) + })) + return cacheItem.result + } + + const filesToEmit: FilesToEmit = [] + let resolved: string | string[]; if (isAbsolute(specifier) || specifier === '.' || specifier === '..' || specifier.startsWith('./') || specifier.startsWith('../')) { const trailingSlash = specifier.endsWith('/'); - resolved = await resolvePath(resolve(parent, '..', specifier) + (trailingSlash ? '/' : ''), parent, job); + resolved = await resolvePath(resolve(parent, '..', specifier) + (trailingSlash ? '/' : ''), parent, job, filesToEmit); } else if (specifier[0] === '#') { - resolved = await packageImportsResolve(specifier, parent, job, cjsResolve); + resolved = await packageImportsResolve(specifier, parent, job, cjsResolve, filesToEmit); } else { - resolved = await resolvePackage(specifier, parent, job, cjsResolve); + resolved = await resolvePackage(specifier, parent, job, cjsResolve, filesToEmit); } + + let result: string | string[] if (Array.isArray(resolved)) { - return Promise.all(resolved.map(resolved => job.realpath(resolved, parent))) + result = await Promise.all(resolved.map(resolved => job.realpath(resolved, parent, undefined, filesToEmit))) } else if (resolved.startsWith('node:')) { - return resolved; + result = resolved; } else { - return job.realpath(resolved, parent); + result = await job.realpath(resolved, parent, undefined, filesToEmit); } + ;(job as any).resolveCache.set(cacheKey, { + result, + filesToEmit + }) + + await Promise.all(filesToEmit.map(async (item) => { + await job.emitFile(item.file, item.type, item.parent, item.isRealpath) + })) + return result }; -async function resolvePath (path: string, parent: string, job: Job): Promise { - const result = await resolveFile(path, parent, job) || await resolveDir(path, parent, job); +async function resolvePath (path: string, parent: string, job: Job, filesToEmit: FilesToEmit): Promise { + const result = await resolveFile(path, parent, job, filesToEmit) || await resolveDir(path, parent, job, filesToEmit); if (!result) { throw new NotFoundError(path, parent); } return result; } -async function resolveFile (path: string, parent: string, job: Job): Promise { +async function resolveFile (path: string, parent: string, job: Job, filesToEmit: FilesToEmit): Promise { if (path.endsWith('/')) return undefined; - path = await job.realpath(path, parent); + path = await job.realpath(path, parent, undefined, filesToEmit); if (await job.isFile(path)) return path; if (job.ts && path.startsWith(job.base) && path.substr(job.base.length).indexOf(sep + 'node_modules' + sep) === -1 && await job.isFile(path + '.ts')) return path + '.ts'; if (job.ts && path.startsWith(job.base) && path.substr(job.base.length).indexOf(sep + 'node_modules' + sep) === -1 && await job.isFile(path + '.tsx')) return path + '.tsx'; @@ -46,18 +77,18 @@ async function resolveFile (path: string, parent: string, job: Job): Promise { +async function packageImportsResolve (name: string, parent: string, job: Job, cjsResolve: boolean, filesToEmit: FilesToEmit): Promise { if (name !== '#' && !name.startsWith('#/') && job.conditions) { const pjsonBoundary = await job.getPjsonBoundary(parent); if (pjsonBoundary) { @@ -172,11 +203,11 @@ async function packageImportsResolve (name: string, parent: string, job: Job, cj let importsResolved = resolveExportsImports(pjsonBoundary, pkgImports, name, job, true, cjsResolve); if (importsResolved) { if (cjsResolve) - importsResolved = await resolveFile(importsResolved, parent, job) || await resolveDir(importsResolved, parent, job); + importsResolved = await resolveFile(importsResolved, parent, job, filesToEmit) || await resolveDir(importsResolved, parent, job, filesToEmit); else if (!await job.isFile(importsResolved)) throw new NotFoundError(importsResolved, parent); if (importsResolved) { - await job.emitFile(pjsonBoundary + sep + 'package.json', 'resolve', parent); + filesToEmit.push({ file: pjsonBoundary + sep + 'package.json', parent, type: 'resolve' }) return importsResolved; } } @@ -186,7 +217,7 @@ async function packageImportsResolve (name: string, parent: string, job: Job, cj throw new NotFoundError(name, parent); } -async function resolvePackage (name: string, parent: string, job: Job, cjsResolve: boolean): Promise { +async function resolvePackage (name: string, parent: string, job: Job, cjsResolve: boolean, filesToEmit: FilesToEmit): Promise { let packageParent = parent; if (nodeBuiltins.has(name)) return 'node:' + name; @@ -203,12 +234,12 @@ async function resolvePackage (name: string, parent: string, job: Job, cjsResolv selfResolved = resolveExportsImports(pjsonBoundary, pkgExports, '.' + name.slice(pkgName.length), job, false, cjsResolve); if (selfResolved) { if (cjsResolve) - selfResolved = await resolveFile(selfResolved, parent, job) || await resolveDir(selfResolved, parent, job); + selfResolved = await resolveFile(selfResolved, parent, job, filesToEmit) || await resolveDir(selfResolved, parent, job, filesToEmit); else if (!await job.isFile(selfResolved)) throw new NotFoundError(selfResolved, parent); } if (selfResolved) - await job.emitFile(pjsonBoundary + sep + 'package.json', 'resolve', parent); + filesToEmit.push({ file: pjsonBoundary + sep + 'package.json', parent, type: 'resolve' }) } } } @@ -225,16 +256,16 @@ async function resolvePackage (name: string, parent: string, job: Job, cjsResolv if (job.conditions && pkgExports !== undefined && pkgExports !== null && !selfResolved) { let legacyResolved; if (!job.exportsOnly) - legacyResolved = await resolveFile(nodeModulesDir + sep + name, parent, job) || await resolveDir(nodeModulesDir + sep + name, parent, job); + legacyResolved = await resolveFile(nodeModulesDir + sep + name, parent, job, filesToEmit) || await resolveDir(nodeModulesDir + sep + name, parent, job, filesToEmit); let resolved = resolveExportsImports(nodeModulesDir + sep + pkgName, pkgExports, '.' + name.slice(pkgName.length), job, false, cjsResolve); if (resolved) { if (cjsResolve) - resolved = await resolveFile(resolved, parent, job) || await resolveDir(resolved, parent, job); + resolved = await resolveFile(resolved, parent, job, filesToEmit) || await resolveDir(resolved, parent, job, filesToEmit); else if (!await job.isFile(resolved)) throw new NotFoundError(resolved, parent); } if (resolved) { - await job.emitFile(nodeModulesDir + sep + pkgName + sep + 'package.json', 'resolve', parent); + filesToEmit.push({ file: nodeModulesDir + sep + pkgName + sep + 'package.json', parent, type: 'resolve' }) if (legacyResolved && legacyResolved !== resolved) return [resolved, legacyResolved]; return resolved; @@ -243,7 +274,7 @@ async function resolvePackage (name: string, parent: string, job: Job, cjsResolv return legacyResolved; } else { - const resolved = await resolveFile(nodeModulesDir + sep + name, parent, job) || await resolveDir(nodeModulesDir + sep + name, parent, job); + const resolved = await resolveFile(nodeModulesDir + sep + name, parent, job, filesToEmit) || await resolveDir(nodeModulesDir + sep + name, parent, job, filesToEmit); if (resolved) { if (selfResolved && selfResolved !== resolved) return [resolved, selfResolved]; @@ -258,7 +289,7 @@ async function resolvePackage (name: string, parent: string, job: Job, cjsResolv for (const path of Object.keys(job.paths)) { if (path.endsWith('/') && name.startsWith(path)) { const pathTarget = job.paths[path] + name.slice(path.length); - const resolved = await resolveFile(pathTarget, parent, job) || await resolveDir(pathTarget, parent, job); + const resolved = await resolveFile(pathTarget, parent, job, filesToEmit) || await resolveDir(pathTarget, parent, job, filesToEmit); if (!resolved) { throw new NotFoundError(name, parent); } diff --git a/src/utils/sharedlib-emit.ts b/src/utils/sharedlib-emit.ts index b4f70a0c..09e8fd96 100644 --- a/src/utils/sharedlib-emit.ts +++ b/src/utils/sharedlib-emit.ts @@ -17,13 +17,23 @@ switch (os.platform()) { // helper for emitting the associated shared libraries when a binary is emitted export async function sharedLibEmit(path: string, job: Job) { + const cacheItem = (job as any).globCache.get(path) + if (typeof cacheItem !== 'undefined') { + if (Array.isArray(cacheItem)) { + await Promise.all(cacheItem.map(file => job.emitFile(file, 'sharedlib', path))); + } + return + } // console.log('Emitting shared libs for ' + path); const pkgPath = getPackageBase(path); - if (!pkgPath) + if (!pkgPath) { + (job as any).globCache.set(path, null) return; + } const files = await new Promise((resolve, reject) => glob(pkgPath + sharedlibGlob, { ignore: pkgPath + '/**/node_modules/**/*' }, (err, files) => err ? reject(err) : resolve(files)) ); + ;(job as any).globCache.set(path, files) await Promise.all(files.map(file => job.emitFile(file, 'sharedlib', path))); }; diff --git a/test/unit.test.js b/test/unit.test.js index c946320a..43937d26 100644 --- a/test/unit.test.js +++ b/test/unit.test.js @@ -49,52 +49,65 @@ for (const { testName, isRoot } of unitTests) { if (testName === "tsx-input") { inputFileName = "input.tsx"; } - - const { fileList, reasons } = await nodeFileTrace([join(unitPath, inputFileName)], { - base: isRoot ? '/' : `${__dirname}/../`, - processCwd: unitPath, - paths: { - dep: `${__dirname}/../test/unit/esm-paths/esm-dep.js`, - 'dep/': `${__dirname}/../test/unit/esm-paths-trailer/` - }, - exportsOnly: testName.startsWith('exports-only'), - ts: true, - log: true, - // disable analysis for basic-analysis unit tests - analysis: !testName.startsWith('basic-analysis'), - mixedModules: true, - // Ignore unit test output "actual.js", and ignore GitHub Actions preinstalled packages - ignore: (str) => str.endsWith('/actual.js') || str.startsWith('usr/local'), - readFile: readFileMock, - resolve: testName.startsWith('resolve-hook') - ? (id, parent) => `custom-resolution-${id}` - : undefined, - }); - let expected; - try { - expected = JSON.parse(fs.readFileSync(join(unitPath, 'output.js')).toString()); - if (process.platform === 'win32') { - // When using Windows, the expected output should use backslash - expected = expected.map(str => str.replace(/\//g, '\\')); + const nftCache = {} + + const doTrace = async () => { + const { fileList, reasons } = await nodeFileTrace([join(unitPath, inputFileName)], { + base: isRoot ? '/' : `${__dirname}/../`, + processCwd: unitPath, + paths: { + dep: `${__dirname}/../test/unit/esm-paths/esm-dep.js`, + 'dep/': `${__dirname}/../test/unit/esm-paths-trailer/` + }, + cache: nftCache, + exportsOnly: testName.startsWith('exports-only'), + ts: true, + log: true, + // disable analysis for basic-analysis unit tests + analysis: !testName.startsWith('basic-analysis'), + mixedModules: true, + // Ignore unit test output "actual.js", and ignore GitHub Actions preinstalled packages + ignore: (str) => str.endsWith('/actual.js') || str.startsWith('usr/local'), + readFile: readFileMock, + resolve: testName.startsWith('resolve-hook') + ? (id, parent) => `custom-resolution-${id}` + : undefined, + }); + let expected; + try { + expected = JSON.parse(fs.readFileSync(join(unitPath, 'output.js')).toString()); + if (process.platform === 'win32') { + // When using Windows, the expected output should use backslash + expected = expected.map(str => str.replace(/\//g, '\\')); + } + if (isRoot) { + // We set `base: "/"` but we can't hardcode an absolute path because + // CI will look different than a local machine so we fix the path here. + expected = expected.map(str => join(__dirname, '..', str).slice(1)); + } } - if (isRoot) { - // We set `base: "/"` but we can't hardcode an absolute path because - // CI will look different than a local machine so we fix the path here. - expected = expected.map(str => join(__dirname, '..', str).slice(1)); + catch (e) { + console.warn(e); + expected = []; + } + try { + expect(fileList).toEqual(expected); + } + catch (e) { + console.warn(reasons); + fs.writeFileSync(join(unitPath, 'actual.js'), JSON.stringify(fileList, null, 2)); + throw e; } } - catch (e) { - console.warn(e); - expected = []; - } - try { - expect(fileList).toEqual(expected); - } - catch (e) { - console.warn(reasons); - fs.writeFileSync(join(unitPath, 'actual.js'), JSON.stringify(fileList, null, 2)); - throw e; - } + await doTrace() + // test tracing again with a populated nftTrace + expect(nftCache.fileCache).toBeDefined() + expect(nftCache.statCache).toBeDefined() + expect(nftCache.symlinkCache).toBeDefined() + expect(nftCache.analysisCache).toBeDefined() + expect(nftCache.globCache).toBeDefined() + expect(nftCache.resolveCache).toBeDefined() + await doTrace() if (testName === "tsx-input") { expect(readFileMock.mock.calls.length).toBe(2);