Add additional items to shared cache

jest.config.js

@@ -4,7 +4,7 @@ module.exports = {
   coverageThreshold: {
     global: {
       branches: 80.5,
-      functions: 95.28,
+      functions: 95.24,
       lines: 85.87,
       statements: -249
     }

prepare-install.js

@@ -7,7 +7,7 @@ if (isWin) {
   const pkgJson = readFileSync(join(__dirname, 'package.json'), 'utf8');
   const pkg = JSON.parse(pkgJson);
 
-  unlinkSync(join(__dirname, 'yarn.lock'));
+  // unlinkSync(join(__dirname, 'yarn.lock'));
   // Delete the integration tests that will never work in Windows
   unlinkSync(join(__dirname, 'test', 'integration', 'tensorflow.js'));
   unlinkSync(join(__dirname, 'test', 'integration', 'argon2.js'));

src/node-file-trace.ts

@@ -3,7 +3,7 @@ import { basename, dirname, extname, relative, resolve, sep } from 'path';
 import fs from 'fs';
 import { promisify } from 'util'
 import analyze, { AnalyzeResult } from './analyze';
-import resolveDependency from './resolve-dependency';
+import resolveDependency, { FilesToEmit } from './resolve-dependency';
 import { isMatch } from 'micromatch';
 import { sharedLibEmit } from './utils/sharedlib-emit';
 import { join } from 'path';
@@ -67,6 +67,9 @@ export class Job {
   private statCache: Map<string, Stats | null>;
   private symlinkCache: Map<string, string | null>;
   private analysisCache: Map<string, AnalyzeResult>;
+  private globCache: Map<string, string[] | null>;
+  private pjsonBoundaryCache: Map<string, string | null>;
+  private resolveCache: Map<string, { result: string | string[], toEmit: [] }>;
   public fileList: Set<string>;
   public esmFileList: Set<string>;
   public processed: Set<string>;
@@ -142,12 +145,18 @@ export class Job {
     this.statCache = cache && cache.statCache || new Map();
     this.symlinkCache = cache && cache.symlinkCache || new Map();
     this.analysisCache = cache && cache.analysisCache || new Map();
+    this.globCache = cache && cache.globCache || new Map();
+    this.pjsonBoundaryCache = cache && cache.pjsonBoundaryCache || new Map();
+    this.resolveCache = cache && cache.resolveCache || new Map();
 
     if (cache) {
       cache.fileCache = this.fileCache;
       cache.statCache = this.statCache;
       cache.symlinkCache = this.symlinkCache;
       cache.analysisCache = this.analysisCache;
+      cache.globCache = this.globCache;
+      cache.pjsonBoundaryCache = this.pjsonBoundaryCache;
+      cache.resolveCache = this.resolveCache;
     }
 
     this.fileList = new Set();
@@ -228,18 +237,23 @@ export class Job {
     }
   }
 
-  async realpath (path: string, parent?: string, seen = new Set()): Promise<string> {
+  async realpath (path: string, parent?: string, seen = new Set(), filesToEmit ?: FilesToEmit): Promise<string> {
     if (seen.has(path)) throw new Error('Recursive symlink detected resolving ' + path);
     seen.add(path);
     const symlink = await this.readlink(path);
     // emit direct symlink paths only
     if (symlink) {
       const parentPath = dirname(path);
       const resolved = resolve(parentPath, symlink);
-      const realParent = await this.realpath(parentPath, parent);
-      if (inPath(path, realParent))
-        await this.emitFile(path, 'resolve', parent, true);
-      return this.realpath(resolved, parent, seen);
+      const realParent = await this.realpath(parentPath, parent, undefined, filesToEmit);
+      if (inPath(path, realParent)) {
+        if (filesToEmit) {
+          filesToEmit.push({file: path, type: 'resolve', parent: parent!, isRealpath: true})
+        } else {
+          await this.emitFile(path, 'resolve', parent, true);
+        }
+      }
+      return this.realpath(resolved, parent, seen, filesToEmit);
     }
     // keep backtracking for realpath, emitting folder symlinks within base
     if (!inPath(path, this.base))
@@ -270,13 +284,20 @@ export class Job {
   }
 
   async getPjsonBoundary (path: string) {
+    const initialPath = path
+    const cacheItem = this.pjsonBoundaryCache.get(path)
+    if (typeof cacheItem !== 'undefined') return cacheItem
+
     const rootSeparatorIndex = path.indexOf(sep);
     let separatorIndex: number;
     while ((separatorIndex = path.lastIndexOf(sep)) > rootSeparatorIndex) {
       path = path.substr(0, separatorIndex);
-      if (await this.isFile(path + sep + 'package.json'))
+      if (await this.isFile(path + sep + 'package.json')) {
+        this.pjsonBoundaryCache.set(initialPath, path)
         return path;
+      }
     }
+    this.pjsonBoundaryCache.set(initialPath, null)
     return undefined;
   }
 

src/resolve-dependency.ts

@@ -1,42 +1,72 @@
 import { isAbsolute, resolve, sep } from 'path';
 import { Job } from './node-file-trace';
 
+export type FilesToEmit = Array<{
+  file: string,
+  type: string,
+  parent?: string,
+  isRealpath?: boolean
+}>
+
 // node resolver
 // custom implementation to emit only needed package.json files for resolver
 // (package.json files are emitted as they are hit)
 export default async function resolveDependency (specifier: string, parent: string, job: Job, cjsResolve = true): Promise<string | string[]> {
+  const cacheKey = specifier + parent
+  const cacheItem = (job as any).resolveCache.get(cacheKey)
+
+  if (cacheItem) {
+    await Promise.all((cacheItem.filesToEmit as FilesToEmit).map(async (item) => {
+      await job.emitFile(item.file, item.type, item.parent, item.isRealpath)
+    }))
+    return cacheItem.result
+  }
+
+  const filesToEmit: FilesToEmit = []
+
   let resolved: string | string[];
   if (isAbsolute(specifier) || specifier === '.' || specifier === '..' || specifier.startsWith('./') || specifier.startsWith('../')) {
     const trailingSlash = specifier.endsWith('/');
-    resolved = await resolvePath(resolve(parent, '..', specifier) + (trailingSlash ? '/' : ''), parent, job);
+    resolved = await resolvePath(resolve(parent, '..', specifier) + (trailingSlash ? '/' : ''), parent, job, filesToEmit);
   }
   else if (specifier[0] === '#') {
-    resolved = await packageImportsResolve(specifier, parent, job, cjsResolve);
+    resolved = await packageImportsResolve(specifier, parent, job, cjsResolve, filesToEmit);
   }
   else {
-    resolved = await resolvePackage(specifier, parent, job, cjsResolve);
+    resolved = await resolvePackage(specifier, parent, job, cjsResolve, filesToEmit);
   }
+
+  let result: string | string[]
 
   if (Array.isArray(resolved)) {
-    return Promise.all(resolved.map(resolved => job.realpath(resolved, parent)))
+    result = await Promise.all(resolved.map(resolved => job.realpath(resolved, parent, undefined, filesToEmit)))
   } else if (resolved.startsWith('node:')) {
-    return resolved;
+    result = resolved;
   } else {
-    return job.realpath(resolved, parent);
+    result = await job.realpath(resolved, parent, undefined, filesToEmit);
   }
+  ;(job as any).resolveCache.set(cacheKey, {
+    result,
+    filesToEmit
+  })
+
+  await Promise.all(filesToEmit.map(async (item) => {
+    await job.emitFile(item.file, item.type, item.parent, item.isRealpath)
+  }))
+  return result
 };
 
-async function resolvePath (path: string, parent: string, job: Job): Promise<string> {
-  const result = await resolveFile(path, parent, job) || await resolveDir(path, parent, job);
+async function resolvePath (path: string, parent: string, job: Job, filesToEmit: FilesToEmit): Promise<string> {
+  const result = await resolveFile(path, parent, job, filesToEmit) || await resolveDir(path, parent, job, filesToEmit);
   if (!result) {
     throw new NotFoundError(path, parent);
   }
   return result;
 }
 
-async function resolveFile (path: string, parent: string, job: Job): Promise<string | undefined> {
+async function resolveFile (path: string, parent: string, job: Job, filesToEmit: FilesToEmit): Promise<string | undefined> {
   if (path.endsWith('/')) return undefined;
-  path = await job.realpath(path, parent);
+  path = await job.realpath(path, parent, undefined, filesToEmit);
   if (await job.isFile(path)) return path;
   if (job.ts && path.startsWith(job.base) && path.substr(job.base.length).indexOf(sep + 'node_modules' + sep) === -1 && await job.isFile(path + '.ts')) return path + '.ts';
   if (job.ts && path.startsWith(job.base) && path.substr(job.base.length).indexOf(sep + 'node_modules' + sep) === -1 && await job.isFile(path + '.tsx')) return path + '.tsx';
@@ -46,18 +76,18 @@ async function resolveFile (path: string, parent: string, job: Job): Promise<str
   return undefined;
 }
 
-async function resolveDir (path: string, parent: string, job: Job) {
+async function resolveDir (path: string, parent: string, job: Job, filesToEmit: FilesToEmit) {
   if (path.endsWith('/')) path = path.slice(0, -1);
   if (!await job.isDir(path)) return;
   const pkgCfg = await getPkgCfg(path, job);
   if (pkgCfg && typeof pkgCfg.main === 'string') {
-    const resolved = await resolveFile(resolve(path, pkgCfg.main), parent, job) || await resolveFile(resolve(path, pkgCfg.main, 'index'), parent, job);
+    const resolved = await resolveFile(resolve(path, pkgCfg.main), parent, job, filesToEmit) || await resolveFile(resolve(path, pkgCfg.main, 'index'), parent, job, filesToEmit);
     if (resolved) {
-      await job.emitFile(path + sep + 'package.json', 'resolve', parent);
+      filesToEmit.push({file: path + sep + 'package.json', parent, type: 'resolve'})
       return resolved;
     }
   }
-  return resolveFile(resolve(path, 'index'), parent, job);
+  return resolveFile(resolve(path, 'index'), parent, job, filesToEmit);
 }
 
 class NotFoundError extends Error {
@@ -162,7 +192,7 @@ function resolveExportsImports (pkgPath: string, obj: PackageTarget, subpath: st
   return undefined;
 }
 
-async function packageImportsResolve (name: string, parent: string, job: Job, cjsResolve: boolean): Promise<string> {
+async function packageImportsResolve (name: string, parent: string, job: Job, cjsResolve: boolean, filesToEmit: FilesToEmit): Promise<string> {
   if (name !== '#' && !name.startsWith('#/') && job.conditions) {
     const pjsonBoundary = await job.getPjsonBoundary(parent);
     if (pjsonBoundary) {
@@ -172,11 +202,11 @@ async function packageImportsResolve (name: string, parent: string, job: Job, cj
         let importsResolved = resolveExportsImports(pjsonBoundary, pkgImports, name, job, true, cjsResolve);
         if (importsResolved) {
           if (cjsResolve)
-            importsResolved = await resolveFile(importsResolved, parent, job) || await resolveDir(importsResolved, parent, job);
+            importsResolved = await resolveFile(importsResolved, parent, job, filesToEmit) || await resolveDir(importsResolved, parent, job, filesToEmit);
           else if (!await job.isFile(importsResolved))
             throw new NotFoundError(importsResolved, parent);
           if (importsResolved) {
-            await job.emitFile(pjsonBoundary + sep + 'package.json', 'resolve', parent);
+            filesToEmit.push({ file: pjsonBoundary + sep + 'package.json', parent, type: 'resolve' })
             return importsResolved;
           }
         }
@@ -186,7 +216,7 @@ async function packageImportsResolve (name: string, parent: string, job: Job, cj
   throw new NotFoundError(name, parent);
 }
 
-async function resolvePackage (name: string, parent: string, job: Job, cjsResolve: boolean): Promise<string | string []> {
+async function resolvePackage (name: string, parent: string, job: Job, cjsResolve: boolean, filesToEmit: FilesToEmit): Promise<string | string []> {
   let packageParent = parent;
   if (nodeBuiltins.has(name)) return 'node:' + name;
 
@@ -203,12 +233,12 @@ async function resolvePackage (name: string, parent: string, job: Job, cjsResolv
         selfResolved = resolveExportsImports(pjsonBoundary, pkgExports, '.' + name.slice(pkgName.length), job, false, cjsResolve);
         if (selfResolved) {
           if (cjsResolve)
-            selfResolved = await resolveFile(selfResolved, parent, job) || await resolveDir(selfResolved, parent, job);
+            selfResolved = await resolveFile(selfResolved, parent, job, filesToEmit) || await resolveDir(selfResolved, parent, job, filesToEmit);
           else if (!await job.isFile(selfResolved))
             throw new NotFoundError(selfResolved, parent);
         }
         if (selfResolved)
-          await job.emitFile(pjsonBoundary + sep + 'package.json', 'resolve', parent);
+          filesToEmit.push({ file: pjsonBoundary + sep + 'package.json', parent, type: 'resolve' })
       }
     }
   }
@@ -225,16 +255,16 @@ async function resolvePackage (name: string, parent: string, job: Job, cjsResolv
     if (job.conditions && pkgExports !== undefined && pkgExports !== null && !selfResolved) {
       let legacyResolved;
       if (!job.exportsOnly)
-        legacyResolved = await resolveFile(nodeModulesDir + sep + name, parent, job) || await resolveDir(nodeModulesDir + sep + name, parent, job);
+        legacyResolved = await resolveFile(nodeModulesDir + sep + name, parent, job, filesToEmit) || await resolveDir(nodeModulesDir + sep + name, parent, job, filesToEmit);
       let resolved = resolveExportsImports(nodeModulesDir + sep + pkgName, pkgExports, '.' + name.slice(pkgName.length), job, false, cjsResolve);
       if (resolved) {
         if (cjsResolve)
-          resolved = await resolveFile(resolved, parent, job) || await resolveDir(resolved, parent, job);
+          resolved = await resolveFile(resolved, parent, job, filesToEmit) || await resolveDir(resolved, parent, job, filesToEmit);
         else if (!await job.isFile(resolved))
           throw new NotFoundError(resolved, parent);
       }
       if (resolved) {
-        await job.emitFile(nodeModulesDir + sep + pkgName + sep + 'package.json', 'resolve', parent);
+        filesToEmit.push({ file: nodeModulesDir + sep + pkgName + sep + 'package.json', parent, type: 'resolve' })
         if (legacyResolved && legacyResolved !== resolved)
           return [resolved, legacyResolved];
         return resolved;
@@ -243,7 +273,7 @@ async function resolvePackage (name: string, parent: string, job: Job, cjsResolv
         return legacyResolved;
     }
     else {
-      const resolved = await resolveFile(nodeModulesDir + sep + name, parent, job) || await resolveDir(nodeModulesDir + sep + name, parent, job);
+      const resolved = await resolveFile(nodeModulesDir + sep + name, parent, job, filesToEmit) || await resolveDir(nodeModulesDir + sep + name, parent, job, filesToEmit);
       if (resolved) {
         if (selfResolved && selfResolved !== resolved)
           return [resolved, selfResolved];
@@ -258,7 +288,7 @@ async function resolvePackage (name: string, parent: string, job: Job, cjsResolv
   for (const path of Object.keys(job.paths)) {
     if (path.endsWith('/') && name.startsWith(path)) {
       const pathTarget = job.paths[path] + name.slice(path.length);
-      const resolved = await resolveFile(pathTarget, parent, job) || await resolveDir(pathTarget, parent, job);
+      const resolved = await resolveFile(pathTarget, parent, job, filesToEmit) || await resolveDir(pathTarget, parent, job, filesToEmit);
       if (!resolved) {
         throw new NotFoundError(name, parent);
       }

src/utils/sharedlib-emit.ts

@@ -17,13 +17,23 @@ switch (os.platform()) {
 
 // helper for emitting the associated shared libraries when a binary is emitted
 export async function sharedLibEmit(path: string, job: Job) {
+  const cacheItem = (job as any).globCache.get(path)
+  if (typeof cacheItem !== 'undefined') {
+    if (Array.isArray(cacheItem)) {
+      await Promise.all(cacheItem.map(file => job.emitFile(file, 'sharedlib', path)));
+    }
+    return
+  }
   // console.log('Emitting shared libs for ' + path);
   const pkgPath = getPackageBase(path);
-  if (!pkgPath)
+  if (!pkgPath) {
+    (job as any).globCache.set(path, null)
     return;
+  }
 
   const files = await new Promise<string[]>((resolve, reject) =>
     glob(pkgPath + sharedlibGlob, { ignore: pkgPath + '/**/node_modules/**/*' }, (err, files) => err ? reject(err) : resolve(files))
   );
+  ;(job as any).globCache.set(path, files)
   await Promise.all(files.map(file => job.emitFile(file, 'sharedlib', path)));
 };