Skip to content

Commit

Permalink
fix(vm): support network imports (#5610)
Browse files Browse the repository at this point in the history
  • Loading branch information
sheremet-va authored Apr 30, 2024
1 parent cc8f058 commit 103a600
Show file tree
Hide file tree
Showing 5 changed files with 88 additions and 29 deletions.
50 changes: 35 additions & 15 deletions packages/vitest/src/runtime/external-executor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ export interface ExternalModulesExecutorOptions {
}

interface ModuleInformation {
type: 'data' | 'builtin' | 'vite' | 'wasm' | 'module' | 'commonjs'
type: 'data' | 'builtin' | 'vite' | 'wasm' | 'module' | 'commonjs' | 'network'
url: string
path: string
}
Expand All @@ -41,6 +41,8 @@ export class ExternalModulesExecutor {
private fs: FileMap
private resolvers: ((id: string, parent: string) => string | undefined)[] = []

#networkSupported: boolean | null = null

constructor(private options: ExternalModulesExecutorOptions) {
this.context = options.context

Expand All @@ -62,6 +64,20 @@ export class ExternalModulesExecutor {
this.resolvers = [this.vite.resolve]
}

async import(identifier: string) {
const module = await this.createModule(identifier)
await this.esm.evaluateModule(module)
return module.namespace
}

require(identifier: string) {
return this.cjs.require(identifier)
}

createRequire(identifier: string) {
return this.cjs.createRequire(identifier)
}

// dynamic import can be used in both ESM and CJS, so we have it in the executor
public importModuleDynamically = async (specifier: string, referencer: VMModule) => {
const module = await this.resolveModule(specifier, referencer.identifier)
Expand Down Expand Up @@ -161,6 +177,9 @@ export class ExternalModulesExecutor {
if (extension === '.node' || isNodeBuiltin(identifier))
return { type: 'builtin', url: identifier, path: identifier }

if (this.isNetworkSupported && (identifier.startsWith('http:') || identifier.startsWith('https:')))
return { type: 'network', url: identifier, path: identifier }

const isFileUrl = identifier.startsWith('file://')
const pathUrl = isFileUrl ? fileURLToPath(identifier.split('?')[0]) : identifier
const fileUrl = isFileUrl ? identifier : pathToFileURL(pathUrl).toString()
Expand Down Expand Up @@ -209,31 +228,32 @@ export class ExternalModulesExecutor {
case 'vite':
return await this.vite.createViteModule(url)
case 'wasm':
return await this.esm.createWebAssemblyModule(url, this.fs.readBuffer(path))
return await this.esm.createWebAssemblyModule(url, () => this.fs.readBuffer(path))
case 'module':
return await this.esm.createEsModule(url, this.fs.readFile(path))
return await this.esm.createEsModule(url, () => this.fs.readFile(path))
case 'commonjs': {
const exports = this.require(path)
return this.wrapCommonJsSynteticModule(identifier, exports)
}
case 'network': {
return this.esm.createNetworkModule(url)
}
default: {
const _deadend: never = type
return _deadend
}
}
}

async import(identifier: string) {
const module = await this.createModule(identifier)
await this.esm.evaluateModule(module)
return module.namespace
}

require(identifier: string) {
return this.cjs.require(identifier)
}

createRequire(identifier: string) {
return this.cjs.createRequire(identifier)
private get isNetworkSupported() {
if (this.#networkSupported == null) {
if (process.execArgv.includes('--experimental-network-imports'))
this.#networkSupported = true
else if (process.env.NODE_OPTIONS?.includes('--experimental-network-imports'))
this.#networkSupported = true
else
this.#networkSupported = false
}
return this.#networkSupported
}
}
43 changes: 39 additions & 4 deletions packages/vitest/src/runtime/vm/esm-executor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ export class EsmExecutor {
private esmLinkMap = new WeakMap<VMModule, Promise<void>>()
private context: vm.Context

#httpIp = IPnumber('127.0.0.0')

constructor(private executor: ExternalModulesExecutor, options: EsmExecutorOptions) {
this.context = options.context
}
Expand All @@ -38,10 +40,11 @@ export class EsmExecutor {
return m
}

public async createEsModule(fileUrl: string, code: string) {
public async createEsModule(fileUrl: string, getCode: () => Promise<string> | string) {
const cached = this.moduleCache.get(fileUrl)
if (cached)
return cached
const code = await getCode()
// TODO: should not be allowed in strict mode, implement in #2854
if (fileUrl.endsWith('.json')) {
const m = new SyntheticModule(
Expand Down Expand Up @@ -77,15 +80,35 @@ export class EsmExecutor {
return m
}

public async createWebAssemblyModule(fileUrl: string, code: Buffer) {
public async createWebAssemblyModule(fileUrl: string, getCode: () => Buffer) {
const cached = this.moduleCache.get(fileUrl)
if (cached)
return cached
const m = this.loadWebAssemblyModule(code, fileUrl)
const m = this.loadWebAssemblyModule(getCode(), fileUrl)
this.moduleCache.set(fileUrl, m)
return m
}

public async createNetworkModule(fileUrl: string) {
// https://nodejs.org/api/esm.html#https-and-http-imports
if (fileUrl.startsWith('http:')) {
const url = new URL(fileUrl)
if (
url.hostname !== 'localhost'
&& url.hostname !== '::1'
&& (IPnumber(url.hostname) & IPmask(8)) !== this.#httpIp
) {
throw new Error(
// we don't know the importer, so it's undefined (the same happens in --pool=threads)
`import of '${fileUrl}' by undefined is not supported: `
+ 'http can only be used to load local resources (use https instead).',
)
}
}

return this.createEsModule(fileUrl, () => fetch(fileUrl).then(r => r.text()))
}

public async loadWebAssemblyModule(source: Buffer, identifier: string) {
const cached = this.moduleCache.get(identifier)
if (cached)
Expand Down Expand Up @@ -187,6 +210,18 @@ export class EsmExecutor {
return module
}

return this.createEsModule(identifier, code)
return this.createEsModule(identifier, () => code)
}
}

function IPnumber(address: string) {
const ip = address.match(/^(\d+)\.(\d+)\.(\d+)\.(\d+)$/)
if (ip)
return (+ip[1] << 24) + (+ip[2] << 16) + (+ip[3] << 8) + (+ip[4])

throw new Error(`Expected IP address, received ${address}`)
}

function IPmask(maskSize: number) {
return -1 << (32 - maskSize)
}
10 changes: 6 additions & 4 deletions packages/vitest/src/runtime/vm/vite-executor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,12 @@ export class ViteExecutor {
const cached = this.esm.resolveCachedModule(fileUrl)
if (cached)
return cached
const result = await this.options.transform(fileUrl, 'web')
if (!result.code)
throw new Error(`[vitest] Failed to transform ${fileUrl}. Does the file exist?`)
return this.esm.createEsModule(fileUrl, result.code)
return this.esm.createEsModule(fileUrl, async () => {
const result = await this.options.transform(fileUrl, 'web')
if (!result.code)
throw new Error(`[vitest] Failed to transform ${fileUrl}. Does the file exist?`)
return result.code
})
}

private createViteClientModule() {
Expand Down
7 changes: 7 additions & 0 deletions test/cli/fixtures/network-imports/basic.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,10 @@ import slash from 'http://localhost:9602/slash@3.0.0.js'
test('network imports', () => {
expect(slash('foo\\bar')).toBe('foo/bar')
})

test('doesn\'t work for http outside localhost', async () => {
// @ts-expect-error network imports
await expect(() => import('http://100.0.0.0/')).rejects.toThrowError(
'import of \'http://100.0.0.0/\' by undefined is not supported: http can only be used to load local resources (use https instead).',
)
})
7 changes: 1 addition & 6 deletions test/cli/test/network-imports.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,6 @@ const config = {
forks: {
execArgv: ['--experimental-network-imports'],
},
// not supported?
// FAIL test/basic.test.ts [ test/basic.test.ts ]
// Error: ENOENT: no such file or directory, open 'http://localhost:9602/slash@3.0.0.js'
// ❯ Object.openSync node:fs:596:3
// ❯ readFileSync node:fs:464:35
vmThreads: {
execArgv: ['--experimental-network-imports'],
},
Expand All @@ -25,7 +20,7 @@ const config = {
it.each([
'threads',
'forks',
// 'vmThreads',
'vmThreads',
])('importing from network in %s', async (pool) => {
const { stderr, exitCode } = await runVitest({
...config,
Expand Down

0 comments on commit 103a600

Please sign in to comment.