From 0a86886f2ee16316dbacc578e50c0cc3c5d6d623 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Tue, 5 Mar 2024 18:29:02 -0800 Subject: [PATCH] [js/test] align test runner flags with ort.env --- js/web/script/test-runner-cli-args.ts | 140 ++++++++++-------- js/web/test/test-main.ts | 44 +----- .../templates/web-browserstack-ci.yml | 5 +- .../azure-pipelines/templates/win-web-ci.yml | 4 +- .../templates/win-web-multi-browsers.yml | 6 +- 5 files changed, 87 insertions(+), 112 deletions(-) diff --git a/js/web/script/test-runner-cli-args.ts b/js/web/script/test-runner-cli-args.ts index ed4dd76a6e31..b2b212bdb9bc 100644 --- a/js/web/script/test-runner-cli-args.ts +++ b/js/web/script/test-runner-cli-args.ts @@ -29,8 +29,10 @@ Options: *** General Options *** -h, --help Print this message. - -d, --debug Specify to run test runner in debug mode. - Debug mode outputs verbose log for test runner, sets up environment debug flag, and keeps karma not to exit after tests completed. + -d, --debug Specify to run test runner in debug mode. Debug mode does the following: + - outputs verbose log for test runner + - sets up environment debug flag (env.debug = true) + - opens Chromium debug port at 9333 and keeps karma not to exit after tests completed. -b=<...>, --backend=<...> Specify one or more backend(s) to run the test upon. Backends can be one or more of the following, splitted by comma: webgl @@ -47,38 +49,55 @@ Options: bs (for BrowserStack tests) -p, --profile Enable profiler. Profiler will generate extra logs which include the information of events time consumption + -t, --trace Enable trace. -P[=<...>], --perf[=<...>] Generate performance number. Cannot be used with flag --debug. This flag can be used with a number as value, specifying the total count of test cases to run. The test cases may be used multiple times. Default value is 10. -c, --file-cache Enable file cache. + +*** Session Options *** + -u=<...>, --optimized-model-file-path=<...> Specify whether to dump the optimized model. + -o=<...>, --graph-optimization-level=<...> Specify graph optimization level. + Default is 'all'. Valid values are 'disabled', 'basic', 'extended', 'all'. -i=<...>, --io-binding=<...> Specify the IO binding testing type. Should be one of the following: - none (default) + none (default) gpu-tensor use pre-allocated GPU tensors for inputs and outputs gpu-location use pre-allocated GPU tensors for inputs and set preferredOutputLocation to 'gpu-buffer' -*** Session Options *** - -u=<...>, --optimized-model-file-path=<...> Specify whether to dump the optimized model. - -o=<...>, --graph-optimization-level=<...> Specify graph optimization level. - Default is 'all'. Valid values are 'disabled', 'basic', 'extended', 'all'. *** Logging Options *** - --log-verbose=<...> Set log level to verbose - --log-info=<...> Set log level to info - --log-warning=<...> Set log level to warning - --log-error=<...> Set log level to error - The 4 flags above specify the logging configuration. Each flag allows to specify one or more category(s), splitted by comma. If use the flags without value, the log level will be applied to all category. + --log-verbose Set log level to verbose + --log-info Set log level to info + --log-warning Set log level to warning + --log-error Set log level to error + The 4 flags above specify the logging configuration. *** Backend Options *** + --wasm.<...>=<...> Set global environment flags for each backend. + --webgl.<...>=<...> These flags can be used multiple times to set multiple flags. For example: + --webgpu.<...>=<...> --webgpu.profiling.mode=default --wasm.numThreads=1 --wasm.simd=false + --webnn.<...>=<...> + + --webnn-device-type Set the WebNN device type (cpu/gpu) + -x, --wasm-number-threads Set the WebAssembly number of threads + ("--wasm-number-threads" is deprecated. use "--wasm.numThreads" or "-x" instead) --wasm-init-timeout Set the timeout for WebAssembly backend initialization, in milliseconds + (deprecated. use "--wasm.initTimeout" instead) --wasm-enable-simd Set whether to enable SIMD + (deprecated. use "--wasm.simd" instead) --wasm-enable-proxy Set whether to enable proxy worker + (deprecated. use "--wasm.proxy" instead) --webgl-context-id Set the WebGL context ID (webgl/webgl2) + (deprecated. use "--webgl.contextId" instead) --webgl-matmul-max-batch-size Set the WebGL matmulMaxBatchSize + (deprecated. use "--webgl.matmulMaxBatchSize" instead) --webgl-texture-cache-mode Set the WebGL texture cache mode (initializerOnly/full) + (deprecated. use "--webgl.textureCacheMode" instead) --webgl-texture-pack-mode Set the WebGL texture pack mode (true/false) + (deprecated. use "--webgl.pack" instead) --webgpu-profiling-mode Set the WebGPU profiling mode (off/default) - --webnn-device-type Set the WebNN device type (cpu/gpu) + (deprecated. use "--webgpu.profiling.mode" instead) *** Browser Options *** @@ -171,7 +190,6 @@ export interface TestRunnerCliArgs { cpuOptions?: InferenceSession.CpuExecutionProviderOption; cudaOptions?: InferenceSession.CudaExecutionProviderOption; - cudaFlags?: Record; wasmOptions?: InferenceSession.WebAssemblyExecutionProviderOption; webglOptions?: InferenceSession.WebGLExecutionProviderOption; webnnOptions?: InferenceSession.WebNNExecutionProviderOption; @@ -260,40 +278,29 @@ function parseCpuOptions(_args: minimist.ParsedArgs): InferenceSession.CpuExecut return {name: 'cpu'}; } -function parseCpuFlags(_args: minimist.ParsedArgs): Record { - return {}; -} - function parseWasmOptions(_args: minimist.ParsedArgs): InferenceSession.WebAssemblyExecutionProviderOption { return {name: 'wasm'}; } function parseWasmFlags(args: minimist.ParsedArgs): Env.WebAssemblyFlags { - const numThreads = args.x || args['wasm-number-threads']; + const wasm = args.wasm || {}; + const numThreads = wasm.numThreads = wasm.numThreads ?? (args.x ?? args['wasm-number-threads']); if (typeof numThreads !== 'undefined' && typeof numThreads !== 'number') { - throw new Error('Flag "x"/"wasm-number-threads" must be a number value'); + throw new Error('Flag "wasm.numThreads"/"x"/"wasm-number-threads" must be a number value'); } - const initTimeout = args['wasm-init-timeout']; + const initTimeout = wasm.initTimeout = wasm.initTimeout ?? args['wasm-init-timeout']; if (typeof initTimeout !== 'undefined' && typeof initTimeout !== 'number') { - throw new Error('Flag "wasm-init-timeout" must be a number value'); - } - let simd = args['wasm-enable-simd']; - if (simd === 'true') { - simd = true; - } else if (simd === 'false') { - simd = false; - } else if (typeof simd !== 'undefined' && typeof simd !== 'boolean') { - throw new Error('Flag "wasm-enable-simd" must be a boolean value'); - } - let proxy = args['wasm-enable-proxy']; - if (proxy === 'true') { - proxy = true; - } else if (proxy === 'false') { - proxy = false; - } else if (typeof proxy !== 'undefined' && typeof proxy !== 'boolean') { - throw new Error('Flag "wasm-enable-proxy" must be a boolean value'); - } - return {numThreads, initTimeout, simd, proxy}; + throw new Error('Flag "wasm.initTimeout"/"wasm-init-timeout" must be a number value'); + } + const simd = wasm.simd = parseBooleanArg(wasm.simd ?? args['wasm-enable-simd']); + if (typeof simd !== 'undefined' && typeof simd !== 'boolean') { + throw new Error('Flag "wasm.simd"/"wasm-enable-simd" must be a boolean value'); + } + const proxy = wasm.proxy = parseBooleanArg(wasm.proxy ?? args['wasm-enable-proxy']); + if (typeof proxy !== 'undefined' && typeof proxy !== 'boolean') { + throw new Error('Flag "wasm.proxy"/"wasm-enable-proxy" must be a boolean value'); + } + return wasm; } function parseWebglOptions(_args: minimist.ParsedArgs): InferenceSession.WebGLExecutionProviderOption { @@ -301,39 +308,43 @@ function parseWebglOptions(_args: minimist.ParsedArgs): InferenceSession.WebGLEx } function parseWebglFlags(args: minimist.ParsedArgs): Partial { - const contextId = args['webgl-context-id']; + const webgl = args.webgl || {}; + const contextId = webgl.contextId = webgl.contextId ?? args['webgl-context-id']; if (contextId !== undefined && contextId !== 'webgl' && contextId !== 'webgl2') { - throw new Error('Flag "webgl-context-id" is invalid'); + throw new Error('Flag "webgl.contextId"/"webgl-context-id" is invalid'); } - const matmulMaxBatchSize = args['webgl-matmul-max-batch-size']; + const matmulMaxBatchSize = webgl.matmulMaxBatchSize = webgl.matmulMaxBatchSize ?? args['webgl-matmul-max-batch-size']; if (matmulMaxBatchSize !== undefined && typeof matmulMaxBatchSize !== 'number') { - throw new Error('Flag "webgl-matmul-max-batch-size" must be a number value'); + throw new Error('Flag "webgl.matmulMaxBatchSize"/"webgl-matmul-max-batch-size" must be a number value'); } - const textureCacheMode = args['webgl-texture-cache-mode']; + const textureCacheMode = webgl.textureCacheMode = webgl.textureCacheMode ?? args['webgl-texture-cache-mode']; if (textureCacheMode !== undefined && textureCacheMode !== 'initializerOnly' && textureCacheMode !== 'full') { - throw new Error('Flag "webgl-texture-cache-mode" is invalid'); + throw new Error('Flag "webgl.textureCacheMode"/"webgl-texture-cache-mode" is invalid'); } - const pack = args['webgl-texture-pack-mode']; + const pack = webgl.pack = parseBooleanArg(webgl.pack ?? args['webgl-texture-pack-mode']); if (pack !== undefined && typeof pack !== 'boolean') { - throw new Error('Flag "webgl-texture-pack-mode" is invalid'); + throw new Error('Flag "webgl.pack"/"webgl-texture-pack-mode" is invalid'); } - const async = args['webgl-async']; + const async = webgl.async = parseBooleanArg(webgl.async ?? args['webgl-async']); if (async !== undefined && typeof async !== 'boolean') { - throw new Error('Flag "webgl-async" is invalid'); + throw new Error('Flag "webgl.async"/"webgl-async" is invalid'); } - return {contextId, matmulMaxBatchSize, textureCacheMode, pack}; + return webgl; } function parseWebgpuFlags(args: minimist.ParsedArgs): Partial { - const profilingMode = args['webgpu-profiling-mode']; + const webgpu = args.webgpu || {}; + const profilingMode = (webgpu.profiling = webgpu.profiling ?? {}).mode = + webgpu?.profiling?.mode ?? webgpu.profilingMode ?? args['webgpu-profiling-mode']; if (profilingMode !== undefined && profilingMode !== 'off' && profilingMode !== 'default') { throw new Error('Flag "webgpu-profiling-mode" is invalid'); } - const validateInputContent = args['webgpu-validate-input-content']; + const validateInputContent = webgpu.validateInputContent = + parseBooleanArg(webgpu.validateInputContent ?? args['webgpu-validate-input-content']); if (validateInputContent !== undefined && typeof validateInputContent !== 'boolean') { throw new Error('Flag "webgpu-validate-input-content" is invalid'); } - return {profilingMode, validateInputContent}; + return webgpu; } function parseWebNNOptions(args: minimist.ParsedArgs): InferenceSession.WebNNExecutionProviderOption { @@ -344,12 +355,11 @@ function parseWebNNOptions(args: minimist.ParsedArgs): InferenceSession.WebNNExe return {name: 'webnn', deviceType}; } -function parseGlobalEnvFlags(args: minimist.ParsedArgs): NonNullable { +function parseGlobalEnvFlags(args: minimist.ParsedArgs) { const wasm = parseWasmFlags(args); const webgl = parseWebglFlags(args); const webgpu = parseWebgpuFlags(args); - const cpuFlags = parseCpuFlags(args); - return {webgl, wasm, webgpu, ...cpuFlags}; + return {webgl, wasm, webgpu}; } export function parseTestRunnerCliArgs(cmdlineArgs: string[]): TestRunnerCliArgs { @@ -394,15 +404,14 @@ export function parseTestRunnerCliArgs(cmdlineArgs: string[]): TestRunnerCliArgs } } - const globalEnvFlags = parseGlobalEnvFlags(args); - // Options: // --log-verbose=<...> // --log-info=<...> // --log-warning=<...> // --log-error=<...> const logConfig = parseLogConfig(args); - globalEnvFlags.logLevel = logConfig[0]?.config.minimalSeverity; + let logLevel = logConfig[0]?.config.minimalSeverity; + // Option: -p, --profile const profile = (args.profile || args.p) ? true : false; if (profile) { @@ -410,9 +419,18 @@ export function parseTestRunnerCliArgs(cmdlineArgs: string[]): TestRunnerCliArgs logConfig.push({category: 'Profiler.node', config: {minimalSeverity: 'verbose'}}); logConfig.push({category: 'Profiler.op', config: {minimalSeverity: 'verbose'}}); logConfig.push({category: 'Profiler.backend', config: {minimalSeverity: 'verbose'}}); - globalEnvFlags.logLevel = 'verbose'; + logLevel = 'verbose'; } + // Option: -t, --trace + const trace = parseBooleanArg(args.trace || args.t, false); + + // Options: + // --wasm.<...>=<...> + // --webgl.<...>=<...> + // --webgpu.<...>=<...> + const globalEnvFlags = {...parseGlobalEnvFlags(args), debug, trace, logLevel}; + // Option: -P[=<...>], --perf[=<...>] const perfArg = (args.perf || args.P); const perf = perfArg ? true : false; diff --git a/js/web/test/test-main.ts b/js/web/test/test-main.ts index 2d83ce1e095c..96e374f87aed 100644 --- a/js/web/test/test-main.ts +++ b/js/web/test/test-main.ts @@ -19,49 +19,7 @@ if (ORT_WEB_TEST_CONFIG.model.some(testGroup => testGroup.tests.some(test => tes } // set flags -const options = ORT_WEB_TEST_CONFIG.options; -if (options.debug !== undefined) { - ort.env.debug = options.debug; -} -if (options.globalEnvFlags) { - const flags = options.globalEnvFlags; - if (flags.logLevel !== undefined) { - ort.env.logLevel = flags.logLevel; - } - if (flags.webgl?.contextId !== undefined) { - ort.env.webgl.contextId = flags.webgl.contextId; - } - if (flags.webgl?.matmulMaxBatchSize !== undefined) { - ort.env.webgl.matmulMaxBatchSize = flags.webgl.matmulMaxBatchSize; - } - if (flags.webgl?.textureCacheMode !== undefined) { - ort.env.webgl.textureCacheMode = flags.webgl.textureCacheMode; - } - if (flags.webgl?.pack !== undefined) { - ort.env.webgl.pack = flags.webgl.pack; - } - if (flags.webgl?.async !== undefined) { - ort.env.webgl.async = flags.webgl.async; - } - if (flags.wasm?.numThreads !== undefined) { - ort.env.wasm.numThreads = flags.wasm.numThreads; - } - if (flags.wasm?.simd !== undefined) { - ort.env.wasm.simd = flags.wasm.simd; - } - if (flags.wasm?.proxy !== undefined) { - ort.env.wasm.proxy = flags.wasm.proxy; - } - if (flags.wasm?.initTimeout !== undefined) { - ort.env.wasm.initTimeout = flags.wasm.initTimeout; - } - if (flags.webgpu?.profilingMode !== undefined) { - ort.env.webgpu.profiling = {mode: flags.webgpu.profilingMode}; - } - if (flags.webgpu?.validateInputContent !== undefined) { - ort.env.webgpu.validateInputContent = flags.webgpu.validateInputContent; - } -} +Object.assign(ort.env, ORT_WEB_TEST_CONFIG.options.globalEnvFlags); // Set logging configuration for (const logConfig of ORT_WEB_TEST_CONFIG.log) { diff --git a/tools/ci_build/github/azure-pipelines/templates/web-browserstack-ci.yml b/tools/ci_build/github/azure-pipelines/templates/web-browserstack-ci.yml index 96e6ff89cd4f..9ab2d3401de4 100644 --- a/tools/ci_build/github/azure-pipelines/templates/web-browserstack-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/web-browserstack-ci.yml @@ -71,7 +71,7 @@ jobs: timeoutInMinutes: 20 - script: | export ORT_WEB_TEST_BS_BROWSERS=BS_MAC_11_Safari_14,BS_MAC_11_Chrome_91,BS_ANDROID_11_Pixel_5 - npm test -- suite0 --env=bs --wasm-init-timeout=30000 --file-cache + npm test -- suite0 -e=bs --wasm.initTimeout=30000 --file-cache workingDirectory: '$(Build.SourcesDirectory)/js/web' displayName: 'npm test (Suite0, BS_ANDROID, BS_MAC)' env: @@ -80,7 +80,7 @@ jobs: continueOnError: true - script: | export ORT_WEB_TEST_BS_BROWSERS=BS_IOS_14_iPhoneXS - npm test -- suite1 --env=bs --wasm-init-timeout=30000 --file-cache --backend=wasm + npm test -- suite1 -e=bs --wasm.initTimeout=30000 --file-cache --backend=wasm workingDirectory: '$(Build.SourcesDirectory)/js/web' displayName: 'npm test (Suite1, BS_IOS)' continueOnError: true @@ -95,4 +95,3 @@ jobs: - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 displayName: 'Clean Agent Directories' condition: always() - diff --git a/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml index 9553bc1bc354..1eb2ee6f6409 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml @@ -173,11 +173,11 @@ jobs: displayName: 'Run ort-web tests (Suite1, webgpu, IO-binding=gpu-location)' condition: eq('${{ parameters.RunWebGpuTests }}', 'true') - script: | - npm test -- --webgl-texture-pack-mode -b=webgl -e=chrome --karma-debug + npm test -- --webgl.pack -b=webgl -e=chrome --karma-debug workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'Run ort-web tests - WebGL: packed mode' - script: | - npm test -- --wasm-enable-proxy -b=wasm -e=chrome --karma-debug + npm test -- --wasm.proxy -b=wasm -e=chrome --karma-debug workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'Run ort-web tests - WebAssembly: proxy' condition: and(succeeded(), eq('${{ parameters.BuildConfig }}', 'Release')) diff --git a/tools/ci_build/github/azure-pipelines/templates/win-web-multi-browsers.yml b/tools/ci_build/github/azure-pipelines/templates/win-web-multi-browsers.yml index 31ee488318a0..79bf0b5e7136 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-web-multi-browsers.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-web-multi-browsers.yml @@ -68,15 +68,15 @@ jobs: workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'npm ci /js/web/' - script: | - npm test -- suite0 -b=wasm,webgl --wasm-init-timeout=30000 --file-cache + npm test -- suite0 -b=wasm,webgl --wasm.initTimeout=30000 --file-cache workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'npm test (Suite0, Chrome)' - script: | - npm test -- suite0 -b=wasm,webgl --env=firefox --wasm-init-timeout=30000 --file-cache + npm test -- suite0 -b=wasm,webgl -e=firefox --wasm.initTimeout=30000 --file-cache workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'npm test (Suite0, Firefox)' - script: | - npm test -- suite0 -b=wasm,webgl --env=edge --wasm-init-timeout=30000 --file-cache + npm test -- suite0 -b=wasm,webgl -e=edge --wasm.initTimeout=30000 --file-cache workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'npm test (Suite0, Edge)' - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3