Skip to content

Commit

Permalink
feat(vitest): run typecheck during tests (#4324)
Browse files Browse the repository at this point in the history
  • Loading branch information
sheremet-va authored Oct 19, 2023
1 parent df4606d commit a1aadd7
Show file tree
Hide file tree
Showing 32 changed files with 330 additions and 148 deletions.
5 changes: 0 additions & 5 deletions docs/advanced/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@ Vitest instance requires the current test mode. It can be either:
- `test` when running runtime tests
- `benchmark` when running benchmarks
- `typecheck` when running type tests
### mode
Expand All @@ -64,10 +63,6 @@ Test mode will only call functions inside `test` or `it`, and throws an error wh
Benchmark mode calls `bench` functions and throws an error, when it encounters `test` or `it`. This mode uses `benchmark.include` and `benchmark.exclude` options in the config to find benchmark files.
#### typecheck
Typecheck mode doesn't _run_ tests. It only analyses types and gives a summary. This mode uses `typecheck.include` and `typecheck.exclude` options in the config to find files to analyze.
### start
You can start running tests or benchmarks with `start` method. You can pass an array of strings to filter test files.
20 changes: 19 additions & 1 deletion docs/config/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -547,7 +547,7 @@ export default defineConfig({

### poolMatchGlobs

- **Type:** `[string, 'threads' | 'forks' | 'vmThreads'][]`
- **Type:** `[string, 'threads' | 'forks' | 'vmThreads' | 'typescript'][]`
- **Default:** `[]`
- **Version:** Since Vitest 0.29.4

Expand Down Expand Up @@ -1637,6 +1637,24 @@ Changes the order in which setup files are executed.

Options for configuring [typechecking](/guide/testing-types) test environment.

#### typecheck.enabled

- **Type**: `boolean`
- **Default**: `false`
- **CLI**: `--typecheck`, `--typecheck.enabled`
- **Version**: Since Vitest 1.0.0-beta.3

Enable typechecking alongside your regular tests.

#### typecheck.only

- **Type**: `boolean`
- **Default**: `false`
- **CLI**: `--typecheck.only`
- **Version**: Since Vitest 1.0.0-beta.3

Run only typecheck tests, when typechecking is enabled. When using CLI, this option will automatically enable typechecking.

#### typecheck.checker

- **Type**: `'tsc' | 'vue-tsc' | string`
Expand Down
3 changes: 3 additions & 0 deletions docs/guide/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,9 @@ Run only [benchmark](https://vitest.dev/guide/features.html#benchmarking-experim
| `--inspect-brk` | Enables Node.js inspector with break |
| `--bail <number>` | Stop test execution when given number of tests have failed |
| `--retry <times>` | Retry the test specific number of times if it fails |
| `--typecheck [options]` | Custom options for typecheck pool. If passed without options, enables typechecking |
| `--typecheck.enabled` | Enable typechecking alongside tests (default: `false`) |
| `--typecheck.only` | Run only typecheck tests. This automatically enables typecheck (default: `false`) |
| `-h, --help` | Display available CLI options |

::: tip
Expand Down
10 changes: 5 additions & 5 deletions docs/guide/testing-types.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,12 +66,12 @@ assertType<string>(answr) //

## Run typechecking

Add this command to your `scripts` section in `package.json`:
To enabled typechecking, just add `--typecheck` flag to your Vitest command in `package.json`:

```json
{
"scripts": {
"typecheck": "vitest typecheck"
"test": "vitest --typecheck"
}
}
```
Expand All @@ -80,13 +80,13 @@ Now you can run typecheck:

```sh
# npm
npm run typecheck
npm run test

# yarn
yarn typecheck
yarn test

# pnpm
pnpm run typecheck
pnpm run test
```

Vitest uses `tsc --noEmit` or `vue-tsc --noEmit`, depending on your configuration, so you can remove these scripts from your pipeline.
4 changes: 2 additions & 2 deletions packages/runner/src/utils/tasks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ function isAtomTest(s: Task): s is Test | Custom {

export function getTests(suite: Arrayable<Task>): (Test | Custom)[] {
const tests: (Test | Custom)[] = []
const suite_arr = toArray(suite)
for (const s of suite_arr) {
const arraySuites = toArray(suite)
for (const s of arraySuites) {
if (isAtomTest(s)) {
tests.push(s)
}
Expand Down
1 change: 1 addition & 0 deletions packages/ui/client/components/Suites.vue
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ async function onRunCurrent() {
<TasksList :tasks="current.tasks" :nested="true">
<template #header>
<StatusIcon mx-1 :task="current" />
<div v-if="current.type === 'suite' && current.meta.typecheck" i-logos:typescript-icon flex-shrink-0 mr-1 />
<span data-testid="filenames" font-bold text-sm flex-auto ws-nowrap overflow-hidden truncate>{{ name }}</span>
<div class="flex text-lg">
<IconButton
Expand Down
1 change: 1 addition & 0 deletions packages/ui/client/components/TaskItem.vue
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ const duration = computed(() => {
hover="bg-active"
>
<StatusIcon :task="task" mr-2 />
<div v-if="task.type === 'suite' && task.meta.typecheck" i-logos:typescript-icon flex-shrink-0 mr-2 />
<div flex items-end gap-2 :text="task?.result?.state === 'fail' ? 'red-500' : ''">
<span text-sm truncate font-light>{{ task.name }}</span>
<span v-if="typeof duration === 'number'" text="xs" op20 style="white-space: nowrap">
Expand Down
2 changes: 1 addition & 1 deletion packages/ui/client/components/views/ViewEditor.vue
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ function createErrorElement(e: ErrorWithDiff) {
div.className = 'op80 flex gap-x-2 items-center'
const pre = document.createElement('pre')
pre.className = 'c-red-600 dark:c-red-400'
pre.textContent = `${' '.repeat(stack.column)}^ ${e?.nameStr}: ${e?.message}`
pre.textContent = `${' '.repeat(stack.column)}^ ${e?.nameStr || e.name}: ${e?.message || ''}`
div.appendChild(pre)
const span = document.createElement('span')
span.className = 'i-carbon-launch c-red-600 dark:c-red-400 hover:cursor-pointer min-w-1em min-h-1em'
Expand Down
27 changes: 21 additions & 6 deletions packages/ui/client/composables/summary.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { hasFailedSnapshot } from '@vitest/ws-client'
import type { Benchmark, Task, Test, TypeCheck } from 'vitest/src'
import type { Custom, Task, Test } from 'vitest/src'
import { files, testRunState } from '~/composables/client'

type Nullable<T> = T | null | undefined
Expand Down Expand Up @@ -33,7 +33,7 @@ export const testsSkipped = computed(() => testsIgnore.value.filter(f => f.mode
export const testsTodo = computed(() => testsIgnore.value.filter(f => f.mode === 'todo'))
export const totalTests = computed(() => testsFailed.value.length + testsSuccess.value.length)
export const time = computed(() => {
const t = getTests(tests.value).reduce((acc, t) => {
const t = files.value.reduce((acc, t) => {
acc += Math.max(0, t.collectDuration || 0)
acc += Math.max(0, t.setupDuration || 0)
acc += Math.max(0, t.result?.duration || 0)
Expand All @@ -53,10 +53,25 @@ function toArray<T>(array?: Nullable<Arrayable<T>>): Array<T> {
return [array]
}

function isAtomTest(s: Task): s is Test | Benchmark | TypeCheck {
return (s.type === 'test' || s.type === 'benchmark' || s.type === 'typecheck')
function isAtomTest(s: Task): s is Test | Custom {
return (s.type === 'test' || s.type === 'custom')
}

function getTests(suite: Arrayable<Task>): (Test | Benchmark | TypeCheck)[] {
return toArray(suite).flatMap(s => isAtomTest(s) ? [s] : s.tasks.flatMap(c => isAtomTest(c) ? [c] : getTests(c)))
function getTests(suite: Arrayable<Task>): (Test | Custom)[] {
const tests: (Test | Custom)[] = []
const arraySuites = toArray(suite)
for (const s of arraySuites) {
if (isAtomTest(s)) {
tests.push(s)
}
else {
for (const task of s.tasks) {
if (isAtomTest(task))
tests.push(task)
else
tests.push(...getTests(task))
}
}
}
return tests
}
1 change: 1 addition & 0 deletions packages/ui/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
},
"devDependencies": {
"@faker-js/faker": "^8.0.2",
"@iconify-json/logos": "^1.1.37",
"@testing-library/cypress": "^9.0.0",
"@types/codemirror": "^5.60.8",
"@types/d3-force": "^3.0.4",
Expand Down
9 changes: 9 additions & 0 deletions packages/vitest/src/node/cli-api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,15 @@ export async function startVitest(
if (typeof options.browser === 'object' && !('enabled' in options.browser))
options.browser.enabled = true

if (typeof options.typecheck === 'boolean')
options.typecheck = { enabled: true }

if (typeof options.typecheck?.only === 'boolean') {
options.typecheck ??= {}
options.typecheck.only = true
options.typecheck.enabled = true
}

const ctx = await createVitest(mode, options, viteOverrides)

if (mode === 'test' && ctx.config.coverage.enabled) {
Expand Down
12 changes: 3 additions & 9 deletions packages/vitest/src/node/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ cli
.option('--bail <number>', 'Stop test execution when given number of tests have failed', { default: 0 })
.option('--retry <times>', 'Retry the test specific number of times if it fails', { default: 0 })
.option('--diff <path>', 'Path to a diff config that will be used to generate diff interface')
.option('--typecheck [options]', 'Custom options for typecheck pool')
.option('--typecheck.enabled', 'Enable typechecking alongside tests (default: false)')
.option('--typecheck.only', 'Run only typecheck tests. This automatically enables typecheck (default: false)')
.help()

cli
Expand All @@ -73,10 +76,6 @@ cli
.command('bench [...filters]')
.action(benchmark)

cli
.command('typecheck [...filters]')
.action(typecheck)

cli
.command('[...filters]')
.action((filters, options) => start('test', filters, options))
Expand Down Expand Up @@ -130,11 +129,6 @@ async function benchmark(cliFilters: string[], options: CliOptions): Promise<voi
await start('benchmark', cliFilters, options)
}

async function typecheck(cliFilters: string[] = [], options: CliOptions = {}) {
console.warn(c.yellow('Testing types with tsc and vue-tsc is an experimental feature.\nBreaking changes might not follow semver, please pin Vitest\'s version when using it.'))
await start('typecheck', cliFilters, options)
}

function normalizeCliOptions(argv: CliOptions): CliOptions {
if (argv.root)
argv.root = normalize(argv.root)
Expand Down
9 changes: 5 additions & 4 deletions packages/vitest/src/node/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -378,10 +378,11 @@ export function resolveConfig(

resolved.environmentMatchGlobs = (resolved.environmentMatchGlobs || []).map(i => [resolve(resolved.root, i[0]), i[1]])

if (mode === 'typecheck') {
resolved.include = resolved.typecheck.include
resolved.exclude = resolved.typecheck.exclude
}
resolved.typecheck ??= {} as any
resolved.typecheck.enabled ??= false

if (resolved.typecheck.enabled)
console.warn(c.yellow('Testing types with tsc and vue-tsc is an experimental feature.\nBreaking changes might not follow semver, please pin Vitest\'s version when using it.'))

resolved.browser ??= {} as any
resolved.browser.enabled ??= false
Expand Down
20 changes: 10 additions & 10 deletions packages/vitest/src/node/core.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ export class Vitest {
}

private _onRestartListeners: OnServerRestartHandler[] = []
private _onClose: (() => Awaited<unknown>)[] = []
private _onSetServer: OnServerRestartHandler[] = []
private _onCancelListeners: ((reason: CancelReason) => Promise<void> | void)[] = []

Expand All @@ -93,7 +94,7 @@ export class Vitest {
this.cache = new VitestCache()
this.snapshot = new SnapshotManager({ ...resolved.snapshotOptions })

if (this.config.watch && this.mode !== 'typecheck')
if (this.config.watch)
this.registerWatcher()

this.vitenode = new ViteNodeServer(server, this.config.server)
Expand Down Expand Up @@ -308,15 +309,8 @@ export class Vitest {
return Promise.all(this.projects.map(w => w.initBrowserProvider()))
}

typecheck(filters?: string[]) {
return Promise.all(this.projects.map(project => project.typecheck(filters)))
}

async start(filters?: string[]) {
if (this.mode === 'typecheck') {
await this.typecheck(filters)
return
}
this._onClose = []

try {
await this.initCoverageProvider()
Expand Down Expand Up @@ -748,7 +742,7 @@ export class Vitest {

async close() {
if (!this.closingPromise) {
const closePromises = this.projects.map(w => w.close().then(() => w.server = undefined as any))
const closePromises: unknown[] = this.projects.map(w => w.close().then(() => w.server = undefined as any))
// close the core workspace server only once
// it's possible that it's not initialized at all because it's not running any tests
if (!this.coreWorkspaceProject || !this.projects.includes(this.coreWorkspaceProject))
Expand All @@ -757,6 +751,8 @@ export class Vitest {
if (this.pool)
closePromises.push(this.pool.close().then(() => this.pool = undefined))

closePromises.push(...this._onClose.map(fn => fn()))

this.closingPromise = Promise.allSettled(closePromises).then((results) => {
results.filter(r => r.status === 'rejected').forEach((err) => {
this.logger.error('error during close', (err as PromiseRejectedResult).reason)
Expand Down Expand Up @@ -834,4 +830,8 @@ export class Vitest {
onCancel(fn: (reason: CancelReason) => void) {
this._onCancelListeners.push(fn)
}

onClose(fn: () => void) {
this._onClose.push(fn)
}
}
19 changes: 15 additions & 4 deletions packages/vitest/src/node/logger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -95,10 +95,21 @@ export class Logger {
const comma = c.dim(', ')
if (filters?.length)
this.console.error(c.dim('filter: ') + c.yellow(filters.join(comma)))
if (config.include)
this.console.error(c.dim('include: ') + c.yellow(config.include.join(comma)))
if (config.exclude)
this.console.error(c.dim('exclude: ') + c.yellow(config.exclude.join(comma)))
this.ctx.projects.forEach((project) => {
const config = project.config
const name = project.getName()
const output = project.isCore() || !name ? '' : `[${name}]`
if (output)
this.console.error(c.bgCyan(`${output} Config`))
if (config.include)
this.console.error(c.dim('include: ') + c.yellow(config.include.join(comma)))
if (config.exclude)
this.console.error(c.dim('exclude: ') + c.yellow(config.exclude.join(comma)))
if (config.typecheck.enabled) {
this.console.error(c.dim('typecheck include: ') + c.yellow(config.typecheck.include.join(comma)))
this.console.error(c.dim('typecheck exclude: ') + c.yellow(config.typecheck.exclude.join(comma)))
}
})
if (config.watchExclude)
this.console.error(c.dim('watch exclude: ') + c.yellow(config.watchExclude.join(comma)))

Expand Down
19 changes: 17 additions & 2 deletions packages/vitest/src/node/pool.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { createThreadsPool } from './pools/threads'
import { createBrowserPool } from './pools/browser'
import { createVmThreadsPool } from './pools/vm-threads'
import type { WorkspaceProject } from './workspace'
import { createTypecheckPool } from './pools/typecheck'

export type WorkspaceSpec = [project: WorkspaceProject, testFile: string]
export type RunWithFiles = (files: WorkspaceSpec[], invalidates?: string[]) => Promise<void>
Expand All @@ -35,12 +36,20 @@ export function createPool(ctx: Vitest): ProcessPool {
threads: null,
browser: null,
vmThreads: null,
typescript: null,
}

function getDefaultPoolName(project: WorkspaceProject): Pool {
function getDefaultPoolName(project: WorkspaceProject, file: string): Pool {
if (project.config.browser.enabled)
return 'browser'

if (project.config.typecheck.enabled) {
for (const glob of project.config.typecheck.include) {
if (mm.isMatch(file, glob, { cwd: project.config.root }))
return 'typescript'
}
}

return project.config.pool
}

Expand All @@ -51,7 +60,7 @@ export function createPool(ctx: Vitest): ProcessPool {
if (mm.isMatch(file, glob, { cwd: project.config.root }))
return pool as Pool
}
return getDefaultPoolName(project)
return getDefaultPoolName(project, file)
}

async function runTests(files: WorkspaceSpec[], invalidate?: string[]) {
Expand Down Expand Up @@ -93,6 +102,7 @@ export function createPool(ctx: Vitest): ProcessPool {
threads: [],
browser: [],
vmThreads: [],
typescript: [],
}

for (const spec of files) {
Expand Down Expand Up @@ -123,6 +133,11 @@ export function createPool(ctx: Vitest): ProcessPool {
return pools.threads.runTests(files, invalidate)
}

if (pool === 'typescript') {
pools.typescript ??= createTypecheckPool(ctx)
return pools.typescript.runTests(files)
}

pools.forks ??= createChildProcessPool(ctx, options)
return pools.forks.runTests(files, invalidate)
}))
Expand Down
Loading

0 comments on commit a1aadd7

Please sign in to comment.