From 93c862a5d9a25def694a9d60eaa4860431929ec7 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Fri, 4 Mar 2022 12:27:18 -0800 Subject: [PATCH 1/9] chore: Try out nextest in CI Starting with flakey test retries. Signed-off-by: Jesse Szwedko --- .config/nextest.toml | 2 ++ Makefile | 6 +++--- scripts/environment/prepare.sh | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 .config/nextest.toml diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 0000000000000..07216b1ff16eb --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,2 @@ +[profile.default] +retries = 3 diff --git a/Makefile b/Makefile index e1ddb78c3e291..216a7392beff7 100644 --- a/Makefile +++ b/Makefile @@ -285,7 +285,7 @@ target/%/vector.tar.gz: target/%/vector CARGO_HANDLES_FRESHNESS .PHONY: test test: ## Run the unit test suite - ${MAYBE_ENVIRONMENT_EXEC} cargo test --workspace --no-fail-fast --no-default-features --features "${DEFAULT_FEATURES} metrics-benches codecs-benches language-benches remap-benches statistic-benches ${DNSTAP_BENCHES} benches" ${SCOPE} + ${MAYBE_ENVIRONMENT_EXEC} cargo nextest run -- --workspace --no-fail-fast --no-default-features --features "${DEFAULT_FEATURES} metrics-benches codecs-benches language-benches remap-benches statistic-benches ${DNSTAP_BENCHES} benches" ${SCOPE} .PHONY: test-all test-all: test test-behavior test-integration ## Runs all tests, unit, behaviorial, and integration. @@ -330,7 +330,7 @@ ifeq ($(AUTOSPAWN), true) @scripts/setup_integration_env.sh nats start sleep 10 # Many services are very slow... Give them a sec.. endif - ${MAYBE_ENVIRONMENT_EXEC} cargo test --no-fail-fast --no-default-features --features nats-integration-tests --lib ::nats:: + ${MAYBE_ENVIRONMENT_EXEC} cargo nextest run --no-fail-fast --no-default-features --features nats-integration-tests --lib ::nats:: ifeq ($(AUTODESPAWN), true) @scripts/setup_integration_env.sh nats stop endif @@ -369,7 +369,7 @@ test-shutdown-cleanup: .PHONY: test-cli test-cli: ## Runs cli tests - ${MAYBE_ENVIRONMENT_EXEC} cargo test --no-fail-fast --no-default-features --features cli-tests --test cli -- --test-threads 4 + ${MAYBE_ENVIRONMENT_EXEC} cargo nextest run --no-fail-fast --no-default-features --features cli-tests --test cli -- --test-threads 4 ##@ Benching (Supports `ENVIRONMENT=true`) diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index d83a1a9c0c697..9e260d146e5ea 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -5,6 +5,7 @@ rustup show # causes installation of version from rust-toolchain.toml rustup default "$(rustup show active-toolchain | awk '{print $1;}')" rustup run stable cargo install cargo-deb --version 1.29.2 rustup run stable cargo install cross --version 0.2.1 +rustup run stable cargo install nextest --version 0.9.8 cd scripts bundle install From c1ee7540ce34c7d2ebadcaa9e5eeddc87fd3e847 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Fri, 4 Mar 2022 14:08:39 -0800 Subject: [PATCH 2/9] Split up benches and non-benches tests `nextest` doesn't work with criterion benches: https://github.com/nextest-rs/nextest/issues/96 Signed-off-by: Jesse Szwedko --- .github/workflows/test.yml | 3 +++ Makefile | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 744defee94bf6..1a2eca95aa01d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -117,6 +117,9 @@ jobs: - run: make test env: CARGO_BUILD_JOBS: 5 + - run: make test-benches + env: + CARGO_BUILD_JOBS: 5 - name: Stop sccache run: sccache --stop-server diff --git a/Makefile b/Makefile index 216a7392beff7..945def0b537c7 100644 --- a/Makefile +++ b/Makefile @@ -285,10 +285,14 @@ target/%/vector.tar.gz: target/%/vector CARGO_HANDLES_FRESHNESS .PHONY: test test: ## Run the unit test suite - ${MAYBE_ENVIRONMENT_EXEC} cargo nextest run -- --workspace --no-fail-fast --no-default-features --features "${DEFAULT_FEATURES} metrics-benches codecs-benches language-benches remap-benches statistic-benches ${DNSTAP_BENCHES} benches" ${SCOPE} + ${MAYBE_ENVIRONMENT_EXEC} cargo nextest run --workspace --no-fail-fast --no-default-features --features "${DEFAULT_FEATURES}" ${SCOPE} + +.PHONY: test-benches +test-benches: ## Run the bench test suite + ${MAYBE_ENVIRONMENT_EXEC} cargo test --benches --workspace --no-fail-fast --no-default-features --features "${DEFAULT_FEATURES} metrics-benches codecs-benches language-benches remap-benches statistic-benches ${DNSTAP_BENCHES} benches" ${SCOPE} -- --bench .PHONY: test-all -test-all: test test-behavior test-integration ## Runs all tests, unit, behaviorial, and integration. +test-all: test test-benches test-behavior test-integration ## Runs all tests, unit, benches, behaviorial, and integration. .PHONY: test-x86_64-unknown-linux-gnu test-x86_64-unknown-linux-gnu: cross-test-x86_64-unknown-linux-gnu ## Runs unit tests on the x86_64-unknown-linux-gnu triple From d48dad89472f1ed57e24b8b9ee14eeb595c609ca Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Fri, 4 Mar 2022 14:25:47 -0800 Subject: [PATCH 3/9] Avoid testing benches with nextest Since criterion doesn't support the flags mentioned on https://nexte.st/book/custom-test-harnesses.html Signed-off-by: Jesse Szwedko --- Cargo.toml | 8 -------- lib/file-source/Cargo.toml | 1 - lib/tracing-limit/Cargo.toml | 1 - lib/vrl/compiler/Cargo.toml | 1 - lib/vrl/stdlib/Cargo.toml | 1 - lib/vrl/vrl/Cargo.toml | 1 - 6 files changed, 13 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1e9df44525e93..04c3ce6b2aa48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -816,44 +816,37 @@ enrichment-tables-benches = ["enrichment-tables-file"] [[bench]] name = "default" harness = false -test = true required-features = ["benches"] [[bench]] name = "dnstap" path = "benches/dnstap/mod.rs" harness = false -test = true required-features = ["dnstap-benches"] [[bench]] name = "remap" harness = false -test = true required-features = ["remap-benches"] [[bench]] name = "enrichment_tables_file" harness = false -test = true required-features = ["enrichment-tables-benches"] [[bench]] name = "languages" harness = false -test = true required-features = ["language-benches"] [[bench]] name = "loki" harness = false -test = true required-features = ["loki-benches"] [[bench]] name = "distribution_statistic" harness = false -test = true required-features = ["statistic-benches"] [[bench]] @@ -867,5 +860,4 @@ required-features = ["transform-benches"] name = "codecs" path = "benches/codecs/main.rs" harness = false -test = false required-features = ["codecs-benches"] diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index 4661cd8404773..ebc6d12de62eb 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -79,4 +79,3 @@ pretty_assertions = "1.1.0" [[bench]] name = "buffer" harness = false -test = true diff --git a/lib/tracing-limit/Cargo.toml b/lib/tracing-limit/Cargo.toml index 06c7c9645d776..ce1b806c6ca4b 100644 --- a/lib/tracing-limit/Cargo.toml +++ b/lib/tracing-limit/Cargo.toml @@ -21,4 +21,3 @@ tracing-subscriber = { version = "0.3.9", default-features = false, features = [ [[bench]] name = "limit" harness = false -test = true diff --git a/lib/vrl/compiler/Cargo.toml b/lib/vrl/compiler/Cargo.toml index d566b67589cab..3716c5fb852de 100644 --- a/lib/vrl/compiler/Cargo.toml +++ b/lib/vrl/compiler/Cargo.toml @@ -35,7 +35,6 @@ vector_common = { path = "../../vector-common", default-features = false, featur [[bench]] name = "kind" harness = false -test = true [features] test = [] diff --git a/lib/vrl/stdlib/Cargo.toml b/lib/vrl/stdlib/Cargo.toml index d5df30e0cac13..45dfe88f0ce8b 100644 --- a/lib/vrl/stdlib/Cargo.toml +++ b/lib/vrl/stdlib/Cargo.toml @@ -303,5 +303,4 @@ bench = false [[bench]] name = "benches" harness = false -test = true required-features = ["default"] diff --git a/lib/vrl/vrl/Cargo.toml b/lib/vrl/vrl/Cargo.toml index 6e4330c3f89ba..93dac03d79685 100644 --- a/lib/vrl/vrl/Cargo.toml +++ b/lib/vrl/vrl/Cargo.toml @@ -25,4 +25,3 @@ vrl-stdlib = { path = "../stdlib" } [[bench]] name = "vm" harness = false -test = true From e53801651fa17eaa4ba6113b58c9d4cf8c9174c2 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Fri, 4 Mar 2022 14:39:20 -0800 Subject: [PATCH 4/9] Fix cargo nextest installation Signed-off-by: Jesse Szwedko --- scripts/environment/prepare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index 9e260d146e5ea..14d3dcfbd1ae7 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -5,7 +5,7 @@ rustup show # causes installation of version from rust-toolchain.toml rustup default "$(rustup show active-toolchain | awk '{print $1;}')" rustup run stable cargo install cargo-deb --version 1.29.2 rustup run stable cargo install cross --version 0.2.1 -rustup run stable cargo install nextest --version 0.9.8 +rustup run stable cargo install cargo-nextest --version 0.9.8 cd scripts bundle install From 4d4cc83825d317a62f2f97ed9e0b7fe71abdc1a3 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Fri, 4 Mar 2022 14:47:11 -0800 Subject: [PATCH 5/9] Run docs tests too since nextest doesn't support them Signed-off-by: Jesse Szwedko --- .github/workflows/test.yml | 3 +++ Makefile | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1a2eca95aa01d..4fa994e802038 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -117,6 +117,9 @@ jobs: - run: make test env: CARGO_BUILD_JOBS: 5 + - run: make test-doc + env: + CARGO_BUILD_JOBS: 5 - run: make test-benches env: CARGO_BUILD_JOBS: 5 diff --git a/Makefile b/Makefile index 945def0b537c7..88cfa63f018d2 100644 --- a/Makefile +++ b/Makefile @@ -291,8 +291,12 @@ test: ## Run the unit test suite test-benches: ## Run the bench test suite ${MAYBE_ENVIRONMENT_EXEC} cargo test --benches --workspace --no-fail-fast --no-default-features --features "${DEFAULT_FEATURES} metrics-benches codecs-benches language-benches remap-benches statistic-benches ${DNSTAP_BENCHES} benches" ${SCOPE} -- --bench +.PHONY: test-docs +test-docs: ## Run the docs test suite + ${MAYBE_ENVIRONMENT_EXEC} cargo test --doc --workspace --no-fail-fast --no-default-features --features "${DEFAULT_FEATURES}" ${SCOPE} + .PHONY: test-all -test-all: test test-benches test-behavior test-integration ## Runs all tests, unit, benches, behaviorial, and integration. +test-all: test test-benches test-docs test-behavior test-integration ## Runs all tests: unit, benches, docs, behaviorial, and integration. .PHONY: test-x86_64-unknown-linux-gnu test-x86_64-unknown-linux-gnu: cross-test-x86_64-unknown-linux-gnu ## Runs unit tests on the x86_64-unknown-linux-gnu triple From efbef582761ec354ea0b24713f3b5b008e6f95ed Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Fri, 4 Mar 2022 15:21:54 -0800 Subject: [PATCH 6/9] Fix docs test Signed-off-by: Jesse Szwedko --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4fa994e802038..c9650457cdb0e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -117,7 +117,7 @@ jobs: - run: make test env: CARGO_BUILD_JOBS: 5 - - run: make test-doc + - run: make test-docs env: CARGO_BUILD_JOBS: 5 - run: make test-benches From b06da2e6a3e3617c158f29ca05ce9b96aed23d94 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Mon, 7 Mar 2022 10:11:31 -0800 Subject: [PATCH 7/9] Don't test benches since there isn't a way to test just them Signed-off-by: Jesse Szwedko --- .github/workflows/test.yml | 7 +------ Makefile | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c9650457cdb0e..110d685c0bd3a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -117,12 +117,6 @@ jobs: - run: make test env: CARGO_BUILD_JOBS: 5 - - run: make test-docs - env: - CARGO_BUILD_JOBS: 5 - - run: make test-benches - env: - CARGO_BUILD_JOBS: 5 - name: Stop sccache run: sccache --stop-server @@ -163,6 +157,7 @@ jobs: - run: make test-cli - run: make test-behavior - run: make check-examples + - run: make test-docs - name: Stop sccache run: sccache --stop-server diff --git a/Makefile b/Makefile index 88cfa63f018d2..e08e973d32c47 100644 --- a/Makefile +++ b/Makefile @@ -283,20 +283,27 @@ target/%/vector.tar.gz: target/%/vector CARGO_HANDLES_FRESHNESS ##@ Testing (Supports `ENVIRONMENT=true`) +# nextest doesn't support running doc tests yet so this is split out as +# `test-docs` +# https://github.com/nextest-rs/nextest/issues/16 +# +# criterion doesn't support the flags needed by nextest to run so these are left +# out for now +# https://github.com/bheisler/criterion.rs/issues/562 +# +# `cargo test` lacks support for testing _just_ benches otherwise we'd have +# a target for that +# https://github.com/rust-lang/cargo/issues/6454 .PHONY: test test: ## Run the unit test suite ${MAYBE_ENVIRONMENT_EXEC} cargo nextest run --workspace --no-fail-fast --no-default-features --features "${DEFAULT_FEATURES}" ${SCOPE} -.PHONY: test-benches -test-benches: ## Run the bench test suite - ${MAYBE_ENVIRONMENT_EXEC} cargo test --benches --workspace --no-fail-fast --no-default-features --features "${DEFAULT_FEATURES} metrics-benches codecs-benches language-benches remap-benches statistic-benches ${DNSTAP_BENCHES} benches" ${SCOPE} -- --bench - .PHONY: test-docs test-docs: ## Run the docs test suite ${MAYBE_ENVIRONMENT_EXEC} cargo test --doc --workspace --no-fail-fast --no-default-features --features "${DEFAULT_FEATURES}" ${SCOPE} .PHONY: test-all -test-all: test test-benches test-docs test-behavior test-integration ## Runs all tests: unit, benches, docs, behaviorial, and integration. +test-all: test test-docs test-behavior test-integration ## Runs all tests: unit, docs, behaviorial, and integration. .PHONY: test-x86_64-unknown-linux-gnu test-x86_64-unknown-linux-gnu: cross-test-x86_64-unknown-linux-gnu ## Runs unit tests on the x86_64-unknown-linux-gnu triple From 39a8c07b9c9f0e9f8fad6c5748d27db35561581f Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Mon, 7 Mar 2022 10:13:05 -0800 Subject: [PATCH 8/9] Install nextest for Windows Signed-off-by: Jesse Szwedko --- scripts/environment/bootstrap-windows-2019.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/environment/bootstrap-windows-2019.ps1 b/scripts/environment/bootstrap-windows-2019.ps1 index 16884cfa53116..5c31ec3cf7161 100644 --- a/scripts/environment/bootstrap-windows-2019.ps1 +++ b/scripts/environment/bootstrap-windows-2019.ps1 @@ -5,3 +5,5 @@ if ($env:CI -ne $null) { } else { $env:Path += ";$HOME\.cargo\bin" } + +rustup run stable cargo install cargo-nextest --version 0.9.8 From 1715fdb2c548eebc925172317fcad49c696118e8 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Mon, 7 Mar 2022 14:50:49 -0800 Subject: [PATCH 9/9] Fix CLI test running Signed-off-by: Jesse Szwedko --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e08e973d32c47..0e8c508d7d0d3 100644 --- a/Makefile +++ b/Makefile @@ -384,7 +384,7 @@ test-shutdown-cleanup: .PHONY: test-cli test-cli: ## Runs cli tests - ${MAYBE_ENVIRONMENT_EXEC} cargo nextest run --no-fail-fast --no-default-features --features cli-tests --test cli -- --test-threads 4 + ${MAYBE_ENVIRONMENT_EXEC} cargo nextest run --no-fail-fast --no-default-features --features cli-tests --test cli --test-threads 4 ##@ Benching (Supports `ENVIRONMENT=true`)