From e7f440326714c0c91adc39d45439f080023a2742 Mon Sep 17 00:00:00 2001 From: Dima Pristupa Date: Tue, 8 Oct 2024 12:56:06 +0300 Subject: [PATCH 01/13] E2E: add (or expand) `config`/`init`/`add`/`rename` command tests (#878) * KamuCliApiServerHarnessOptions::with_kamu_config(): introduce * E2E: cover "kamu config" command * E2E, test_init_in_an_existing_workspace(): add * E2E, test_config_set_value(): respect potential CI agent podman configuration * Updated yanked crate: futures-channel * E2E, test_config_get_with_default(): respect potential CI agent podman configuration * kamu-cli-e2e-common: extract "player-scores" & "leaderboard" datasets into statics * assert_cmd: add patch * E2E, test_add_dataset_from_stdin(): add * E2E, test_add_dataset_from_stdin(): add * AddCommand::validate_args(): fix "name" arg processing * E2E, test_add_dataset_with_name(): mix add commands * E2E, test_add_dataset_with_replace(): add * Interact: fix if is_tty * CHANGELOG.md: update * E2E, test_delete_dataset(): add * E2E, test_delete_dataset_recursive(): add * E2E, test_delete_dataset_all(): add * E2E, test_rename_dataset(): add * assert_cmd: use patch (now from the kamu-data org) * kamu-cli-puppet: remove unused dep * Extracted to the separate PR --- CHANGELOG.md | 5 + Cargo.lock | 5 +- Cargo.toml | 1 + src/app/cli/src/commands/add_command.rs | 3 +- .../cli/src/services/config/config_service.rs | 2 + src/e2e/app/cli/common/Cargo.toml | 1 + src/e2e/app/cli/common/src/e2e_harness.rs | 33 ++- .../common/src/kamu_api_server_client_ext.rs | 171 ++++++------ .../app/cli/inmem/tests/tests/commands/mod.rs | 3 + .../tests/tests/commands/test_add_command.rs | 21 ++ .../tests/commands/test_config_command.rs | 53 ++++ .../tests/commands/test_delete_command.rs | 33 +++ .../tests/tests/commands/test_init_command.rs | 8 + .../tests/commands/test_rename_command.rs | 19 ++ src/e2e/app/cli/repo-tests/Cargo.toml | 1 + .../app/cli/repo-tests/src/commands/mod.rs | 6 + .../src/commands/test_add_command.rs | 169 ++++++++++++ .../src/commands/test_config_command.rs | 246 ++++++++++++++++++ .../src/commands/test_delete_command.rs | 186 +++++++++++++ .../src/commands/test_init_command.rs | 25 ++ .../src/commands/test_rename_command.rs | 59 +++++ 21 files changed, 961 insertions(+), 89 deletions(-) create mode 100644 src/e2e/app/cli/inmem/tests/tests/commands/test_config_command.rs create mode 100644 src/e2e/app/cli/inmem/tests/tests/commands/test_delete_command.rs create mode 100644 src/e2e/app/cli/inmem/tests/tests/commands/test_rename_command.rs create mode 100644 src/e2e/app/cli/repo-tests/src/commands/test_config_command.rs create mode 100644 src/e2e/app/cli/repo-tests/src/commands/test_delete_command.rs create mode 100644 src/e2e/app/cli/repo-tests/src/commands/test_rename_command.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index d0a3119c2..892b4ded2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,11 @@ Recommendation: for ease of reading, use the following order: ## [0.204.5] - 2024-10-08 ### Added - Postgres implementation for dataset entry and account Re-BAC repositories +- Added (or expanded) E2E tests for: + - `kamu config` command + - `kamu init` command + - `kamu add` command + - `kamu rename` command ### Changed - `kamu repo alias list`: added JSON output alongside with other formats mentioned in the command's help - Private Datasets, `DatasetEntry` integration that will allow us to build dataset indexing diff --git a/Cargo.lock b/Cargo.lock index 6605ab05e..793fa976f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1140,8 +1140,7 @@ checksum = "71938f30533e4d95a6d17aa530939da3842c2ab6f4f84b9dae68447e4129f74a" [[package]] name = "assert_cmd" version = "2.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1835b7f27878de8525dc71410b5a31cdcc5f230aed5ba5df968e09c201b23d" +source = "git+https://github.com/kamu-data/assert_cmd?branch=deactivate-output-truncation#c83b0859dcbce6f9bb941ea07b615cc90aebd003" dependencies = [ "anstyle", "bstr", @@ -5690,6 +5689,7 @@ dependencies = [ "internal-error", "kamu-cli-e2e-common-macros", "kamu-cli-puppet", + "lazy_static", "opendatafabric", "pretty_assertions", "regex", @@ -5762,6 +5762,7 @@ dependencies = [ "kamu-cli-e2e-common", "kamu-cli-puppet", "opendatafabric", + "pretty_assertions", "reqwest", "tokio", "tokio-retry", diff --git a/Cargo.toml b/Cargo.toml index 6be5b77ce..5268799ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -271,3 +271,4 @@ debug = "line-tables-only" # datafusion-odata = { git = 'https://github.com/kamu-data/datafusion-odata.git', branch = '42.0.0-axum-0.6' } # datafusion-ethers = { git = "https://github.com/kamu-data/datafusion-ethers.git", tag = "42.0.0" } # object_store = { git = 'https://github.com/s373r/arrow-rs', branch = 'add-debug-logs', package = "object_store" } +assert_cmd = { git = 'https://github.com/kamu-data/assert_cmd', branch = "deactivate-output-truncation" } diff --git a/src/app/cli/src/commands/add_command.rs b/src/app/cli/src/commands/add_command.rs index 19b387831..1a863f354 100644 --- a/src/app/cli/src/commands/add_command.rs +++ b/src/app/cli/src/commands/add_command.rs @@ -223,7 +223,8 @@ impl Command for AddCommand { "No manifest references or paths were provided", )); } - if self.name.is_some() && (self.recursive || self.snapshot_refs.len() != 1) { + if self.name.is_some() && (self.recursive || !(self.snapshot_refs.len() == 1 || self.stdin)) + { return Err(CLIError::usage_error( "Name override can be used only when adding a single manifest", )); diff --git a/src/app/cli/src/services/config/config_service.rs b/src/app/cli/src/services/config/config_service.rs index 1317c19f5..2baaf9842 100644 --- a/src/app/cli/src/services/config/config_service.rs +++ b/src/app/cli/src/services/config/config_service.rs @@ -256,6 +256,8 @@ impl ConfigService { fn path_for_scope(&self, scope: ConfigScope) -> PathBuf { match scope { + // TODO: Respect `XDG_CONFIG_HOME` when working with configs + // https://github.com/kamu-data/kamu-cli/issues/848 ConfigScope::User => dirs::home_dir() .expect("Cannot determine user home directory") .join(CONFIG_FILENAME), diff --git a/src/e2e/app/cli/common/Cargo.toml b/src/e2e/app/cli/common/Cargo.toml index 0714bf122..8f31999fb 100644 --- a/src/e2e/app/cli/common/Cargo.toml +++ b/src/e2e/app/cli/common/Cargo.toml @@ -30,6 +30,7 @@ opendatafabric = { workspace = true } async-trait = "0.1" chrono = { version = "0.4", default-features = false, features = ["now"] } indoc = "2" +lazy_static = { version = "1" } pretty_assertions = "1" regex = "1" reqwest = { version = "0.12", default-features = false, features = ["json"] } diff --git a/src/e2e/app/cli/common/src/e2e_harness.rs b/src/e2e/app/cli/common/src/e2e_harness.rs index dadb71910..539e4a3eb 100644 --- a/src/e2e/app/cli/common/src/e2e_harness.rs +++ b/src/e2e/app/cli/common/src/e2e_harness.rs @@ -34,6 +34,7 @@ pub struct KamuCliApiServerHarnessOptions { potential_workspace: PotentialWorkspace, env_vars: Vec<(String, String)>, frozen_system_time: Option>, + kamu_config: Option, } impl KamuCliApiServerHarnessOptions { @@ -70,13 +71,18 @@ impl KamuCliApiServerHarnessOptions { self.with_frozen_system_time(today) } + + pub fn with_kamu_config(mut self, content: &str) -> Self { + self.kamu_config = Some(content.into()); + + self + } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub struct KamuCliApiServerHarness { options: KamuCliApiServerHarnessOptions, - kamu_config: Option, } impl KamuCliApiServerHarness { @@ -174,11 +180,21 @@ impl KamuCliApiServerHarness { Self::new(options, Some(kamu_config)) } - fn new(options: KamuCliApiServerHarnessOptions, kamu_config: Option) -> Self { - Self { - options, - kamu_config, + fn new( + mut options: KamuCliApiServerHarnessOptions, + generated_kamu_config: Option, + ) -> Self { + assert!( + !(options.kamu_config.is_some() && generated_kamu_config.is_some()), + "There can be only one configuration file: either preset from the test options or \ + generated based on the storage type" + ); + + if options.kamu_config.is_none() { + options.kamu_config = generated_kamu_config; } + + Self { options } } pub async fn run_api_server(self, fixture: Fixture) @@ -208,7 +224,8 @@ impl KamuCliApiServerHarness { let KamuCliApiServerHarnessOptions { potential_workspace, env_vars, - frozen_system_time: freeze_system_time, + frozen_system_time, + kamu_config, } = self.options; let mut kamu = match potential_workspace { @@ -218,14 +235,14 @@ impl KamuCliApiServerHarness { KamuCliPuppet::new_workspace_tmp_with(NewWorkspaceOptions { is_multi_tenant, - kamu_config: self.kamu_config, + kamu_config, env_vars, }) .await } }; - kamu.set_system_time(freeze_system_time); + kamu.set_system_time(frozen_system_time); kamu } diff --git a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs index a79385dbd..aafe912e5 100644 --- a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs +++ b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs @@ -8,12 +8,101 @@ // by the Apache License, Version 2.0. use async_trait::async_trait; +use lazy_static::lazy_static; use reqwest::{Method, StatusCode}; use crate::{KamuApiServerClient, RequestBody}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +pub const DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR: &str = indoc::indoc!( + r#" + kind: DatasetSnapshot + version: 1 + content: + name: player-scores + kind: Root + metadata: + - kind: AddPushSource + sourceName: default + read: + kind: NdJson + schema: + - "match_time TIMESTAMP" + - "match_id BIGINT" + - "player_id STRING" + - "score BIGINT" + merge: + kind: Ledger + primaryKey: + - match_id + - player_id + - kind: SetVocab + eventTimeColumn: match_time + "# +); + +pub const DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR: &str = indoc::indoc!( + r#" + kind: DatasetSnapshot + version: 1 + content: + name: leaderboard + kind: Derivative + metadata: + - kind: SetTransform + inputs: + - datasetRef: player-scores + alias: player_scores + transform: + kind: Sql + engine: risingwave + queries: + - alias: leaderboard + # Note we are using explicit `crate materialized view` statement below + # because RW does not currently support Top-N queries directly on sinks. + # + # Note `partition by 1` is currently required by RW engine + # See: https://docs.risingwave.com/docs/current/window-functions/#syntax + query: | + create materialized view leaderboard as + select + * + from ( + select + row_number() over (partition by 1 order by score desc) as place, + match_time, + match_id, + player_id, + score + from player_scores + ) + where place <= 2 + - query: | + select * from leaderboard + - kind: SetVocab + eventTimeColumn: match_time + "# +); + +lazy_static! { + // https://github.com/kamu-data/kamu-cli/blob/master/examples/leaderboard/player-scores.yaml + pub static ref DATASET_ROOT_PLAYER_SCORES_SNAPSHOT: String = { + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR + .escape_default() + .to_string() + }; + + // https://github.com/kamu-data/kamu-cli/blob/master/examples/leaderboard/leaderboard.yaml + pub static ref DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT: String = { + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR + .escape_default() + .to_string() + }; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + pub type AccessToken = String; pub type DatasetId = String; @@ -108,37 +197,8 @@ impl KamuApiServerClientExt for KamuApiServerClient { } async fn create_player_scores_dataset(&self, token: &AccessToken) -> DatasetId { - // https://github.com/kamu-data/kamu-cli/blob/master/examples/leaderboard/player-scores.yaml - let snapshot = indoc::indoc!( - r#" - kind: DatasetSnapshot - version: 1 - content: - name: player-scores - kind: Root - metadata: - - kind: AddPushSource - sourceName: default - read: - kind: NdJson - schema: - - "match_time TIMESTAMP" - - "match_id BIGINT" - - "player_id STRING" - - "score BIGINT" - merge: - kind: Ledger - primaryKey: - - match_id - - player_id - - kind: SetVocab - eventTimeColumn: match_time - "# - ) - .escape_default() - .to_string(); - - self.create_dataset(&snapshot, token).await + self.create_dataset(&DATASET_ROOT_PLAYER_SCORES_SNAPSHOT, token) + .await } async fn create_player_scores_dataset_with_data(&self, token: &AccessToken) -> DatasetId { @@ -166,53 +226,8 @@ impl KamuApiServerClientExt for KamuApiServerClient { } async fn create_leaderboard(&self, token: &AccessToken) -> DatasetId { - // https://github.com/kamu-data/kamu-cli/blob/master/examples/leaderboard/leaderboard.yaml - let snapshot = indoc::indoc!( - r#" - kind: DatasetSnapshot - version: 1 - content: - name: leaderboard - kind: Derivative - metadata: - - kind: SetTransform - inputs: - - datasetRef: player-scores - alias: player_scores - transform: - kind: Sql - engine: risingwave - queries: - - alias: leaderboard - # Note we are using explicit `crate materialized view` statement below - # because RW does not currently support Top-N queries directly on sinks. - # - # Note `partition by 1` is currently required by RW engine - # See: https://docs.risingwave.com/docs/current/window-functions/#syntax - query: | - create materialized view leaderboard as - select - * - from ( - select - row_number() over (partition by 1 order by score desc) as place, - match_time, - match_id, - player_id, - score - from player_scores - ) - where place <= 2 - - query: | - select * from leaderboard - - kind: SetVocab - eventTimeColumn: match_time - "# - ) - .escape_default() - .to_string(); - - self.create_dataset(&snapshot, token).await + self.create_dataset(&DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT, token) + .await } } diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/mod.rs b/src/e2e/app/cli/inmem/tests/tests/commands/mod.rs index 77dccbd11..695c4f9d4 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/mod.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/mod.rs @@ -9,8 +9,11 @@ mod test_add_command; mod test_complete_command; +mod test_config_command; +mod test_delete_command; mod test_ingest_command; mod test_init_command; +mod test_rename_command; mod test_repo_alias_command; mod test_sql_command; mod test_system_api_server_gql_query; diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_add_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_add_command.rs index 2964dc298..b7b39e306 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_add_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_add_command.rs @@ -11,6 +11,27 @@ use kamu_cli_e2e_common::prelude::*; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_add_dataset_from_stdin +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_add_dataset_with_name +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_add_dataset_with_replace +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + kamu_cli_execute_command_e2e_test!( storage = inmem, fixture = kamu_cli_e2e_repo_tests::test_add_recursive diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_config_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_config_command.rs new file mode 100644 index 000000000..9fb7d78c3 --- /dev/null +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_config_command.rs @@ -0,0 +1,53 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_config_set_value +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_config_reset_key +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_config_get_with_default +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_config_get_from_config + options = Options::default().with_kamu_config( + indoc::indoc!( + r#" + kind: CLIConfig + version: 1 + content: + engine: + runtime: podman + uploads: + maxFileSizeInMb: 42 + "# + ) + ) +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_delete_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_delete_command.rs new file mode 100644 index 000000000..0906123b3 --- /dev/null +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_delete_command.rs @@ -0,0 +1,33 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_delete_dataset +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_delete_dataset_recursive +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_delete_dataset_all +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_init_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_init_command.rs index 1d937dd5f..7f27502c5 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_init_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_init_command.rs @@ -43,3 +43,11 @@ kamu_cli_execute_command_e2e_test!( ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_init_in_an_existing_workspace, + options = Options::default().with_no_workspace() +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_rename_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_rename_command.rs new file mode 100644 index 000000000..a8666b29b --- /dev/null +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_rename_command.rs @@ -0,0 +1,19 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_rename_dataset +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/Cargo.toml b/src/e2e/app/cli/repo-tests/Cargo.toml index 7b44ffd70..8ba810ed7 100644 --- a/src/e2e/app/cli/repo-tests/Cargo.toml +++ b/src/e2e/app/cli/repo-tests/Cargo.toml @@ -36,6 +36,7 @@ opendatafabric = { workspace = true } chrono = { version = "0.4", default-features = false } indoc = "2" +pretty_assertions = { version = "1" } reqwest = { version = "0.12", default-features = false, features = [] } tokio = { version = "1", default-features = false, features = [] } tokio-retry = "0.3" diff --git a/src/e2e/app/cli/repo-tests/src/commands/mod.rs b/src/e2e/app/cli/repo-tests/src/commands/mod.rs index 19a8906ae..d3763d842 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/mod.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/mod.rs @@ -9,8 +9,11 @@ mod test_add_command; mod test_complete_command; +mod test_config_command; +mod test_delete_command; mod test_ingest_command; mod test_init_command; +mod test_rename_command; mod test_repo_alias_command; mod test_sql_command; mod test_system_api_server_gql_query; @@ -18,8 +21,11 @@ mod test_system_generate_token_command; pub use test_add_command::*; pub use test_complete_command::*; +pub use test_config_command::*; +pub use test_delete_command::*; pub use test_ingest_command::*; pub use test_init_command::*; +pub use test_rename_command::*; pub use test_repo_alias_command::*; pub use test_sql_command::*; pub use test_system_api_server_gql_query::*; diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_add_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_add_command.rs index 4a10c33ad..8cba1ea12 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_add_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_add_command.rs @@ -8,12 +8,181 @@ // by the Apache License, Version 2.0. use kamu::testing::MetadataFactory; +use kamu_cli_e2e_common::DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR; use kamu_cli_puppet::extensions::KamuCliPuppetExt; use kamu_cli_puppet::KamuCliPuppet; use opendatafabric as odf; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +pub async fn test_add_dataset_from_stdin(kamu: KamuCliPuppet) { + let assert = kamu + .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!( + r#" + Added: player-scores + Added 1 dataset(s) + "# + )), + "Unexpected output:\n{stderr}", + ); + + let dataset_names = kamu + .list_datasets() + .await + .into_iter() + .map(|dataset| dataset.name) + .collect::>(); + + assert_eq!(dataset_names, ["player-scores"]); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_add_dataset_with_name(kamu: KamuCliPuppet) { + // Add from stdio + { + let assert = kamu + .execute_with_input( + ["add", "--stdin", "--name", "player-scores-1"], + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, + ) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!( + r#" + Added: player-scores-1 + Added 1 dataset(s) + "# + )), + "Unexpected output:\n{stderr}", + ); + } + // Add from a file + { + let snapshot_path = kamu.workspace_path().join("player-scores.yaml"); + + std::fs::write( + snapshot_path.clone(), + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, + ) + .unwrap(); + + let assert = kamu + .execute([ + "add", + "--name", + "player-scores-2", + snapshot_path.to_str().unwrap(), + ]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!( + r#" + Added: player-scores-2 + Added 1 dataset(s) + "# + )), + "Unexpected output:\n{stderr}", + ); + } + + let dataset_names = kamu + .list_datasets() + .await + .into_iter() + .map(|dataset| dataset.name) + .collect::>(); + + assert_eq!(dataset_names, ["player-scores-1", "player-scores-2"]); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_add_dataset_with_replace(kamu: KamuCliPuppet) { + { + let assert = kamu + .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!( + r#" + Added: player-scores + "# + )), + "Unexpected output:\n{stderr}", + ); + } + { + let assert = kamu + .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!( + r#" + Skipped: player-scores: Already exists + Added 0 dataset(s) + "# + )), + "Unexpected output:\n{stderr}", + ); + } + { + let assert = kamu + .execute_with_input( + ["--yes", "add", "--stdin", "--replace"], + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, + ) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!( + r#" + Added: player-scores + Added 1 dataset(s) + "# + )), + "Unexpected output:\n{stderr}", + ); + } + + let dataset_names = kamu + .list_datasets() + .await + .into_iter() + .map(|dataset| dataset.name) + .collect::>(); + + assert_eq!(dataset_names, ["player-scores"]); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + pub async fn test_add_recursive(kamu: KamuCliPuppet) { // Plain manifest let snapshot = MetadataFactory::dataset_snapshot().name("plain").build(); diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_config_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_config_command.rs new file mode 100644 index 000000000..f8d11beff --- /dev/null +++ b/src/e2e/app/cli/repo-tests/src/commands/test_config_command.rs @@ -0,0 +1,246 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_puppet::KamuCliPuppet; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_config_set_value(kamu: KamuCliPuppet) { + // 0. CI sets container runtime to podman for some targets, so we simulate this + // behavior for all others. + { + let assert = kamu + .execute(["config", "set", "engine.runtime", "podman"]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Set engine.runtime to podman in workspace scope"), + "Unexpected output:\n{stderr}", + ); + } + + // 1. Set flow for the "engine.networkNs" key + { + let assert = kamu.execute(["config", "list"]).await.success(); + let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + pretty_assertions::assert_eq!( + stdout, + indoc::indoc!( + r#" + engine: + runtime: podman + + "# + ) + ); + } + { + let assert = kamu + .execute(["config", "set", "engine.networkNs", "host"]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Set engine.networkNs to host in workspace scope"), + "Unexpected output:\n{stderr}", + ); + } + { + let assert = kamu + .execute(["config", "get", "engine.networkNs"]) + .await + .success(); + + let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + pretty_assertions::assert_eq!( + stdout, + indoc::indoc!( + r#" + host + + "# + ) + ); + } + { + let assert = kamu.execute(["config", "list"]).await.success(); + let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + pretty_assertions::assert_eq!( + stdout, + indoc::indoc!( + r#" + engine: + runtime: podman + networkNs: host + + "# + ) + ); + } + // 2. Set flow for the "uploads.maxFileSizeInMb" key + { + let assert = kamu + .execute(["config", "set", "uploads.maxFileSizeInMb", "42"]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Set uploads.maxFileSizeInMb to 42 in workspace scope"), + "Unexpected output:\n{stderr}", + ); + } + { + let assert = kamu + .execute(["config", "get", "uploads.maxFileSizeInMb"]) + .await + .success(); + + let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + pretty_assertions::assert_eq!( + stdout, + indoc::indoc!( + r#" + 42 + + "# + ) + ); + } + { + let assert = kamu.execute(["config", "list"]).await.success(); + let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + pretty_assertions::assert_eq!( + stdout, + indoc::indoc!( + r#" + engine: + runtime: podman + networkNs: host + uploads: + maxFileSizeInMb: 42 + + "# + ) + ); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_config_reset_key(kamu: KamuCliPuppet) { + { + let assert = kamu + .execute(["config", "set", "engine.networkNs", "host"]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Set engine.networkNs to host in workspace scope"), + "Unexpected output:\n{stderr}", + ); + } + { + let assert = kamu + .execute(["config", "set", "engine.networkNs"]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Removed engine.networkNs from workspace scope"), + "Unexpected output:\n{stderr}", + ); + } + { + let assert = kamu + .execute(["config", "get", "engine.networkNs"]) + .await + .failure(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Error: Key engine.networkNs not found"), + "Unexpected output:\n{stderr}", + ); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_config_get_with_default(kamu: KamuCliPuppet) { + { + let assert = kamu + .execute(["config", "get", "engine.networkNs"]) + .await + .failure(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Error: Key engine.networkNs not found"), + "Unexpected output:\n{stderr}", + ); + } + { + let assert = kamu + .execute(["config", "get", "engine.networkNs", "--with-defaults"]) + .await + .success(); + + let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + pretty_assertions::assert_eq!( + stdout, + indoc::indoc!( + r#" + private + + "# + ) + ); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_config_get_from_config(kamu: KamuCliPuppet) { + let assert = kamu.execute(["config", "list"]).await.success(); + let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + pretty_assertions::assert_eq!( + stdout, + indoc::indoc!( + r#" + engine: + runtime: podman + uploads: + maxFileSizeInMb: 42 + + "# + ) + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_delete_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_delete_command.rs new file mode 100644 index 000000000..8555ff0af --- /dev/null +++ b/src/e2e/app/cli/repo-tests/src/commands/test_delete_command.rs @@ -0,0 +1,186 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::{ + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, +}; +use kamu_cli_puppet::extensions::KamuCliPuppetExt; +use kamu_cli_puppet::KamuCliPuppet; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_delete_dataset(kamu: KamuCliPuppet) { + { + let assert = kamu.execute(["delete", "player-scores"]).await.failure(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Error: Dataset not found: player-scores"), + "Unexpected output:\n{stderr}", + ); + } + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + { + let assert = kamu + .execute(["--yes", "delete", "player-scores"]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Deleted 1 dataset(s)"), + "Unexpected output:\n{stderr}", + ); + } + + let dataset_names = kamu + .list_datasets() + .await + .into_iter() + .map(|dataset| dataset.name) + .collect::>(); + + assert!( + dataset_names.is_empty(), + "Unexpected dataset names: {dataset_names:?}" + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_delete_dataset_recursive(kamu: KamuCliPuppet) { + // 1. Root + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + // 2. Derivative (from 1.) + kamu.execute_with_input( + ["add", "--stdin"], + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + ) + .await + .success(); + + // 3. One more root + kamu.execute_with_input( + ["add", "--stdin", "--name", "another-root"], + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, + ) + .await + .success(); + + { + let dataset_names = kamu + .list_datasets() + .await + .into_iter() + .map(|dataset| dataset.name) + .collect::>(); + + assert_eq!( + dataset_names, + ["another-root", "leaderboard", "player-scores"] + ); + } + { + let assert = kamu + .execute(["--yes", "delete", "player-scores", "--recursive"]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Deleted 2 dataset(s)"), + "Unexpected output:\n{stderr}", + ); + } + { + let dataset_names = kamu + .list_datasets() + .await + .into_iter() + .map(|dataset| dataset.name) + .collect::>(); + + assert_eq!(dataset_names, ["another-root"]); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_delete_dataset_all(kamu: KamuCliPuppet) { + // 1. Root + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + // 2. Derivative (from 1.) + kamu.execute_with_input( + ["add", "--stdin"], + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + ) + .await + .success(); + + // 3. One more root + kamu.execute_with_input( + ["add", "--stdin", "--name", "another-root"], + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, + ) + .await + .success(); + + { + let dataset_names = kamu + .list_datasets() + .await + .into_iter() + .map(|dataset| dataset.name) + .collect::>(); + + assert_eq!( + dataset_names, + ["another-root", "leaderboard", "player-scores"] + ); + } + { + let assert = kamu.execute(["--yes", "delete", "--all"]).await.success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Deleted 3 dataset(s)"), + "Unexpected output:\n{stderr}", + ); + } + { + let dataset_names = kamu + .list_datasets() + .await + .into_iter() + .map(|dataset| dataset.name) + .collect::>(); + + assert!( + dataset_names.is_empty(), + "Unexpected dataset names: {dataset_names:?}" + ); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_init_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_init_command.rs index 06573431b..7b2c1e3a2 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_init_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_init_command.rs @@ -86,3 +86,28 @@ pub async fn test_init_exist_ok_mt(mut kamu: KamuCliPuppet) { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_init_in_an_existing_workspace(mut kamu: KamuCliPuppet) { + kamu.set_workspace_path_in_tmp_dir(); + + { + let assert = kamu.execute(["init"]).await.success(); + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Initialized an empty workspace"), + "Unexpected output:\n{stderr}", + ); + } + { + let assert = kamu.execute(["init"]).await.failure(); + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Error: Directory is already a kamu workspace"), + "Unexpected output:\n{stderr}", + ); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_rename_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_rename_command.rs new file mode 100644 index 000000000..987a6aba8 --- /dev/null +++ b/src/e2e/app/cli/repo-tests/src/commands/test_rename_command.rs @@ -0,0 +1,59 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR; +use kamu_cli_puppet::extensions::KamuCliPuppetExt; +use kamu_cli_puppet::KamuCliPuppet; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_rename_dataset(kamu: KamuCliPuppet) { + { + let assert = kamu + .execute(["rename", "player-scores", "top-player-scores"]) + .await + .failure(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Error: Dataset not found: player-scores"), + "Unexpected output:\n{stderr}", + ); + } + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + { + let assert = kamu + .execute(["rename", "player-scores", "top-player-scores"]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Dataset renamed"), + "Unexpected output:\n{stderr}", + ); + } + + let dataset_names = kamu + .list_datasets() + .await + .into_iter() + .map(|dataset| dataset.name) + .collect::>(); + + assert_eq!(dataset_names, ["top-player-scores"]); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// From 40fd121bdaf4d06963bede04eccf32d01cae1260 Mon Sep 17 00:00:00 2001 From: Dima Pristupa Date: Fri, 11 Oct 2024 19:28:35 +0300 Subject: [PATCH 02/13] E2E: add (or expand) `ingest`/`inspect`/`log`/`new`/`reset`/`search`/etc command tests (#883) * E2E, ingest_data_to_player_scores_from_stdio(): add * clippy fixes * E2E, test_ingest_from_stdin(): also check data via the tail command * E2E, ingest_data_to_player_scores_from_stdio(): check data for all ingest steps * E2E, test_ingest_recursive(): add... a part of the test * E2E, test_ingest_with_source_name(): add * E2E, test_inspect_lineage(): add * IngestCommand: add a todo * DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_{1,2,3}: stabilize tests * Rename: LineageCommand -> InspectLineageCommand * E2E: stabilize ingest command tests * KamuCliPuppetExt::list_blocks(): add * E2E, test_inspect_query(): add * E2E, test_inspect_schema(): add * kamu-cli, Login: fix a doc string typo * E2E, test_log(): add * E2E, test_new_{root,derivative}(): add * E2E, test_reset(): add * cli-reference.md: update * test_repo_alias_command -> test_repo_command * E2E, KamuApiServerClientExt::ingest_data(): add * E2E, test_search_multi_user(): add * E2E, test_search_by_name(): add * E2E, test_search_by_repo(): add * E2E, test_sql_command(): add * E2E, test_gc(): add * E2E, test_system_info(): add * E2E, test_system_diagnose(): add * CI: test fixes * assert_ingest_data_to_player_scores_from_stdio(): stabilize rows order * assert_ingest_data_to_player_scores_from_stdio(): stabilize rows order [2] * test_sql_command(): use the default name * assert_ingest_data_to_player_scores_from_stdio(): stabilize rows order [3] * CHANGELOG.md: update * CI: windows build fixes * E2E, test_tail(): add * assert_ingest_data_to_player_scores_from_stdio(): stabilize rows order [4] * assert_ingest_data_to_player_scores_from_stdio(): stabilize rows order [5] * pretty_assertions::assert_eq!(): expected first, actual after * Remove extra files * E2E: cover "kamu login" / "kamu logout" commands --- CHANGELOG.md | 13 + resources/cli-reference.md | 4 +- .../graphql/src/mutations/datasets_mut.rs | 2 + src/app/cli/src/cli.rs | 2 +- src/app/cli/src/cli_commands.rs | 2 +- src/app/cli/src/commands/ingest_command.rs | 2 + ..._command.rs => inspect_lineage_command.rs} | 6 +- src/app/cli/src/commands/log_command.rs | 2 + .../cli/src/commands/login_silent_command.rs | 3 + src/app/cli/src/commands/mod.rs | 4 +- src/app/cli/src/commands/reset_command.rs | 4 + src/app/cli/src/commands/tail_command.rs | 4 + src/app/cli/src/services/gc_service.rs | 17 +- src/domain/core/src/services/reset_service.rs | 4 + src/e2e/app/cli/common/src/e2e_harness.rs | 12 +- .../common/src/kamu_api_server_client_ext.rs | 96 ++++- .../app/cli/inmem/tests/tests/commands/mod.rs | 12 +- .../tests/commands/test_ingest_command.rs | 29 ++ .../tests/commands/test_inspect_command.rs | 35 ++ .../tests/tests/commands/test_log_command.rs | 21 + .../tests/commands/test_login_command.rs | 27 ++ .../tests/tests/commands/test_new_command.rs | 26 ++ ..._alias_command.rs => test_repo_command.rs} | 0 .../tests/commands/test_reset_command.rs | 20 + .../tests/commands/test_search_command.rs | 66 +++ .../tests/tests/commands/test_sql_command.rs | 9 + .../commands/test_system_diagnose_command.rs | 19 + .../tests/commands/test_system_gc_command.rs | 16 + .../commands/test_system_info_command.rs | 19 + .../tests/tests/commands/test_tail_command.rs | 21 + .../app/cli/repo-tests/src/commands/mod.rs | 24 +- .../src/commands/test_config_command.rs | 28 +- .../src/commands/test_ingest_command.rs | 309 +++++++++++++- .../src/commands/test_inspect_command.rs | 250 ++++++++++++ .../src/commands/test_log_command.rs | 316 +++++++++++++++ .../src/commands/test_login_command.rs | 185 +++++++++ .../src/commands/test_new_command.rs | 68 ++++ ..._alias_command.rs => test_repo_command.rs} | 2 + .../src/commands/test_reset_command.rs | 87 ++++ .../src/commands/test_search_command.rs | 382 ++++++++++++++++++ .../src/commands/test_sql_command.rs | 116 +++++- .../src/commands/test_system_gc_command.rs | 25 ++ .../src/commands/test_system_info_command.rs | 18 + .../src/commands/test_system_info_diagnose.rs | 18 + .../src/commands/test_tail_command.rs | 68 ++++ src/e2e/app/cli/repo-tests/src/lib.rs | 2 + src/e2e/app/cli/repo-tests/src/test_flow.rs | 4 +- src/infra/core/src/query_service_impl.rs | 2 +- .../src/remote_repository_registry_impl.rs | 3 +- .../kamu-cli-puppet/src/kamu_cli_puppet.rs | 14 +- .../src/kamu_cli_puppet_ext.rs | 63 ++- 51 files changed, 2403 insertions(+), 78 deletions(-) rename src/app/cli/src/commands/{lineage_command.rs => inspect_lineage_command.rs} (99%) create mode 100644 src/e2e/app/cli/inmem/tests/tests/commands/test_inspect_command.rs create mode 100644 src/e2e/app/cli/inmem/tests/tests/commands/test_log_command.rs create mode 100644 src/e2e/app/cli/inmem/tests/tests/commands/test_login_command.rs create mode 100644 src/e2e/app/cli/inmem/tests/tests/commands/test_new_command.rs rename src/e2e/app/cli/inmem/tests/tests/commands/{test_repo_alias_command.rs => test_repo_command.rs} (100%) create mode 100644 src/e2e/app/cli/inmem/tests/tests/commands/test_reset_command.rs create mode 100644 src/e2e/app/cli/inmem/tests/tests/commands/test_search_command.rs create mode 100644 src/e2e/app/cli/inmem/tests/tests/commands/test_system_diagnose_command.rs create mode 100644 src/e2e/app/cli/inmem/tests/tests/commands/test_system_gc_command.rs create mode 100644 src/e2e/app/cli/inmem/tests/tests/commands/test_system_info_command.rs create mode 100644 src/e2e/app/cli/inmem/tests/tests/commands/test_tail_command.rs create mode 100644 src/e2e/app/cli/repo-tests/src/commands/test_inspect_command.rs create mode 100644 src/e2e/app/cli/repo-tests/src/commands/test_log_command.rs create mode 100644 src/e2e/app/cli/repo-tests/src/commands/test_login_command.rs create mode 100644 src/e2e/app/cli/repo-tests/src/commands/test_new_command.rs rename src/e2e/app/cli/repo-tests/src/commands/{test_repo_alias_command.rs => test_repo_command.rs} (97%) create mode 100644 src/e2e/app/cli/repo-tests/src/commands/test_reset_command.rs create mode 100644 src/e2e/app/cli/repo-tests/src/commands/test_search_command.rs create mode 100644 src/e2e/app/cli/repo-tests/src/commands/test_system_gc_command.rs create mode 100644 src/e2e/app/cli/repo-tests/src/commands/test_system_info_command.rs create mode 100644 src/e2e/app/cli/repo-tests/src/commands/test_system_info_diagnose.rs create mode 100644 src/e2e/app/cli/repo-tests/src/commands/test_tail_command.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 892b4ded2..dc906eb53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,19 @@ Recommendation: for ease of reading, use the following order: - `kamu init` command - `kamu add` command - `kamu rename` command + - `kamu ingest` command + - `kamu inspect` command + - `kamu log` command + - `kamu new` command + - `kamu reset` command + - `kamu search` command + - `kamu sql` command + - `kamu system gc` command + - `kamu system info` command + - `kamu system diagnose` command + - `kamu tail` command + - `kamu login` command + - `kamu logout` command ### Changed - `kamu repo alias list`: added JSON output alongside with other formats mentioned in the command's help - Private Datasets, `DatasetEntry` integration that will allow us to build dataset indexing diff --git a/resources/cli-reference.md b/resources/cli-reference.md index 0355784ae..dea607fce 100644 --- a/resources/cli-reference.md +++ b/resources/cli-reference.md @@ -20,7 +20,7 @@ To regenerate this schema from existing code, use the following command: * `inspect` — Group of commands for exploring dataset metadata * `list [ls]` — List all datasets in the workspace * `log` — Shows dataset metadata history -* `login` — Authentiates with a remote ODF server interactively +* `login` — Authenticates with a remote ODF server interactively * `logout` — Logs out from a remote Kamu server * `new` — Creates a new dataset manifest from a template * `notebook` — Starts the notebook server for exploring the data in the workspace @@ -508,7 +508,7 @@ Using a filter to inspect blocks containing query changes of a derivative datase ## `kamu login` -Authentiates with a remote ODF server interactively +Authenticates with a remote ODF server interactively **Usage:** `kamu login [OPTIONS] [SERVER] [COMMAND]` diff --git a/src/adapter/graphql/src/mutations/datasets_mut.rs b/src/adapter/graphql/src/mutations/datasets_mut.rs index aa023cb81..9357a72d0 100644 --- a/src/adapter/graphql/src/mutations/datasets_mut.rs +++ b/src/adapter/graphql/src/mutations/datasets_mut.rs @@ -110,6 +110,8 @@ impl DatasetsMut { } // TODO: Multi-tenancy + // https://github.com/kamu-data/kamu-cli/issues/891 + // TODO: Multi-tenant resolution for derivative dataset inputs (should it only // work by ID?) #[allow(unused_variables)] diff --git a/src/app/cli/src/cli.rs b/src/app/cli/src/cli.rs index 11610e789..187ea0e3c 100644 --- a/src/app/cli/src/cli.rs +++ b/src/app/cli/src/cli.rs @@ -634,7 +634,7 @@ pub struct Log { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -/// Authentiates with a remote ODF server interactively +/// Authenticates with a remote ODF server interactively #[derive(Debug, clap::Args)] pub struct Login { #[command(subcommand)] diff --git a/src/app/cli/src/cli_commands.rs b/src/app/cli/src/cli_commands.rs index 09c790b6b..3dbeaf4f1 100644 --- a/src/app/cli/src/cli_commands.rs +++ b/src/app/cli/src/cli_commands.rs @@ -127,7 +127,7 @@ pub fn get_command( } } cli::Command::Inspect(c) => match c.subcommand { - cli::InspectSubCommand::Lineage(sc) => Box::new(LineageCommand::new( + cli::InspectSubCommand::Lineage(sc) => Box::new(InspectLineageCommand::new( cli_catalog.get_one()?, cli_catalog.get_one()?, cli_catalog.get_one()?, diff --git a/src/app/cli/src/commands/ingest_command.rs b/src/app/cli/src/commands/ingest_command.rs index bbffd7ca2..67471f9f0 100644 --- a/src/app/cli/src/commands/ingest_command.rs +++ b/src/app/cli/src/commands/ingest_command.rs @@ -145,6 +145,8 @@ impl Command for IngestCommand { _ => Ok(()), }?; + // TODO: `kamu ingest`: implement `--recursive` mode + // https://github.com/kamu-data/kamu-cli/issues/886 if self.recursive { unimplemented!("Sorry, recursive ingest is not yet implemented") } diff --git a/src/app/cli/src/commands/lineage_command.rs b/src/app/cli/src/commands/inspect_lineage_command.rs similarity index 99% rename from src/app/cli/src/commands/lineage_command.rs rename to src/app/cli/src/commands/inspect_lineage_command.rs index 1503238f9..c1200342b 100644 --- a/src/app/cli/src/commands/lineage_command.rs +++ b/src/app/cli/src/commands/inspect_lineage_command.rs @@ -31,7 +31,7 @@ pub enum LineageOutputFormat { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub struct LineageCommand { +pub struct InspectLineageCommand { dataset_repo: Arc, provenance_svc: Arc, workspace_layout: Arc, @@ -41,7 +41,7 @@ pub struct LineageCommand { output_config: Arc, } -impl LineageCommand { +impl InspectLineageCommand { pub fn new( dataset_repo: Arc, provenance_svc: Arc, @@ -94,7 +94,7 @@ impl LineageCommand { // TODO: Support temporality and evolution #[async_trait::async_trait(?Send)] -impl Command for LineageCommand { +impl Command for InspectLineageCommand { async fn run(&mut self) -> Result<(), CLIError> { use futures::{StreamExt, TryStreamExt}; let mut dataset_handles: Vec<_> = if self.dataset_refs.is_empty() { diff --git a/src/app/cli/src/commands/log_command.rs b/src/app/cli/src/commands/log_command.rs index 87561fe55..ca5852b20 100644 --- a/src/app/cli/src/commands/log_command.rs +++ b/src/app/cli/src/commands/log_command.rs @@ -29,6 +29,8 @@ use crate::output::OutputConfig; pub enum MetadataLogOutputFormat { Shell, Yaml, + // TODO: `kamu log`: support `--output-format json` + // https://github.com/kamu-data/kamu-cli/issues/887 } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/app/cli/src/commands/login_silent_command.rs b/src/app/cli/src/commands/login_silent_command.rs index f67160ef3..daef6049f 100644 --- a/src/app/cli/src/commands/login_silent_command.rs +++ b/src/app/cli/src/commands/login_silent_command.rs @@ -16,16 +16,19 @@ use crate::{odf_server, CLIError, Command}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +#[derive(Debug)] pub enum LoginSilentMode { OAuth(LoginSilentModeOAuth), Password(LoginSilentModePassword), } +#[derive(Debug)] pub struct LoginSilentModeOAuth { pub provider: String, pub access_token: String, } +#[derive(Debug)] pub struct LoginSilentModePassword { pub login: String, pub password: String, diff --git a/src/app/cli/src/commands/mod.rs b/src/app/cli/src/commands/mod.rs index 5525e8f3d..385ad0a10 100644 --- a/src/app/cli/src/commands/mod.rs +++ b/src/app/cli/src/commands/mod.rs @@ -20,9 +20,9 @@ mod delete_command; mod gc_command; mod ingest_command; mod init_command; +mod inspect_lineage_command; mod inspect_query_command; mod inspect_schema_command; -mod lineage_command; mod list_command; mod log_command; mod login_command; @@ -71,9 +71,9 @@ pub use delete_command::*; pub use gc_command::*; pub use ingest_command::*; pub use init_command::*; +pub use inspect_lineage_command::*; pub use inspect_query_command::*; pub use inspect_schema_command::*; -pub use lineage_command::*; pub use list_command::*; pub use log_command::*; pub use login_command::*; diff --git a/src/app/cli/src/commands/reset_command.rs b/src/app/cli/src/commands/reset_command.rs index 8a4ba34d5..32f69a4df 100644 --- a/src/app/cli/src/commands/reset_command.rs +++ b/src/app/cli/src/commands/reset_command.rs @@ -15,6 +15,8 @@ use opendatafabric::*; use super::{CLIError, Command}; use crate::Interact; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + pub struct ResetCommand { interact: Arc, dataset_repo: Arc, @@ -66,3 +68,5 @@ impl Command for ResetCommand { Ok(()) } } + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/app/cli/src/commands/tail_command.rs b/src/app/cli/src/commands/tail_command.rs index e715e71ca..b025ceeb3 100644 --- a/src/app/cli/src/commands/tail_command.rs +++ b/src/app/cli/src/commands/tail_command.rs @@ -17,6 +17,8 @@ use opendatafabric::*; use super::{CLIError, Command}; use crate::output::*; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + pub struct TailCommand { query_svc: Arc, dataset_ref: DatasetRef, @@ -93,3 +95,5 @@ impl Command for TailCommand { Ok(()) } } + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/app/cli/src/services/gc_service.rs b/src/app/cli/src/services/gc_service.rs index 29d0e4e98..9c91e7638 100644 --- a/src/app/cli/src/services/gc_service.rs +++ b/src/app/cli/src/services/gc_service.rs @@ -9,11 +9,17 @@ use std::sync::Arc; -use chrono::{DateTime, Duration, Utc}; +use chrono::{DateTime, Duration, TimeDelta, Utc}; use internal_error::{InternalError, ResultIntoInternal}; use crate::WorkspaceLayout; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +const EVICTION_THRESHOLD: TimeDelta = Duration::hours(24); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + pub struct GcService { workspace_layout: Arc, } @@ -56,9 +62,6 @@ impl GcService { /// Evict stale entries to manage cache size #[tracing::instrument(level = "debug", skip_all)] pub fn evict_cache(&self) -> Result { - // TODO: Make const after https://github.com/chronotope/chrono/issues/309 - // Or make into a config option - let eviction_threshold: Duration = Duration::hours(24); let now = Utc::now(); let mut entries_freed = 0; let mut bytes_freed = 0; @@ -69,7 +72,7 @@ impl GcService { let mtime: DateTime = chrono::DateTime::from(entry.metadata().int_err()?.modified().int_err()?); - if (now - mtime) > eviction_threshold { + if (now - mtime) > EVICTION_THRESHOLD { if entry.path().is_dir() { bytes_freed += fs_extra::dir::get_size(entry.path()).int_err()?; std::fs::remove_dir_all(entry.path()).int_err()?; @@ -93,7 +96,11 @@ impl GcService { } } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + pub struct GcResult { pub entries_freed: usize, pub bytes_freed: u64, } + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/core/src/services/reset_service.rs b/src/domain/core/src/services/reset_service.rs index 17240e704..51b3738d2 100644 --- a/src/domain/core/src/services/reset_service.rs +++ b/src/domain/core/src/services/reset_service.rs @@ -14,6 +14,8 @@ use thiserror::Error; use crate::entities::SetRefError; use crate::*; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + #[async_trait::async_trait] pub trait ResetService: Send + Sync { async fn reset_dataset( @@ -103,3 +105,5 @@ pub struct OldHeadMismatchError { pub current_head: Multihash, pub old_head: Multihash, } + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/common/src/e2e_harness.rs b/src/e2e/app/cli/common/src/e2e_harness.rs index 539e4a3eb..40d4992d2 100644 --- a/src/e2e/app/cli/common/src/e2e_harness.rs +++ b/src/e2e/app/cli/common/src/e2e_harness.rs @@ -9,7 +9,7 @@ use std::future::Future; -use chrono::{DateTime, NaiveTime, Utc}; +use chrono::{DateTime, NaiveTime, TimeZone, Utc}; use kamu_cli_puppet::extensions::KamuCliPuppetExt; use kamu_cli_puppet::{KamuCliPuppet, NewWorkspaceOptions}; use regex::Regex; @@ -55,12 +55,18 @@ impl KamuCliApiServerHarnessOptions { self } - pub fn with_frozen_system_time(mut self, value: DateTime) -> Self { + pub fn with_custom_frozen_system_time(mut self, value: DateTime) -> Self { self.frozen_system_time = Some(value); self } + pub fn with_frozen_system_time(self) -> Self { + let t = Utc.with_ymd_and_hms(2050, 1, 2, 3, 4, 5).unwrap(); + + self.with_custom_frozen_system_time(t) + } + pub fn with_today_as_frozen_system_time(self) -> Self { let today = { let now = Utc::now(); @@ -69,7 +75,7 @@ impl KamuCliApiServerHarnessOptions { .unwrap() }; - self.with_frozen_system_time(today) + self.with_custom_frozen_system_time(today) } pub fn with_kamu_config(mut self, content: &str) -> Self { diff --git a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs index aafe912e5..a8a53ac4e 100644 --- a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs +++ b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs @@ -9,12 +9,14 @@ use async_trait::async_trait; use lazy_static::lazy_static; +use opendatafabric::{DatasetAlias, DatasetName}; use reqwest::{Method, StatusCode}; use crate::{KamuApiServerClient, RequestBody}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +/// pub const DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR: &str = indoc::indoc!( r#" kind: DatasetSnapshot @@ -42,6 +44,7 @@ pub const DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR: &str = indoc::indoc!( "# ); +/// pub const DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR: &str = indoc::indoc!( r#" kind: DatasetSnapshot @@ -86,14 +89,14 @@ pub const DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR: &str = indoc::indoc!( ); lazy_static! { - // https://github.com/kamu-data/kamu-cli/blob/master/examples/leaderboard/player-scores.yaml + /// pub static ref DATASET_ROOT_PLAYER_SCORES_SNAPSHOT: String = { DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR .escape_default() .to_string() }; - // https://github.com/kamu-data/kamu-cli/blob/master/examples/leaderboard/leaderboard.yaml + /// pub static ref DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT: String = { DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR .escape_default() @@ -101,6 +104,36 @@ lazy_static! { }; } +/// NOTE: 1 millisecond for stable order within tests +/// +/// +pub const DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1: &str = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} + {"match_time": "2000-01-01 00:00:00.001", "match_id": 1, "player_id": "Bob", "score": 80} + "# +); + +/// NOTE: 1 millisecond for stable order within tests +/// +/// +pub const DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2: &str = indoc::indoc!( + r#" + {"match_time": "2000-01-02", "match_id": 2, "player_id": "Alice", "score": 70} + {"match_time": "2000-01-02 00:00:00.001", "match_id": 2, "player_id": "Charlie", "score": 90} + "# +); + +/// NOTE: 1 millisecond for stable order within tests +/// +/// +pub const DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_3: &str = indoc::indoc!( + r#" + {"match_time": "2000-01-03", "match_id": 3, "player_id": "Bob", "score": 60} + {"match_time": "2000-01-03 00:00:00.001", "match_id": 3, "player_id": "Charlie", "score": 110} + "# +); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub type AccessToken = String; @@ -111,11 +144,26 @@ pub type DatasetId = String; #[async_trait] pub trait KamuApiServerClientExt { async fn login_as_kamu(&self) -> AccessToken; + async fn login_as_e2e_user(&self) -> AccessToken; + + // TODO: also return alias, after solving this bug: + // https://github.com/kamu-data/kamu-cli/issues/891 async fn create_dataset(&self, dataset_snapshot_yaml: &str, token: &AccessToken) -> DatasetId; + async fn create_player_scores_dataset(&self, token: &AccessToken) -> DatasetId; + + /// NOTE: only for single-tenant workspaces async fn create_player_scores_dataset_with_data(&self, token: &AccessToken) -> DatasetId; + async fn create_leaderboard(&self, token: &AccessToken) -> DatasetId; + + async fn ingest_data( + &self, + dataset_alias: &DatasetAlias, + data: RequestBody, + token: &AccessToken, + ); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -204,21 +252,16 @@ impl KamuApiServerClientExt for KamuApiServerClient { async fn create_player_scores_dataset_with_data(&self, token: &AccessToken) -> DatasetId { let dataset_id = self.create_player_scores_dataset(token).await; - self.rest_api_call_assert( - Some(token.clone()), - Method::POST, - "player-scores/ingest", - Some(RequestBody::NdJson( - indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Bob", "score": 80} - "#, - ) - .into(), - )), - StatusCode::OK, - None, + // TODO: Use the alias from the reply, after fixing the bug: + // https://github.com/kamu-data/kamu-cli/issues/891 + + // At the moment, only single-tenant + let dataset_alias = DatasetAlias::new(None, DatasetName::new_unchecked("player-scores")); + + self.ingest_data( + &dataset_alias, + RequestBody::NdJson(DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1.into()), + token, ) .await; @@ -229,6 +272,25 @@ impl KamuApiServerClientExt for KamuApiServerClient { self.create_dataset(&DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT, token) .await } + + async fn ingest_data( + &self, + dataset_alias: &DatasetAlias, + data: RequestBody, + token: &AccessToken, + ) { + let endpoint = format!("{dataset_alias}/ingest"); + + self.rest_api_call_assert( + Some(token.clone()), + Method::POST, + endpoint.as_str(), + Some(data), + StatusCode::OK, + None, + ) + .await; + } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/mod.rs b/src/e2e/app/cli/inmem/tests/tests/commands/mod.rs index 695c4f9d4..52308e3fa 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/mod.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/mod.rs @@ -13,8 +13,18 @@ mod test_config_command; mod test_delete_command; mod test_ingest_command; mod test_init_command; +mod test_inspect_command; +mod test_log_command; +mod test_login_command; +mod test_new_command; mod test_rename_command; -mod test_repo_alias_command; +mod test_repo_command; +mod test_reset_command; +mod test_search_command; mod test_sql_command; mod test_system_api_server_gql_query; +mod test_system_diagnose_command; +mod test_system_gc_command; mod test_system_generate_token_command; +mod test_system_info_command; +mod test_tail_command; diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_ingest_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_ingest_command.rs index 7974dae1c..3687ca7d5 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_ingest_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_ingest_command.rs @@ -14,6 +14,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, fixture = kamu_cli_e2e_repo_tests::test_push_ingest_from_file_ledger, + options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); @@ -22,6 +23,34 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, fixture = kamu_cli_e2e_repo_tests::test_push_ingest_from_file_snapshot_with_event_time, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_ingest_from_stdin, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_ingest_recursive, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_ingest_with_source_name, + options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_inspect_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_inspect_command.rs new file mode 100644 index 000000000..0d22b67d5 --- /dev/null +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_inspect_command.rs @@ -0,0 +1,35 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_inspect_lineage, +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_inspect_query, + options = Options::default().with_frozen_system_time() +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_inspect_schema, + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_log_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_log_command.rs new file mode 100644 index 000000000..d83b3b951 --- /dev/null +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_log_command.rs @@ -0,0 +1,21 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_log, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_login_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_login_command.rs new file mode 100644 index 000000000..d47541b57 --- /dev/null +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_login_command.rs @@ -0,0 +1,27 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_login_logout_password, +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_login_logout_oauth, + options = Options::default().with_multi_tenant() +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_new_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_new_command.rs new file mode 100644 index 000000000..fd38ab3d6 --- /dev/null +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_new_command.rs @@ -0,0 +1,26 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_new_root, +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_new_derivative, +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_repo_alias_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_repo_command.rs similarity index 100% rename from src/e2e/app/cli/inmem/tests/tests/commands/test_repo_alias_command.rs rename to src/e2e/app/cli/inmem/tests/tests/commands/test_repo_command.rs diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_reset_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_reset_command.rs new file mode 100644 index 000000000..17c46fee1 --- /dev/null +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_reset_command.rs @@ -0,0 +1,20 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_reset, + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_search_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_search_command.rs new file mode 100644 index 000000000..3d9b2c90d --- /dev/null +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_search_command.rs @@ -0,0 +1,66 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_search_multi_user + // We need synthetic time for the tests, but the third-party JWT code + // uses the current time. Assuming that the token lifetime is 24 hours, we will + // use the projected date (the current day) as a workaround. + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time() + .with_kamu_config( + indoc::indoc!( + r#" + kind: CLIConfig + version: 1 + content: + users: + predefined: + - accountName: kamu + "# + ) + ), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_search_by_name + // We need synthetic time for the tests, but the third-party JWT code + // uses the current time. Assuming that the token lifetime is 24 hours, we will + // use the projected date (the current day) as a workaround. + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_search_by_repo + // We need synthetic time for the tests, but the third-party JWT code + // uses the current time. Assuming that the token lifetime is 24 hours, we will + // use the projected date (the current day) as a workaround. + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_sql_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_sql_command.rs index a48fd8e50..7315367ca 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_sql_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_sql_command.rs @@ -27,3 +27,12 @@ kamu_cli_execute_command_e2e_test!( ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_sql_command, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_system_diagnose_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_system_diagnose_command.rs new file mode 100644 index 000000000..44d851352 --- /dev/null +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_system_diagnose_command.rs @@ -0,0 +1,19 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_system_diagnose +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_system_gc_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_system_gc_command.rs new file mode 100644 index 000000000..b36419994 --- /dev/null +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_system_gc_command.rs @@ -0,0 +1,16 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!(storage = inmem, fixture = kamu_cli_e2e_repo_tests::test_gc); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_system_info_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_system_info_command.rs new file mode 100644 index 000000000..00c824968 --- /dev/null +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_system_info_command.rs @@ -0,0 +1,19 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_system_info +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_tail_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_tail_command.rs new file mode 100644 index 000000000..ad25c37f6 --- /dev/null +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_tail_command.rs @@ -0,0 +1,21 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_tail, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/mod.rs b/src/e2e/app/cli/repo-tests/src/commands/mod.rs index d3763d842..5125a93ce 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/mod.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/mod.rs @@ -13,11 +13,21 @@ mod test_config_command; mod test_delete_command; mod test_ingest_command; mod test_init_command; +mod test_inspect_command; +mod test_log_command; +mod test_login_command; +mod test_new_command; mod test_rename_command; -mod test_repo_alias_command; +mod test_repo_command; +mod test_reset_command; +mod test_search_command; mod test_sql_command; mod test_system_api_server_gql_query; +mod test_system_gc_command; mod test_system_generate_token_command; +mod test_system_info_command; +mod test_system_info_diagnose; +mod test_tail_command; pub use test_add_command::*; pub use test_complete_command::*; @@ -25,8 +35,18 @@ pub use test_config_command::*; pub use test_delete_command::*; pub use test_ingest_command::*; pub use test_init_command::*; +pub use test_inspect_command::*; +pub use test_log_command::*; +pub use test_login_command::*; +pub use test_new_command::*; pub use test_rename_command::*; -pub use test_repo_alias_command::*; +pub use test_repo_command::*; +pub use test_reset_command::*; +pub use test_search_command::*; pub use test_sql_command::*; pub use test_system_api_server_gql_query::*; +pub use test_system_gc_command::*; pub use test_system_generate_token_command::*; +pub use test_system_info_command::*; +pub use test_system_info_diagnose::*; +pub use test_tail_command::*; diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_config_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_config_command.rs index f8d11beff..b1a3fcf19 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_config_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_config_command.rs @@ -34,14 +34,14 @@ pub async fn test_config_set_value(kamu: KamuCliPuppet) { let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); pretty_assertions::assert_eq!( - stdout, indoc::indoc!( r#" engine: runtime: podman "# - ) + ), + stdout ); } { @@ -66,13 +66,13 @@ pub async fn test_config_set_value(kamu: KamuCliPuppet) { let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); pretty_assertions::assert_eq!( - stdout, indoc::indoc!( r#" host "# - ) + ), + stdout ); } { @@ -80,7 +80,6 @@ pub async fn test_config_set_value(kamu: KamuCliPuppet) { let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); pretty_assertions::assert_eq!( - stdout, indoc::indoc!( r#" engine: @@ -88,7 +87,8 @@ pub async fn test_config_set_value(kamu: KamuCliPuppet) { networkNs: host "# - ) + ), + stdout ); } // 2. Set flow for the "uploads.maxFileSizeInMb" key @@ -114,13 +114,13 @@ pub async fn test_config_set_value(kamu: KamuCliPuppet) { let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); pretty_assertions::assert_eq!( - stdout, indoc::indoc!( r#" 42 "# - ) + ), + stdout ); } { @@ -128,7 +128,6 @@ pub async fn test_config_set_value(kamu: KamuCliPuppet) { let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); pretty_assertions::assert_eq!( - stdout, indoc::indoc!( r#" engine: @@ -138,7 +137,8 @@ pub async fn test_config_set_value(kamu: KamuCliPuppet) { maxFileSizeInMb: 42 "# - ) + ), + stdout ); } } @@ -212,13 +212,13 @@ pub async fn test_config_get_with_default(kamu: KamuCliPuppet) { let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); pretty_assertions::assert_eq!( - stdout, indoc::indoc!( r#" private "# - ) + ), + stdout ); } } @@ -230,7 +230,6 @@ pub async fn test_config_get_from_config(kamu: KamuCliPuppet) { let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); pretty_assertions::assert_eq!( - stdout, indoc::indoc!( r#" engine: @@ -239,7 +238,8 @@ pub async fn test_config_get_from_config(kamu: KamuCliPuppet) { maxFileSizeInMb: 42 "# - ) + ), + stdout ); } diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_ingest_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_ingest_command.rs index 1181b36de..04224bcaa 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_ingest_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_ingest_command.rs @@ -9,17 +9,21 @@ use std::path::Path; -use chrono::{TimeZone, Utc}; use indoc::indoc; +use kamu_cli_e2e_common::{ + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_3, + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, +}; use kamu_cli_puppet::extensions::KamuCliPuppetExt; use kamu_cli_puppet::KamuCliPuppet; use opendatafabric::*; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_push_ingest_from_file_ledger(mut kamu: KamuCliPuppet) { - kamu.set_system_time(Some(Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap())); - +pub async fn test_push_ingest_from_file_ledger(kamu: KamuCliPuppet) { kamu.add_dataset(DatasetSnapshot { name: "population".try_into().unwrap(), kind: DatasetKind::Root, @@ -86,9 +90,9 @@ pub async fn test_push_ingest_from_file_ledger(mut kamu: KamuCliPuppet) { +--------+----+----------------------+----------------------+------+------------+ | offset | op | system_time | event_time | city | population | +--------+----+----------------------+----------------------+------+------------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2020-01-01T00:00:00Z | A | 1000 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2020-01-01T00:00:00Z | B | 2000 | - | 2 | 0 | 2000-01-01T00:00:00Z | 2020-01-01T00:00:00Z | C | 3000 | + | 0 | 0 | 2050-01-02T03:04:05Z | 2020-01-01T00:00:00Z | A | 1000 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2020-01-01T00:00:00Z | B | 2000 | + | 2 | 0 | 2050-01-02T03:04:05Z | 2020-01-01T00:00:00Z | C | 3000 | +--------+----+----------------------+----------------------+------+------------+ "# ), @@ -96,9 +100,9 @@ pub async fn test_push_ingest_from_file_ledger(mut kamu: KamuCliPuppet) { .await; } -pub async fn test_push_ingest_from_file_snapshot_with_event_time(mut kamu: KamuCliPuppet) { - kamu.set_system_time(Some(Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap())); +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +pub async fn test_push_ingest_from_file_snapshot_with_event_time(kamu: KamuCliPuppet) { kamu.add_dataset(DatasetSnapshot { name: "population".try_into().unwrap(), kind: DatasetKind::Root, @@ -168,9 +172,9 @@ pub async fn test_push_ingest_from_file_snapshot_with_event_time(mut kamu: KamuC +--------+----+----------------------+----------------------+------+------------+ | offset | op | system_time | event_time | city | population | +--------+----+----------------------+----------------------+------+------------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2050-01-01T00:00:00Z | A | 1000 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2050-01-01T00:00:00Z | B | 2000 | - | 2 | 0 | 2000-01-01T00:00:00Z | 2050-01-01T00:00:00Z | C | 3000 | + | 0 | 0 | 2050-01-02T03:04:05Z | 2050-01-01T00:00:00Z | A | 1000 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2050-01-01T00:00:00Z | B | 2000 | + | 2 | 0 | 2050-01-02T03:04:05Z | 2050-01-01T00:00:00Z | C | 3000 | +--------+----+----------------------+----------------------+------+------------+ "# ), @@ -180,8 +184,289 @@ pub async fn test_push_ingest_from_file_snapshot_with_event_time(mut kamu: KamuC //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +pub async fn test_ingest_from_stdin(kamu: KamuCliPuppet) { + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + assert_ingest_data_to_player_scores_from_stdio( + &kamu, + ["ingest", "player-scores", "--stdin"], + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + indoc::indoc!( + r#" + ┌────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ + │ op │ system_time │ match_time │ match_id │ player_id │ score │ + ├────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ + └────┴──────────────────────┴──────────────────────┴──────────┴───────────┴───────┘ + "# + ), + ) + .await; + + assert_ingest_data_to_player_scores_from_stdio( + &kamu, + ["ingest", "player-scores", "--stdin"], + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + indoc::indoc!( + r#" + ┌────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ + │ op │ system_time │ match_time │ match_id │ player_id │ score │ + ├────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-02T00:00:00Z │ 2 │ Alice │ 70 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-02T00:00:00Z │ 2 │ Charlie │ 90 │ + └────┴──────────────────────┴──────────────────────┴──────────┴───────────┴───────┘ + "# + ), + ) + .await; + + assert_ingest_data_to_player_scores_from_stdio( + &kamu, + ["ingest", "player-scores", "--stdin"], + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_3, + indoc::indoc!( + r#" + ┌────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ + │ op │ system_time │ match_time │ match_id │ player_id │ score │ + ├────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-02T00:00:00Z │ 2 │ Alice │ 70 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-02T00:00:00Z │ 2 │ Charlie │ 90 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-03T00:00:00Z │ 3 │ Bob │ 60 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-03T00:00:00Z │ 3 │ Charlie │ 110 │ + └────┴──────────────────────┴──────────────────────┴──────────┴───────────┴───────┘ + "# + ), + ) + .await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_ingest_recursive(kamu: KamuCliPuppet) { + // 0. Add datasets: the root dataset and its derived dataset + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + kamu.execute_with_input( + ["add", "--stdin"], + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + ) + .await + .success(); + + { + let assert = kamu + .execute(["tail", "leaderboard", "--output-format", "table"]) + .await + .failure(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Error: Dataset schema is not yet available: leaderboard"), + "Unexpected output:\n{stderr}", + ); + } + + // TODO: `kamu ingest`: implement `--recursive` mode + // https://github.com/kamu-data/kamu-cli/issues/886 + + // 1. Ingest data: the first chunk + // { + // let assert = kamu + // .execute_with_input( + // ["ingest", "player-scores", "--stdin", "--recursive"], + // DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + // ) + // .await + // .success(); + // + // let stderr = + // std::str::from_utf8(&assert.get_output().stderr).unwrap(); + // + // assert!( + // stderr.contains("Dataset updated"), + // "Unexpected output:\n{stderr}", + // ); + // } + + // TODO: check via the tail command added data in the derived dataset + // (leaderboard) + + // TODO: do the same for 2nd & 3rd chunks +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_ingest_with_source_name(kamu: KamuCliPuppet) { + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + assert_ingest_data_to_player_scores_from_stdio( + &kamu, + [ + "ingest", + "player-scores", + "--stdin", + "--source-name", + "default", + ], + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + indoc::indoc!( + r#" + ┌────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ + │ op │ system_time │ match_time │ match_id │ player_id │ score │ + ├────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ + └────┴──────────────────────┴──────────────────────┴──────────┴───────────┴───────┘ + "# + ), + ) + .await; + + assert_ingest_data_to_player_scores_from_stdio( + &kamu, + [ + "ingest", + "player-scores", + "--stdin", + "--source-name", + "default", + ], + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + indoc::indoc!( + r#" + ┌────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ + │ op │ system_time │ match_time │ match_id │ player_id │ score │ + ├────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-02T00:00:00Z │ 2 │ Alice │ 70 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-02T00:00:00Z │ 2 │ Charlie │ 90 │ + └────┴──────────────────────┴──────────────────────┴──────────┴───────────┴───────┘ + "# + ), + ) + .await; + + assert_ingest_data_to_player_scores_from_stdio( + &kamu, + [ + "ingest", + "player-scores", + "--stdin", + "--source-name", + "default", + ], + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_3, + indoc::indoc!( + r#" + ┌────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ + │ op │ system_time │ match_time │ match_id │ player_id │ score │ + ├────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-02T00:00:00Z │ 2 │ Alice │ 70 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-02T00:00:00Z │ 2 │ Charlie │ 90 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-03T00:00:00Z │ 3 │ Bob │ 60 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-03T00:00:00Z │ 3 │ Charlie │ 110 │ + └────┴──────────────────────┴──────────────────────┴──────────┴───────────┴───────┘ + "# + ), + ) + .await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Helpers +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + fn path(p: &Path) -> &str { p.as_os_str().to_str().unwrap() } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +async fn assert_ingest_data_to_player_scores_from_stdio( + kamu: &KamuCliPuppet, + ingest_cmd: I, + ingest_data: T, + expected_tail_table: &str, +) where + I: IntoIterator + Clone, + S: AsRef, + T: Into> + Clone, +{ + // Ingest + { + let assert = kamu + .execute_with_input(ingest_cmd.clone(), ingest_data.clone()) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Dataset updated"), + "Unexpected output:\n{stderr}", + ); + } + // Trying to ingest the same data + { + let assert = kamu + .execute_with_input(ingest_cmd, ingest_data) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Dataset up-to-date"), + "Unexpected output:\n{stderr}", + ); + } + // Assert ingested data + { + let assert = kamu + .execute([ + "sql", + "--engine", + "datafusion", + "--command", + // Without unstable "offset" column. + // For a beautiful output, cut to seconds + indoc::indoc!( + r#" + SELECT op, + system_time, + DATE_TRUNC('second', match_time) as match_time, + match_id, + player_id, + score + FROM "player-scores" + ORDER BY match_time; + "# + ), + "--output-format", + "table", + ]) + .await + .success(); + + let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + pretty_assertions::assert_eq!(expected_tail_table, stdout); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_inspect_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_inspect_command.rs new file mode 100644 index 000000000..c849f4a7c --- /dev/null +++ b/src/e2e/app/cli/repo-tests/src/commands/test_inspect_command.rs @@ -0,0 +1,250 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::{ + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, +}; +use kamu_cli_puppet::extensions::KamuCliPuppetExt; +use kamu_cli_puppet::KamuCliPuppet; +use opendatafabric::{DatasetName, EnumWithVariants, SetTransform}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_inspect_lineage(kamu: KamuCliPuppet) { + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + kamu.execute_with_input( + ["add", "--stdin"], + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + ) + .await + .success(); + + { + let assert = kamu + .execute(["inspect", "lineage", "--output-format", "shell"]) + .await + .success(); + + let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + pretty_assertions::assert_eq!( + stdout, + indoc::indoc!( + r#" + leaderboard: Derivative + └── player-scores: Root + player-scores: Root + "# + ) + ); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_inspect_query(kamu: KamuCliPuppet) { + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + let player_scores_dataset_id = kamu + .list_datasets() + .await + .into_iter() + .find_map(|dataset| { + if dataset.name == DatasetName::new_unchecked("player-scores") { + Some(dataset.id) + } else { + None + } + }) + .unwrap(); + + kamu.execute_with_input( + ["add", "--stdin"], + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + ) + .await + .success(); + + let leaderboard_transform_block_hash = kamu + .list_blocks(&DatasetName::new_unchecked("leaderboard")) + .await + .into_iter() + .find_map(|block| { + if block.block.event.as_variant::().is_some() { + Some(block.block_hash) + } else { + None + } + }) + .unwrap(); + + { + let assert = kamu + .execute(["inspect", "query", "player-scores"]) + .await + .success(); + + let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + pretty_assertions::assert_eq!(stdout, ""); + } + { + let assert = kamu + .execute(["inspect", "query", "leaderboard"]) + .await + .success(); + + let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + pretty_assertions::assert_eq!( + indoc::formatdoc!( + r#" + Transform: {leaderboard_transform_block_hash} + As Of: 2050-01-02T03:04:05Z + Inputs: + player_scores {player_scores_dataset_id} + Engine: risingwave (None) + Query: leaderboard + create materialized view leaderboard as + select + * + from ( + select + row_number() over (partition by 1 order by score desc) as place, + match_time, + match_id, + player_id, + score + from player_scores + ) + where place <= 2 + Query: leaderboard + select * from leaderboard + "# + ), + stdout + ); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_inspect_schema(kamu: KamuCliPuppet) { + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + { + let assert = kamu + .execute(["inspect", "schema", "player-scores"]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Warning: Dataset schema is not yet available: player-scores"), + "Unexpected output:\n{stderr}", + ); + } + + kamu.execute_with_input( + ["add", "--stdin"], + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + ) + .await + .success(); + + { + let assert = kamu + .execute([ + "inspect", + "schema", + "leaderboard", + "--output-format", + "parquet", + ]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Warning: Dataset schema is not yet available: leaderboard"), + "Unexpected output:\n{stderr}", + ); + } + + kamu.execute_with_input( + ["ingest", "player-scores", "--stdin"], + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await + .success(); + + { + let assert = kamu + .execute([ + "inspect", + "schema", + "player-scores", + "--output-format", + "parquet", + ]) + .await + .success(); + + let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + pretty_assertions::assert_eq!( + indoc::indoc!( + r#" + message arrow_schema { + REQUIRED INT64 offset; + REQUIRED INT32 op; + REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_id; + OPTIONAL BYTE_ARRAY player_id (STRING); + OPTIONAL INT64 score; + } + "# + ), + stdout + ); + } + { + let assert = kamu + .execute([ + "inspect", + "schema", + "leaderboard", + "--output-format", + "parquet", + ]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Warning: Dataset schema is not yet available: leaderboard"), + "Unexpected output:\n{stderr}", + ); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_log_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_log_command.rs new file mode 100644 index 000000000..d24bc9471 --- /dev/null +++ b/src/e2e/app/cli/repo-tests/src/commands/test_log_command.rs @@ -0,0 +1,316 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::assert_matches::assert_matches; + +use chrono::{TimeZone, Timelike, Utc}; +use kamu_cli_e2e_common::{ + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, +}; +use kamu_cli_puppet::extensions::KamuCliPuppetExt; +use kamu_cli_puppet::KamuCliPuppet; +use opendatafabric::{ + AddData, + AddPushSource, + DatasetKind, + DatasetName, + EnumWithVariants, + MergeStrategy, + MergeStrategyLedger, + MetadataEvent, + OffsetInterval, + ReadStep, + ReadStepNdJson, + SetDataSchema, + SetTransform, + SetVocab, + SqlQueryStep, + Transform, + TransformSql, +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_log(kamu: KamuCliPuppet) { + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + kamu.execute_with_input( + ["ingest", "player-scores", "--stdin"], + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await + .success(); + + kamu.execute_with_input( + ["ingest", "player-scores", "--stdin"], + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + ) + .await + .success(); + + { + let mut metadata_blocks = kamu + .list_blocks(&DatasetName::new_unchecked("player-scores")) + .await + .into_iter() + .map(|br| br.block) + .collect::>(); + + pretty_assertions::assert_eq!(6, metadata_blocks.len()); + + { + let block = metadata_blocks.pop().unwrap(); + + pretty_assertions::assert_eq!(0, block.sequence_number); + + assert_matches!( + block.event, + MetadataEvent::Seed(event) + if event.dataset_kind == DatasetKind::Root + ); + } + { + let block = metadata_blocks.pop().unwrap(); + + pretty_assertions::assert_eq!(1, block.sequence_number); + + let actual_push_source = block.event.as_variant::().unwrap(); + let expected_push_source = AddPushSource { + source_name: "default".to_string(), + read: ReadStep::NdJson(ReadStepNdJson { + schema: Some(vec![ + "match_time TIMESTAMP".into(), + "match_id BIGINT".into(), + "player_id STRING".into(), + "score BIGINT".into(), + ]), + date_format: None, + encoding: None, + timestamp_format: None, + }), + preprocess: None, + merge: MergeStrategy::Ledger(MergeStrategyLedger { + primary_key: vec!["match_id".into(), "player_id".into()], + }), + }; + + pretty_assertions::assert_eq!(&expected_push_source, actual_push_source); + } + { + let block = metadata_blocks.pop().unwrap(); + + pretty_assertions::assert_eq!(2, block.sequence_number); + + let actual_set_vocab = block.event.as_variant::().unwrap(); + let expected_set_vocab = SetVocab { + offset_column: None, + operation_type_column: None, + system_time_column: None, + event_time_column: Some("match_time".into()), + }; + + pretty_assertions::assert_eq!(&expected_set_vocab, actual_set_vocab); + } + { + let block = metadata_blocks.pop().unwrap(); + + pretty_assertions::assert_eq!(3, block.sequence_number); + + let actual_set_data_schema = block.event.as_variant::().unwrap(); + let expected_set_data_schema = SetDataSchema { + schema: vec![ + 12, 0, 0, 0, 8, 0, 8, 0, 0, 0, 4, 0, 8, 0, 0, 0, 4, 0, 0, 0, 7, 0, 0, 0, 124, + 1, 0, 0, 60, 1, 0, 0, 244, 0, 0, 0, 180, 0, 0, 0, 108, 0, 0, 0, 56, 0, 0, 0, 4, + 0, 0, 0, 108, 255, 255, 255, 16, 0, 0, 0, 24, 0, 0, 0, 0, 0, 1, 2, 20, 0, 0, 0, + 160, 254, 255, 255, 64, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 115, 99, + 111, 114, 101, 0, 0, 0, 156, 255, 255, 255, 24, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1, + 5, 16, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 4, 0, 0, 0, 9, 0, 0, 0, 112, 108, 97, + 121, 101, 114, 95, 105, 100, 0, 0, 0, 204, 255, 255, 255, 16, 0, 0, 0, 24, 0, + 0, 0, 0, 0, 1, 2, 20, 0, 0, 0, 0, 255, 255, 255, 64, 0, 0, 0, 0, 0, 0, 1, 0, 0, + 0, 0, 8, 0, 0, 0, 109, 97, 116, 99, 104, 95, 105, 100, 0, 0, 0, 0, 16, 0, 20, + 0, 16, 0, 14, 0, 15, 0, 4, 0, 0, 0, 8, 0, 16, 0, 0, 0, 20, 0, 0, 0, 12, 0, 0, + 0, 0, 0, 1, 10, 28, 0, 0, 0, 0, 0, 0, 0, 196, 255, 255, 255, 8, 0, 0, 0, 0, 0, + 1, 0, 3, 0, 0, 0, 85, 84, 67, 0, 10, 0, 0, 0, 109, 97, 116, 99, 104, 95, 116, + 105, 109, 101, 0, 0, 144, 255, 255, 255, 28, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 10, + 36, 0, 0, 0, 0, 0, 0, 0, 8, 0, 12, 0, 10, 0, 4, 0, 8, 0, 0, 0, 8, 0, 0, 0, 0, + 0, 1, 0, 3, 0, 0, 0, 85, 84, 67, 0, 11, 0, 0, 0, 115, 121, 115, 116, 101, 109, + 95, 116, 105, 109, 101, 0, 212, 255, 255, 255, 16, 0, 0, 0, 24, 0, 0, 0, 0, 0, + 0, 2, 20, 0, 0, 0, 196, 255, 255, 255, 32, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, + 0, 0, 0, 111, 112, 0, 0, 16, 0, 20, 0, 16, 0, 0, 0, 15, 0, 4, 0, 0, 0, 8, 0, + 16, 0, 0, 0, 24, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 2, 28, 0, 0, 0, 8, 0, 12, 0, 4, + 0, 11, 0, 8, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 111, + 102, 102, 115, 101, 116, 0, 0, + ], + }; + + pretty_assertions::assert_eq!(&expected_set_data_schema, actual_set_data_schema); + } + { + let block = metadata_blocks.pop().unwrap(); + + pretty_assertions::assert_eq!(4, block.sequence_number); + + let actual_add_data = block.event.as_variant::().unwrap(); + + pretty_assertions::assert_eq!(None, actual_add_data.prev_checkpoint); + pretty_assertions::assert_eq!(None, actual_add_data.prev_offset); + + let actual_new_data = actual_add_data.new_data.as_ref().unwrap(); + + pretty_assertions::assert_eq!( + OffsetInterval { start: 0, end: 1 }, + actual_new_data.offset_interval + ); + pretty_assertions::assert_eq!(1674, actual_new_data.size); + + pretty_assertions::assert_eq!(None, actual_add_data.new_checkpoint); + pretty_assertions::assert_eq!( + Some( + Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0) + .unwrap() + .with_nanosecond(1_000_000) // 1 ms + .unwrap() + ), + actual_add_data.new_watermark + ); + pretty_assertions::assert_eq!(None, actual_add_data.new_source_state); + } + { + let block = metadata_blocks.pop().unwrap(); + + pretty_assertions::assert_eq!(5, block.sequence_number); + + let actual_add_data = block.event.as_variant::().unwrap(); + + pretty_assertions::assert_eq!(None, actual_add_data.prev_checkpoint); + pretty_assertions::assert_eq!(Some(1), actual_add_data.prev_offset); + + let actual_new_data = actual_add_data.new_data.as_ref().unwrap(); + + pretty_assertions::assert_eq!( + OffsetInterval { start: 2, end: 3 }, + actual_new_data.offset_interval + ); + pretty_assertions::assert_eq!(1690, actual_new_data.size); + + pretty_assertions::assert_eq!(None, actual_add_data.new_checkpoint); + pretty_assertions::assert_eq!( + Some( + Utc.with_ymd_and_hms(2000, 1, 2, 0, 0, 0) + .unwrap() + .with_nanosecond(1_000_000) // 1 ms + .unwrap() + ), + actual_add_data.new_watermark + ); + pretty_assertions::assert_eq!(None, actual_add_data.new_source_state); + } + } + + kamu.execute_with_input( + ["add", "--stdin"], + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + ) + .await + .success(); + + { + let mut metadata_blocks = kamu + .list_blocks(&DatasetName::new_unchecked("leaderboard")) + .await + .into_iter() + .map(|br| br.block) + .collect::>(); + + pretty_assertions::assert_eq!(3, metadata_blocks.len()); + + { + let block = metadata_blocks.pop().unwrap(); + + pretty_assertions::assert_eq!(0, block.sequence_number); + + assert_matches!( + block.event, + MetadataEvent::Seed(event) + if event.dataset_kind == DatasetKind::Derivative + ); + } + { + let block = metadata_blocks.pop().unwrap(); + + pretty_assertions::assert_eq!(1, block.sequence_number); + + let actual_set_transform = block.event.as_variant::().unwrap(); + + pretty_assertions::assert_eq!(1, actual_set_transform.inputs.len()); + pretty_assertions::assert_eq!( + Some("player_scores".into()), + actual_set_transform.inputs[0].alias + ); + + let expected_transform = Transform::Sql(TransformSql { + engine: "risingwave".into(), + version: None, + query: None, + queries: Some(vec![ + SqlQueryStep { + alias: Some("leaderboard".into()), + query: indoc::indoc!( + r#" + create materialized view leaderboard as + select + * + from ( + select + row_number() over (partition by 1 order by score desc) as place, + match_time, + match_id, + player_id, + score + from player_scores + ) + where place <= 2 + "# + ) + .into(), + }, + SqlQueryStep { + alias: None, + query: "select * from leaderboard".into(), + }, + ]), + temporal_tables: None, + }); + + pretty_assertions::assert_eq!(expected_transform, actual_set_transform.transform); + } + { + let block = metadata_blocks.pop().unwrap(); + + pretty_assertions::assert_eq!(2, block.sequence_number); + + let actual_set_vocab = block.event.as_variant::().unwrap(); + let expected_set_vocab = SetVocab { + offset_column: None, + operation_type_column: None, + system_time_column: None, + event_time_column: Some("match_time".into()), + }; + + pretty_assertions::assert_eq!(&expected_set_vocab, actual_set_vocab); + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_login_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_login_command.rs new file mode 100644 index 000000000..79f3c005a --- /dev/null +++ b/src/e2e/app/cli/repo-tests/src/commands/test_login_command.rs @@ -0,0 +1,185 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::{ + KamuApiServerClient, + KamuApiServerClientExt, + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, +}; +use kamu_cli_puppet::KamuCliPuppet; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_login_logout_password(kamu_node_api_client: KamuApiServerClient) { + let kamu_node_url = kamu_node_api_client.get_base_url().as_str(); + let kamu = KamuCliPuppet::new_workspace_tmp().await; + + { + let assert = kamu.execute(["logout", kamu_node_url]).await.success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(format!("Not logged in to {kamu_node_url}").as_str()), + "Unexpected output:\n{stderr}", + ); + } + { + let assert = kamu + .execute(["login", kamu_node_url, "--check"]) + .await + .failure(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(format!("Error: No access token found for: {kamu_node_url}").as_str()), + "Unexpected output:\n{stderr}", + ); + } + + { + let assert = kamu + .execute(["login", "password", "kamu", "kamu", kamu_node_url]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(format!("Login successful: {kamu_node_url}").as_str()), + "Unexpected output:\n{stderr}", + ); + } + { + let assert = kamu + .execute(["login", kamu_node_url, "--check"]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(format!("Access token valid: {kamu_node_url}").as_str()), + "Unexpected output:\n{stderr}", + ); + } + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + // Token validation, via an API call that requires authorization + { + let assert = kamu + .execute([ + "push", + "player-scores", + "--to", + &format!("odf+{kamu_node_url}player-scores"), + ]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("1 dataset(s) pushed"), + "Unexpected output:\n{stderr}", + ); + } + { + let assert = kamu.execute(["logout", kamu_node_url]).await.success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(format!("Logged out of {kamu_node_url}").as_str()), + "Unexpected output:\n{stderr}", + ); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_login_logout_oauth(kamu_node_api_client: KamuApiServerClient) { + let kamu_node_url = kamu_node_api_client.get_base_url().as_str(); + let kamu = KamuCliPuppet::new_workspace_tmp().await; + + { + let assert = kamu.execute(["logout", kamu_node_url]).await.success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(format!("Not logged in to {kamu_node_url}").as_str()), + "Unexpected output:\n{stderr}", + ); + } + { + let assert = kamu + .execute(["login", kamu_node_url, "--check"]) + .await + .failure(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(format!("Error: No access token found for: {kamu_node_url}").as_str()), + "Unexpected output:\n{stderr}", + ); + } + + let oauth_token = kamu_node_api_client.login_as_e2e_user().await; + + { + let assert = kamu + .execute(["login", "oauth", "github", &oauth_token, kamu_node_url]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(format!("Login successful: {kamu_node_url}").as_str()), + "Unexpected output:\n{stderr}", + ); + } + { + let assert = kamu + .execute(["login", kamu_node_url, "--check"]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(format!("Access token valid: {kamu_node_url}").as_str()), + "Unexpected output:\n{stderr}", + ); + } + + // Token validation, via an API call that requires authorization + kamu_node_api_client + .create_player_scores_dataset(&oauth_token) + .await; + + { + let assert = kamu.execute(["logout", kamu_node_url]).await.success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(format!("Logged out of {kamu_node_url}").as_str()), + "Unexpected output:\n{stderr}", + ); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_new_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_new_command.rs new file mode 100644 index 000000000..7d9db6442 --- /dev/null +++ b/src/e2e/app/cli/repo-tests/src/commands/test_new_command.rs @@ -0,0 +1,68 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_puppet::KamuCliPuppet; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_new_root(kamu: KamuCliPuppet) { + let assert = kamu + .execute(["new", "--root", "test-dataset"]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!( + r#" + Written new manifest template to: test-dataset.yaml + Follow directions in the file's comments and use `kamu add test-dataset.yaml` when ready. + "# + )), + "Unexpected output:\n{stderr}", + ); + + // TODO: After solving this issue, add `kamu add` calls and populate with + // data + // + // `kamu new`: generate snapshots that will be immediately ready to be + // added/worked on + // https://github.com/kamu-data/kamu-cli/issues/888 +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_new_derivative(kamu: KamuCliPuppet) { + let assert = kamu + .execute(["new", "--derivative", "test-dataset"]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!( + r#" + Written new manifest template to: test-dataset.yaml + Follow directions in the file's comments and use `kamu add test-dataset.yaml` when ready. + "# + )), + "Unexpected output:\n{stderr}", + ); + + // TODO: After solving this issue, add `kamu add` calls and populate with + // data + // + // `kamu new`: generate snapshots that will be immediately ready to be + // added/worked on + // https://github.com/kamu-data/kamu-cli/issues/888 +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_repo_alias_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_repo_command.rs similarity index 97% rename from src/e2e/app/cli/repo-tests/src/commands/test_repo_alias_command.rs rename to src/e2e/app/cli/repo-tests/src/commands/test_repo_command.rs index 154dc993e..bda9155e3 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_repo_alias_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_repo_command.rs @@ -87,6 +87,8 @@ pub async fn test_repository_pull_aliases_commands(kamu: KamuCliPuppet) { assert!(aliases.is_empty()); } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + pub async fn test_repository_push_aliases_commands(kamu: KamuCliPuppet) { kamu.add_dataset(DatasetSnapshot { name: "foo".try_into().unwrap(), diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_reset_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_reset_command.rs new file mode 100644 index 000000000..de1f02d0a --- /dev/null +++ b/src/e2e/app/cli/repo-tests/src/commands/test_reset_command.rs @@ -0,0 +1,87 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::assert_matches::assert_matches; + +use kamu_cli_e2e_common::{ + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, +}; +use kamu_cli_puppet::extensions::KamuCliPuppetExt; +use kamu_cli_puppet::KamuCliPuppet; +use opendatafabric::{DatasetName, MetadataEvent}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_reset(kamu: KamuCliPuppet) { + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + let block_records_after_ingesting = kamu + .list_blocks(&DatasetName::new_unchecked("player-scores")) + .await; + + pretty_assertions::assert_eq!(3, block_records_after_ingesting.len()); + + let set_vocab_block_record = &block_records_after_ingesting[0]; + + assert_matches!( + &set_vocab_block_record.block.event, + MetadataEvent::SetVocab(_), + ); + + pretty_assertions::assert_eq!(3, block_records_after_ingesting.len()); + + kamu.execute_with_input( + ["ingest", "player-scores", "--stdin"], + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await + .success(); + + pretty_assertions::assert_eq!( + 5, + kamu.list_blocks(&DatasetName::new_unchecked("player-scores")) + .await + .len() + ); + + let set_vocab_block_hash = set_vocab_block_record + .block_hash + .as_multibase() + .to_stack_string(); + + { + let assert = kamu + .execute([ + "--yes", + "reset", + "player-scores", + set_vocab_block_hash.as_str(), + ]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Dataset was reset"), + "Unexpected output:\n{stderr}", + ); + } + + let block_records_after_resetting = kamu + .list_blocks(&DatasetName::new_unchecked("player-scores")) + .await; + + pretty_assertions::assert_eq!(block_records_after_ingesting, block_records_after_resetting); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_search_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_search_command.rs new file mode 100644 index 000000000..8be9d1df4 --- /dev/null +++ b/src/e2e/app/cli/repo-tests/src/commands/test_search_command.rs @@ -0,0 +1,382 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::{ + KamuApiServerClient, + KamuApiServerClientExt, + RequestBody, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, +}; +use kamu_cli_puppet::KamuCliPuppet; +use opendatafabric::*; +use reqwest::Url; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_search_multi_user(kamu_node_api_client: KamuApiServerClient) { + let kamu = KamuCliPuppet::new_workspace_tmp().await; + + add_repo_to_workspace(&kamu_node_api_client, &kamu, "kamu-node").await; + + assert_search( + &kamu, + ["search", "player", "--output-format", "table"], + indoc::indoc!( + r#" + ┌───────┬──────┬─────────────┬────────┬─────────┬──────┐ + │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ + ├───────┼──────┼─────────────┼────────┼─────────┼──────┤ + │ │ │ │ │ │ │ + └───────┴──────┴─────────────┴────────┴─────────┴──────┘ + "# + ), + ) + .await; + + let e2e_user_token = kamu_node_api_client.login_as_e2e_user().await; + + kamu_node_api_client + .create_player_scores_dataset(&e2e_user_token) + .await; + + assert_search( + &kamu, + ["search", "player", "--output-format", "table"], + indoc::indoc!( + r#" + ┌──────────────────────────────────┬──────┬─────────────┬────────┬─────────┬──────┐ + │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ + ├──────────────────────────────────┼──────┼─────────────┼────────┼─────────┼──────┤ + │ kamu-node/e2e-user/player-scores │ Root │ - │ 3 │ - │ - │ + └──────────────────────────────────┴──────┴─────────────┴────────┴─────────┴──────┘ + "# + ), + ) + .await; + + let player_scores_alias = DatasetAlias::new( + Some(AccountName::new_unchecked("e2e-user")), + DatasetName::new_unchecked("player-scores"), + ); + + kamu_node_api_client + .ingest_data( + &player_scores_alias, + RequestBody::NdJson(DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1.into()), + &e2e_user_token, + ) + .await; + + assert_search( + &kamu, + ["search", "player", "--output-format", "table"], + indoc::indoc!( + r#" + ┌──────────────────────────────────┬──────┬─────────────┬────────┬─────────┬──────────┐ + │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ + ├──────────────────────────────────┼──────┼─────────────┼────────┼─────────┼──────────┤ + │ kamu-node/e2e-user/player-scores │ Root │ - │ 5 │ 2 │ 1.63 KiB │ + └──────────────────────────────────┴──────┴─────────────┴────────┴─────────┴──────────┘ + "# + ), + ) + .await; + + // The same as DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT, but contains the word + // "player" in the name so that it can be found together with "player-scores" + let dataset_derivative_player_leaderboard_snapshot = indoc::indoc!( + r#" + kind: DatasetSnapshot + version: 1 + content: + name: player-leaderboard + kind: Derivative + metadata: + - kind: SetTransform + inputs: + - datasetRef: player-scores + alias: player_scores + transform: + kind: Sql + engine: risingwave + queries: + - alias: leaderboard + query: | + create materialized view leaderboard as + select + * + from ( + select + row_number() over (partition by 1 order by score desc) as place, + match_time, + match_id, + player_id, + score + from player_scores + ) + where place <= 2 + - query: | + select * from leaderboard + - kind: SetVocab + eventTimeColumn: match_time + "# + ) + .escape_default() + .to_string(); + + kamu_node_api_client + .create_dataset( + &dataset_derivative_player_leaderboard_snapshot, + &e2e_user_token, + ) + .await; + + assert_search( + &kamu, + ["search", "player", "--output-format", "table"], + indoc::indoc!( + r#" + ┌───────────────────────────────────────┬────────────┬─────────────┬────────┬─────────┬──────────┐ + │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ + ├───────────────────────────────────────┼────────────┼─────────────┼────────┼─────────┼──────────┤ + │ kamu-node/e2e-user/player-leaderboard │ Derivative │ - │ 3 │ - │ - │ + │ kamu-node/e2e-user/player-scores │ Root │ - │ 5 │ 2 │ 1.63 KiB │ + └───────────────────────────────────────┴────────────┴─────────────┴────────┴─────────┴──────────┘ + "# + ), + ) + .await; + + let kamu_token = kamu_node_api_client.login_as_kamu().await; + + kamu_node_api_client + .create_player_scores_dataset(&kamu_token) + .await; + + assert_search( + &kamu, + ["search", "player", "--output-format", "table"], + indoc::indoc!( + r#" + ┌───────────────────────────────────────┬────────────┬─────────────┬────────┬─────────┬──────────┐ + │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ + ├───────────────────────────────────────┼────────────┼─────────────┼────────┼─────────┼──────────┤ + │ kamu-node/e2e-user/player-leaderboard │ Derivative │ - │ 3 │ - │ - │ + │ kamu-node/e2e-user/player-scores │ Root │ - │ 5 │ 2 │ 1.63 KiB │ + │ kamu-node/kamu/player-scores │ Root │ - │ 3 │ - │ - │ + └───────────────────────────────────────┴────────────┴─────────────┴────────┴─────────┴──────────┘ + "# + ), + ) + .await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_search_by_name(kamu_node_api_client: KamuApiServerClient) { + let kamu = KamuCliPuppet::new_workspace_tmp().await; + + add_repo_to_workspace(&kamu_node_api_client, &kamu, "kamu-node").await; + + let e2e_user_token = kamu_node_api_client.login_as_e2e_user().await; + + kamu_node_api_client + .create_player_scores_dataset(&e2e_user_token) + .await; + + kamu_node_api_client + .create_leaderboard(&e2e_user_token) + .await; + + assert_search( + &kamu, + ["search", "player", "--output-format", "table"], + indoc::indoc!( + r#" + ┌──────────────────────────────────┬──────┬─────────────┬────────┬─────────┬──────┐ + │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ + ├──────────────────────────────────┼──────┼─────────────┼────────┼─────────┼──────┤ + │ kamu-node/e2e-user/player-scores │ Root │ - │ 3 │ - │ - │ + └──────────────────────────────────┴──────┴─────────────┴────────┴─────────┴──────┘ + "# + ), + ) + .await; + + assert_search( + &kamu, + ["search", "scores", "--output-format", "table"], + indoc::indoc!( + r#" + ┌──────────────────────────────────┬──────┬─────────────┬────────┬─────────┬──────┐ + │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ + ├──────────────────────────────────┼──────┼─────────────┼────────┼─────────┼──────┤ + │ kamu-node/e2e-user/player-scores │ Root │ - │ 3 │ - │ - │ + └──────────────────────────────────┴──────┴─────────────┴────────┴─────────┴──────┘ + "# + ), + ) + .await; + + assert_search( + &kamu, + ["search", "not-relevant-query", "--output-format", "table"], + indoc::indoc!( + r#" + ┌───────┬──────┬─────────────┬────────┬─────────┬──────┐ + │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ + ├───────┼──────┼─────────────┼────────┼─────────┼──────┤ + │ │ │ │ │ │ │ + └───────┴──────┴─────────────┴────────┴─────────┴──────┘ + "# + ), + ) + .await; + + assert_search( + &kamu, + ["search", "lead", "--output-format", "table"], + indoc::indoc!( + r#" + ┌────────────────────────────────┬────────────┬─────────────┬────────┬─────────┬──────┐ + │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ + ├────────────────────────────────┼────────────┼─────────────┼────────┼─────────┼──────┤ + │ kamu-node/e2e-user/leaderboard │ Derivative │ - │ 3 │ - │ - │ + └────────────────────────────────┴────────────┴─────────────┴────────┴─────────┴──────┘ + "# + ), + ) + .await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_search_by_repo(kamu_node_api_client: KamuApiServerClient) { + let kamu = KamuCliPuppet::new_workspace_tmp().await; + + // As a test, add two repos pointing to the same node + add_repo_to_workspace(&kamu_node_api_client, &kamu, "kamu-node").await; + add_repo_to_workspace(&kamu_node_api_client, &kamu, "acme-org-node").await; + + let e2e_user_token = kamu_node_api_client.login_as_e2e_user().await; + + kamu_node_api_client + .create_player_scores_dataset(&e2e_user_token) + .await; + + kamu_node_api_client + .create_leaderboard(&e2e_user_token) + .await; + + assert_search( + &kamu, + ["search", "player", "--output-format", "table"], + indoc::indoc!( + r#" + ┌──────────────────────────────────────┬──────┬─────────────┬────────┬─────────┬──────┐ + │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ + ├──────────────────────────────────────┼──────┼─────────────┼────────┼─────────┼──────┤ + │ acme-org-node/e2e-user/player-scores │ Root │ - │ 3 │ - │ - │ + │ kamu-node/e2e-user/player-scores │ Root │ - │ 3 │ - │ - │ + └──────────────────────────────────────┴──────┴─────────────┴────────┴─────────┴──────┘ + "# + ), + ) + .await; + + assert_search( + &kamu, + [ + "search", + "player", + "--repo", + "acme-org-node", + "--output-format", + "table", + ], + indoc::indoc!( + r#" + ┌──────────────────────────────────────┬──────┬─────────────┬────────┬─────────┬──────┐ + │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ + ├──────────────────────────────────────┼──────┼─────────────┼────────┼─────────┼──────┤ + │ acme-org-node/e2e-user/player-scores │ Root │ - │ 3 │ - │ - │ + └──────────────────────────────────────┴──────┴─────────────┴────────┴─────────┴──────┘ + "# + ), + ) + .await; + + assert_search( + &kamu, + [ + "search", + "player", + "--repo", + "kamu-node", + "--output-format", + "table", + ], + indoc::indoc!( + r#" + ┌──────────────────────────────────┬──────┬─────────────┬────────┬─────────┬──────┐ + │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ + ├──────────────────────────────────┼──────┼─────────────┼────────┼─────────┼──────┤ + │ kamu-node/e2e-user/player-scores │ Root │ - │ 3 │ - │ - │ + └──────────────────────────────────┴──────┴─────────────┴────────┴─────────┴──────┘ + "# + ), + ) + .await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Helpers +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +async fn add_repo_to_workspace( + kamu_node_api_client: &KamuApiServerClient, + kamu: &KamuCliPuppet, + repo_name: &str, +) { + let http_repo = { + let mut url = Url::parse("odf+http://host").unwrap(); + let base_url = kamu_node_api_client.get_base_url(); + url.set_host(base_url.host_str()).unwrap(); + url.set_port(base_url.port()).unwrap(); + url + }; + + let assert = kamu + .execute(["repo", "add", repo_name, http_repo.as_str()]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(format!("Added: {repo_name}").as_str()), + "Unexpected output:\n{stderr}", + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +async fn assert_search(kamu: &KamuCliPuppet, search_cmd: I, expected_table_output: &str) +where + I: IntoIterator, + S: AsRef, +{ + let assert = kamu.execute(search_cmd).await.success(); + + let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + pretty_assertions::assert_eq!(stdout, expected_table_output); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_sql_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_sql_command.rs index abe16c63d..24f3576ab 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_sql_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_sql_command.rs @@ -8,6 +8,10 @@ // by the Apache License, Version 2.0. use indoc::indoc; +use kamu_cli_e2e_common::{ + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, +}; use kamu_cli_puppet::KamuCliPuppet; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -45,7 +49,117 @@ pub async fn test_datafusion_cli_not_launched_in_root_ws(kamu: KamuCliPuppet) { // The workspace search functionality checks for parent folders, // so there is no problem that the process working directory is one of the // subdirectories (kamu-cli/src/e2e/app/cli/inmem) - kamu.execute(["list"]).await.failure(); + + { + let assert = kamu.execute(["list"]).await.failure(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Error: Directory is not a kamu workspace"), + "Unexpected output:\n{stderr}", + ); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_sql_command(kamu: KamuCliPuppet) { + { + let assert = kamu + .execute([ + "sql", + "--command", + "SELECT 42 as answer;", + "--output-format", + "table", + ]) + .await + .success(); + + let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + pretty_assertions::assert_eq!( + indoc::indoc!( + r#" + ┌────────┐ + │ answer │ + ├────────┤ + │ 42 │ + └────────┘ + "# + ), + stdout + ); + } + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + { + let assert = kamu + .execute([ + "sql", + "--command", + "SELECT * FROM \"player-scores\";", + "--output-format", + "table", + ]) + .await + .success(); + + let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + pretty_assertions::assert_eq!( + indoc::indoc!( + r#" + ┌┐ + ││ + ├┤ + ││ + └┘ + "# + ), + stdout + ); + } + + kamu.execute_with_input( + ["ingest", "player-scores", "--stdin"], + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await + .success(); + + { + let assert = kamu + .execute([ + "sql", + "--command", + "SELECT * FROM \"player-scores\";", + "--output-format", + "table", + ]) + .await + .success(); + + let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + pretty_assertions::assert_eq!( + indoc::indoc!( + r#" + ┌────────┬────┬──────────────────────┬──────────────────────────┬──────────┬───────────┬───────┐ + │ offset │ op │ system_time │ match_time │ match_id │ player_id │ score │ + ├────────┼────┼──────────────────────┼──────────────────────────┼──────────┼───────────┼───────┤ + │ 0 │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ + │ 1 │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00.001Z │ 1 │ Bob │ 80 │ + └────────┴────┴──────────────────────┴──────────────────────────┴──────────┴───────────┴───────┘ + "# + ), + stdout + ); + } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_system_gc_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_system_gc_command.rs new file mode 100644 index 000000000..862f7c92f --- /dev/null +++ b/src/e2e/app/cli/repo-tests/src/commands/test_system_gc_command.rs @@ -0,0 +1,25 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_puppet::KamuCliPuppet; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_gc(kamu: KamuCliPuppet) { + let assert = kamu.execute(["system", "gc"]).await.success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Cleaning cache...") && stderr.contains("Workspace is already clean"), + "Unexpected output:\n{stderr}", + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_system_info_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_system_info_command.rs new file mode 100644 index 000000000..d0f8d206e --- /dev/null +++ b/src/e2e/app/cli/repo-tests/src/commands/test_system_info_command.rs @@ -0,0 +1,18 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_puppet::KamuCliPuppet; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_system_info(kamu: KamuCliPuppet) { + kamu.execute(["system", "info"]).await.success(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_system_info_diagnose.rs b/src/e2e/app/cli/repo-tests/src/commands/test_system_info_diagnose.rs new file mode 100644 index 000000000..9ba38606d --- /dev/null +++ b/src/e2e/app/cli/repo-tests/src/commands/test_system_info_diagnose.rs @@ -0,0 +1,18 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_puppet::KamuCliPuppet; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_system_diagnose(kamu: KamuCliPuppet) { + kamu.execute(["system", "diagnose"]).await.success(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_tail_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_tail_command.rs new file mode 100644 index 000000000..54242ad84 --- /dev/null +++ b/src/e2e/app/cli/repo-tests/src/commands/test_tail_command.rs @@ -0,0 +1,68 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::{ + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, +}; +use kamu_cli_puppet::KamuCliPuppet; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_tail(kamu: KamuCliPuppet) { + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + { + let assert = kamu + .execute(["tail", "player-scores", "--output-format", "table"]) + .await + .failure(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains("Error: Dataset schema is not yet available: player-scores"), + "Unexpected output:\n{stderr}", + ); + } + + kamu.execute_with_input( + ["ingest", "player-scores", "--stdin"], + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await + .success(); + + { + let assert = kamu + .execute(["tail", "player-scores", "--output-format", "table"]) + .await + .success(); + + let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + pretty_assertions::assert_eq!( + indoc::indoc!( + r#" + ┌────────┬────┬──────────────────────┬──────────────────────────┬──────────┬───────────┬───────┐ + │ offset │ op │ system_time │ match_time │ match_id │ player_id │ score │ + ├────────┼────┼──────────────────────┼──────────────────────────┼──────────┼───────────┼───────┤ + │ 0 │ +A │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ + │ 1 │ +A │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00.001Z │ 1 │ Bob │ 80 │ + └────────┴────┴──────────────────────┴──────────────────────────┴──────────┴───────────┴───────┘ + "# + ), + stdout + ); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/lib.rs b/src/e2e/app/cli/repo-tests/src/lib.rs index 2bd7d97f9..51d5ab6b2 100644 --- a/src/e2e/app/cli/repo-tests/src/lib.rs +++ b/src/e2e/app/cli/repo-tests/src/lib.rs @@ -7,6 +7,8 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +#![feature(assert_matches)] + mod commands; mod test_auth; mod test_flow; diff --git a/src/e2e/app/cli/repo-tests/src/test_flow.rs b/src/e2e/app/cli/repo-tests/src/test_flow.rs index 855724fc5..aae45ade0 100644 --- a/src/e2e/app/cli/repo-tests/src/test_flow.rs +++ b/src/e2e/app/cli/repo-tests/src/test_flow.rs @@ -615,7 +615,7 @@ pub async fn test_dataset_trigger_flow(kamu_api_server_client: KamuApiServerClie "__typename": "FlowDescriptionUpdateResultSuccess", "numBlocks": 2, "numRecords": 2, - "updatedWatermark": "2000-01-01T00:00:00+00:00" + "updatedWatermark": "2000-01-01T00:00:00.001+00:00" } }, "initiator": { @@ -649,7 +649,7 @@ pub async fn test_dataset_trigger_flow(kamu_api_server_client: KamuApiServerClie "__typename": "FlowDescriptionUpdateResultSuccess", "numBlocks": 2, "numRecords": 2, - "updatedWatermark": "2000-01-01T00:00:00+00:00" + "updatedWatermark": "2000-01-01T00:00:00.001+00:00" } }, "initiator": { diff --git a/src/infra/core/src/query_service_impl.rs b/src/infra/core/src/query_service_impl.rs index 352460f98..d38dd1f9f 100644 --- a/src/infra/core/src/query_service_impl.rs +++ b/src/infra/core/src/query_service_impl.rs @@ -56,7 +56,7 @@ impl QueryServiceImpl { .with_information_schema(true) .with_default_catalog_and_schema("kamu", "kamu"); - // Forcing cese-sensitive identifiers in case-insensitive language seems to + // Forcing case-sensitive identifiers in case-insensitive language seems to // be a lesser evil than following DataFusion's default behavior of forcing // identifiers to lowercase instead of case-insensitive matching. // diff --git a/src/infra/core/src/remote_repository_registry_impl.rs b/src/infra/core/src/remote_repository_registry_impl.rs index cd48a2063..f80402d1d 100644 --- a/src/infra/core/src/remote_repository_registry_impl.rs +++ b/src/infra/core/src/remote_repository_registry_impl.rs @@ -44,7 +44,7 @@ impl RemoteRepositoryRegistryImpl { let file_path = self.repos_dir.join(repo_name); if !file_path.exists() { - // run full scan to support case-insensetive matches + // run full scan to support case-insensitive matches let all_repositories_stream = self.get_all_repositories(); for repository_name in all_repositories_stream { if &repository_name == repo_name { @@ -157,6 +157,7 @@ impl RemoteRepositoryRegistry for RemoteRepositoryRegistryNull { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Config //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + pub struct RemoteReposDir(PathBuf); impl RemoteReposDir { diff --git a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet.rs b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet.rs index 422b4c1dd..915311fc2 100644 --- a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet.rs +++ b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet.rs @@ -14,6 +14,10 @@ use chrono::{DateTime, Utc}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +pub type ExecuteCommandResult = assert_cmd::assert::Assert; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + pub struct KamuCliPuppet { workspace_path: PathBuf, system_time: Option>, @@ -88,7 +92,7 @@ impl KamuCliPuppet { temp_dir.join("e2e-output-data.txt") } - pub async fn execute(&self, cmd: I) -> assert_cmd::assert::Assert + pub async fn execute(&self, cmd: I) -> ExecuteCommandResult where I: IntoIterator, S: AsRef, @@ -96,7 +100,7 @@ impl KamuCliPuppet { self.execute_impl(cmd, None::>).await } - pub async fn execute_with_input(&self, cmd: I, input: T) -> assert_cmd::assert::Assert + pub async fn execute_with_input(&self, cmd: I, input: T) -> ExecuteCommandResult where I: IntoIterator, S: AsRef, @@ -105,11 +109,7 @@ impl KamuCliPuppet { self.execute_impl(cmd, Some(input)).await } - async fn execute_impl( - &self, - cmd: I, - maybe_input: Option, - ) -> assert_cmd::assert::Assert + async fn execute_impl(&self, cmd: I, maybe_input: Option) -> ExecuteCommandResult where I: IntoIterator, S: AsRef, diff --git a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs index 09ae75317..8c9292bf7 100644 --- a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs +++ b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs @@ -14,9 +14,21 @@ use std::path::PathBuf; use async_trait::async_trait; use chrono::{DateTime, Utc}; use datafusion::prelude::{ParquetReadOptions, SessionContext}; -use opendatafabric::serde::yaml::{DatasetKindDef, YamlDatasetSnapshotSerializer}; -use opendatafabric::serde::DatasetSnapshotSerializer; -use opendatafabric::{DatasetID, DatasetKind, DatasetName, DatasetRef, DatasetSnapshot, Multihash}; +use opendatafabric::serde::yaml::{ + DatasetKindDef, + YamlDatasetSnapshotSerializer, + YamlMetadataBlockDeserializer, +}; +use opendatafabric::serde::{DatasetSnapshotSerializer, MetadataBlockDeserializer}; +use opendatafabric::{ + DatasetID, + DatasetKind, + DatasetName, + DatasetRef, + DatasetSnapshot, + MetadataBlock, + Multihash, +}; use serde::Deserialize; use crate::KamuCliPuppet; @@ -35,6 +47,8 @@ pub trait KamuCliPuppetExt { where T: Into + Send; + async fn list_blocks(&self, dataset_name: &DatasetName) -> Vec; + async fn start_api_server(self, e2e_data_file_path: PathBuf) -> ServerOutput; async fn assert_last_data_slice( @@ -111,6 +125,43 @@ impl KamuCliPuppetExt for KamuCliPuppet { stdout.lines().map(ToString::to_string).collect() } + async fn list_blocks(&self, dataset_name: &DatasetName) -> Vec { + let assert = self + .execute(["log", dataset_name.as_str(), "--output-format", "yaml"]) + .await + .success(); + + let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + // TODO: Don't parse the output, after implementation: + // `kamu log`: support `--output-format json` + // https://github.com/kamu-data/kamu-cli/issues/887 + + stdout + .split("---") + .skip(1) + .map(str::trim) + .map(|block_data| { + let Some(pos) = block_data.find('\n') else { + unreachable!() + }; + let (first_line_with_block_hash, metadata_block_str) = block_data.split_at(pos); + + let block_hash = first_line_with_block_hash + .strip_prefix("# Block: ") + .unwrap(); + let block = YamlMetadataBlockDeserializer {} + .read_manifest(metadata_block_str.as_ref()) + .unwrap(); + + BlockRecord { + block_hash: Multihash::from_multibase(block_hash).unwrap(), + block, + } + }) + .collect() + } + async fn start_api_server(self, e2e_data_file_path: PathBuf) -> ServerOutput { let host = Ipv4Addr::LOCALHOST.to_string(); @@ -211,4 +262,10 @@ pub struct RepoAlias { pub alias: String, } +#[derive(Debug, PartialEq, Eq)] +pub struct BlockRecord { + pub block_hash: Multihash, + pub block: MetadataBlock, +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// From a3e19e88700cd2f56d2317466a9f3f6d46c8be70 Mon Sep 17 00:00:00 2001 From: Roman Boiko Date: Mon, 14 Oct 2024 14:46:13 +0200 Subject: [PATCH 03/13] `Push, pull, compact, verify` commands e2e tests (#893) * Add query commitments example * Release (patch) `0.204.5`: fix `--yes / -y` flag: fixed when working from a TTY (#881) * Interact: fix if is_tty * CHANGELOG.md: update * Release (patch): 0.204.5 * Updated yanked crate (futures-util to 0.3.31) * CI: Fixes `kamu-base-with-data-mt` image builds (#885) * Images, kamu-base-with-data-mt: add "kamu" to predefined users * Images, kamu-base-with-data-mt: init-workspace.py use .kamuconfig * CHANGELOG.md: update * Add push pull tests * Add s3 tests * Remove unused code * Test transform engine error * Revert check * Add container extra groups * Add verify and compact commands * Fix grammar * Fix review commets * Revert "Merge branch 'master' into chore/push-and-pul-command-e2e-tests" This reverts commit a4ec39c4d5d891aed60b93bb813083a72c958e0c, reversing changes made to 1130fa5bc1f8816d0b2968b1fdb96e135dea55bf. * Clean code * Fix review comments. Iter 2 * Revert ToDo comment --------- Co-authored-by: Sergii Mikhtoniuk Co-authored-by: Dima Pristupa --- src/app/cli/src/commands/verify_command.rs | 7 +- .../common/src/kamu_api_server_client_ext.rs | 24 +- .../app/cli/inmem/tests/tests/commands/mod.rs | 2 + .../tests/commands/test_compact_command.rs | 36 + .../tests/commands/test_verify_command.rs | 36 + .../tests/test_smart_transfer_protocol.rs | 101 ++ .../app/cli/repo-tests/src/commands/mod.rs | 4 + .../src/commands/test_compact_command.rs | 200 +++ .../src/commands/test_verify_command.rs | 150 ++ src/e2e/app/cli/repo-tests/src/test_flow.rs | 8 +- .../src/test_smart_transfer_protocol.rs | 1314 ++++++++++++++++- .../src/kamu_cli_puppet_ext.rs | 30 +- 12 files changed, 1815 insertions(+), 97 deletions(-) create mode 100644 src/e2e/app/cli/inmem/tests/tests/commands/test_compact_command.rs create mode 100644 src/e2e/app/cli/inmem/tests/tests/commands/test_verify_command.rs create mode 100644 src/e2e/app/cli/repo-tests/src/commands/test_compact_command.rs create mode 100644 src/e2e/app/cli/repo-tests/src/commands/test_verify_command.rs diff --git a/src/app/cli/src/commands/verify_command.rs b/src/app/cli/src/commands/verify_command.rs index 433dcc395..ac3d25778 100644 --- a/src/app/cli/src/commands/verify_command.rs +++ b/src/app/cli/src/commands/verify_command.rs @@ -152,7 +152,6 @@ impl VerifyCommand { Ok(self .verification_svc - .clone() .verify_multi(filtered_requests, options, listener) .await) } @@ -193,14 +192,14 @@ impl VerifyCommand { let mut current_missed_dependencies = vec![]; - for dependecy in summary.dependencies { + for dependency in summary.dependencies { if self .dataset_repo - .resolve_dataset_ref(&DatasetRef::ID(dependecy.clone())) + .resolve_dataset_ref(&DatasetRef::ID(dependency.clone())) .await .is_err() { - current_missed_dependencies.push(dependecy.to_string()); + current_missed_dependencies.push(dependency.to_string()); } } if !current_missed_dependencies.is_empty() { diff --git a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs index a8a53ac4e..ef4fc862e 100644 --- a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs +++ b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs @@ -9,7 +9,7 @@ use async_trait::async_trait; use lazy_static::lazy_static; -use opendatafabric::{DatasetAlias, DatasetName}; +use opendatafabric::{AccountName, DatasetAlias, DatasetName}; use reqwest::{Method, StatusCode}; use crate::{KamuApiServerClient, RequestBody}; @@ -134,6 +134,8 @@ pub const DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_3: &str = indoc::i "# ); +pub const E2E_USER_ACCOUNT_NAME_STR: &str = "e2e-user"; + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub type AccessToken = String; @@ -153,8 +155,11 @@ pub trait KamuApiServerClientExt { async fn create_player_scores_dataset(&self, token: &AccessToken) -> DatasetId; - /// NOTE: only for single-tenant workspaces - async fn create_player_scores_dataset_with_data(&self, token: &AccessToken) -> DatasetId; + async fn create_player_scores_dataset_with_data( + &self, + token: &AccessToken, + account_name_maybe: Option, + ) -> DatasetId; async fn create_leaderboard(&self, token: &AccessToken) -> DatasetId; @@ -249,14 +254,19 @@ impl KamuApiServerClientExt for KamuApiServerClient { .await } - async fn create_player_scores_dataset_with_data(&self, token: &AccessToken) -> DatasetId { + async fn create_player_scores_dataset_with_data( + &self, + token: &AccessToken, + account_name_maybe: Option, + ) -> DatasetId { let dataset_id = self.create_player_scores_dataset(token).await; // TODO: Use the alias from the reply, after fixing the bug: // https://github.com/kamu-data/kamu-cli/issues/891 - - // At the moment, only single-tenant - let dataset_alias = DatasetAlias::new(None, DatasetName::new_unchecked("player-scores")); + let dataset_alias = DatasetAlias::new( + account_name_maybe, + DatasetName::new_unchecked("player-scores"), + ); self.ingest_data( &dataset_alias, diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/mod.rs b/src/e2e/app/cli/inmem/tests/tests/commands/mod.rs index 52308e3fa..406d408d3 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/mod.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/mod.rs @@ -8,6 +8,7 @@ // by the Apache License, Version 2.0. mod test_add_command; +mod test_compact_command; mod test_complete_command; mod test_config_command; mod test_delete_command; @@ -28,3 +29,4 @@ mod test_system_gc_command; mod test_system_generate_token_command; mod test_system_info_command; mod test_tail_command; +mod test_verify_command; diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_compact_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_compact_command.rs new file mode 100644 index 000000000..6ff90ee7b --- /dev/null +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_compact_command.rs @@ -0,0 +1,36 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_compact_hard + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_compact_keep_metadata_only + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_compact_verify + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_verify_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_verify_command.rs new file mode 100644 index 000000000..0a4b5598a --- /dev/null +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_verify_command.rs @@ -0,0 +1,36 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_verify_regular_dataset + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_verify_recursive + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_verify_integrity + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs b/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs index dfa4094c2..abb5120d3 100644 --- a/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs @@ -24,3 +24,104 @@ kamu_cli_run_api_server_e2e_test!( ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_force_push_pull, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_add_alias, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_as, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_all, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_recursive, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_set_watermark, + options = Options::default().with_frozen_system_time(), +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_reset_derivative, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_visibility, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_s3, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_derivative, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/mod.rs b/src/e2e/app/cli/repo-tests/src/commands/mod.rs index 5125a93ce..ef14ffc19 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/mod.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/mod.rs @@ -8,6 +8,7 @@ // by the Apache License, Version 2.0. mod test_add_command; +mod test_compact_command; mod test_complete_command; mod test_config_command; mod test_delete_command; @@ -28,8 +29,10 @@ mod test_system_generate_token_command; mod test_system_info_command; mod test_system_info_diagnose; mod test_tail_command; +mod test_verify_command; pub use test_add_command::*; +pub use test_compact_command::*; pub use test_complete_command::*; pub use test_config_command::*; pub use test_delete_command::*; @@ -50,3 +53,4 @@ pub use test_system_generate_token_command::*; pub use test_system_info_command::*; pub use test_system_info_diagnose::*; pub use test_tail_command::*; +pub use test_verify_command::*; diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_compact_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_compact_command.rs new file mode 100644 index 000000000..316359eb4 --- /dev/null +++ b/src/e2e/app/cli/repo-tests/src/commands/test_compact_command.rs @@ -0,0 +1,200 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::assert_matches::assert_matches; + +use kamu_cli_e2e_common::{ + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, +}; +use kamu_cli_puppet::extensions::KamuCliPuppetExt; +use kamu_cli_puppet::KamuCliPuppet; +use opendatafabric::{DatasetName, MetadataEvent}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_compact_hard(kamu: KamuCliPuppet) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + ) + .await; + + let blocks_before_compacting = kamu.list_blocks(&dataset_name).await; + + let assert = kamu + .execute([ + "--yes", + "system", + "compact", + dataset_name.as_str(), + "--hard", + ]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!( + r#" + 1 dataset(s) were compacted + "# + )), + "Unexpected output:\n{stderr}", + ); + + let blocks_after_compacting = kamu.list_blocks(&dataset_name).await; + assert_eq!( + blocks_before_compacting.len() - 1, + blocks_after_compacting.len() + ); + assert_matches!( + blocks_after_compacting.first().unwrap().block.event, + MetadataEvent::AddData(_) + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_compact_keep_metadata_only(kamu: KamuCliPuppet) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + ) + .await; + + let blocks_before_compacting = kamu.list_blocks(&dataset_name).await; + + let assert = kamu + .execute([ + "--yes", + "system", + "compact", + dataset_name.as_str(), + "--hard", + "--keep-metadata-only", + ]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!( + r#" + 1 dataset(s) were compacted + "# + )), + "Unexpected output:\n{stderr}", + ); + + let blocks_after_compacting = kamu.list_blocks(&dataset_name).await; + assert_eq!( + blocks_before_compacting.len() - 2, + blocks_after_compacting.len() + ); + assert_matches!( + blocks_after_compacting.first().unwrap().block.event, + MetadataEvent::SetDataSchema(_) + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_compact_verify(kamu: KamuCliPuppet) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + ) + .await; + + let blocks_before_compacting = kamu.list_blocks(&dataset_name).await; + + let assert = kamu + .execute([ + "--yes", + "system", + "compact", + dataset_name.as_str(), + "--hard", + "--verify", + ]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains( + indoc::indoc!( + r#" + verify with dataset_ref: player-scores + "# + ) + .trim() + ), + "Unexpected output:\n{stderr}", + ); + + assert!( + stderr.contains(indoc::indoc!( + r#" + 1 dataset(s) were compacted + "# + )), + "Unexpected output:\n{stderr}", + ); + + let blocks_after_compacting = kamu.list_blocks(&dataset_name).await; + assert_eq!( + blocks_before_compacting.len() - 1, + blocks_after_compacting.len() + ); + assert_matches!( + blocks_after_compacting.first().unwrap().block.event, + MetadataEvent::AddData(_) + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_verify_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_verify_command.rs new file mode 100644 index 000000000..088c6e1b2 --- /dev/null +++ b/src/e2e/app/cli/repo-tests/src/commands/test_verify_command.rs @@ -0,0 +1,150 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::{ + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, +}; +use kamu_cli_puppet::extensions::KamuCliPuppetExt; +use kamu_cli_puppet::KamuCliPuppet; +use opendatafabric::DatasetName; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_verify_regular_dataset(kamu: KamuCliPuppet) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90} + "#, + ); + + kamu.ingest_data(&dataset_name, data).await; + + let assert = kamu + .execute(["verify", dataset_name.as_str()]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!( + r#" + 1 dataset(s) are valid + "# + )), + "Unexpected output:\n{stderr}", + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_verify_recursive(kamu: KamuCliPuppet) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + let dataset_derivative_name = DatasetName::new_unchecked("leaderboard"); + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + kamu.execute_with_input( + ["add", "--stdin"], + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + ) + .await + .success(); + + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90} + "#, + ); + + kamu.ingest_data(&dataset_name, data).await; + + kamu.execute(["pull", dataset_derivative_name.as_str()]) + .await + .success(); + + // Call verify without recursive flag + let assert = kamu + .execute(["verify", dataset_derivative_name.as_str()]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!( + r#" + 1 dataset(s) are valid + "# + )), + "Unexpected output:\n{stderr}", + ); + + // Call verify wit recursive flag + let assert = kamu + .execute(["verify", dataset_derivative_name.as_str(), "--recursive"]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!( + r#" + 2 dataset(s) are valid + "# + )), + "Unexpected output:\n{stderr}", + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_verify_integrity(kamu: KamuCliPuppet) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90} + "#, + ); + + kamu.ingest_data(&dataset_name, data).await; + + let assert = kamu + .execute(["verify", dataset_name.as_str(), "--integrity"]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!( + r#" + 1 dataset(s) are valid + "# + )), + "Unexpected output:\n{stderr}", + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/test_flow.rs b/src/e2e/app/cli/repo-tests/src/test_flow.rs index aae45ade0..eb96c840a 100644 --- a/src/e2e/app/cli/repo-tests/src/test_flow.rs +++ b/src/e2e/app/cli/repo-tests/src/test_flow.rs @@ -17,7 +17,7 @@ pub async fn test_get_dataset_list_flows(kamu_api_server_client: KamuApiServerCl let token = kamu_api_server_client.login_as_kamu().await; let dataset_id = kamu_api_server_client - .create_player_scores_dataset_with_data(&token) + .create_player_scores_dataset_with_data(&token, None) .await; // The query is almost identical to kamu-web-ui, for ease of later edits. @@ -88,7 +88,7 @@ pub async fn test_dataset_all_flows_paused(kamu_api_server_client: KamuApiServer let token = kamu_api_server_client.login_as_kamu().await; let dataset_id = kamu_api_server_client - .create_player_scores_dataset_with_data(&token) + .create_player_scores_dataset_with_data(&token, None) .await; // The query is almost identical to kamu-web-ui, for ease of later edits. @@ -147,7 +147,7 @@ pub async fn test_dataset_flows_initiators(kamu_api_server_client: KamuApiServer let token = kamu_api_server_client.login_as_kamu().await; let dataset_id = kamu_api_server_client - .create_player_scores_dataset_with_data(&token) + .create_player_scores_dataset_with_data(&token, None) .await; // The query is almost identical to kamu-web-ui, for ease of later edits. @@ -228,7 +228,7 @@ pub async fn test_dataset_trigger_flow(kamu_api_server_client: KamuApiServerClie let token = kamu_api_server_client.login_as_kamu().await; let _root_dataset_id = kamu_api_server_client - .create_player_scores_dataset_with_data(&token) + .create_player_scores_dataset_with_data(&token, None) .await; let derivative_dataset_id = kamu_api_server_client.create_leaderboard(&token).await; diff --git a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs index 4de9a1663..5deee2f3e 100644 --- a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs @@ -7,29 +7,110 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use kamu_cli_e2e_common::{KamuApiServerClient, KamuApiServerClientExt}; +use std::str::FromStr; + +use chrono::DateTime; +use kamu::testing::LocalS3Server; +use kamu_cli_e2e_common::{ + KamuApiServerClient, + KamuApiServerClientExt, + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, + E2E_USER_ACCOUNT_NAME_STR, +}; +use kamu_cli_puppet::extensions::{KamuCliPuppetExt, RepoAlias}; use kamu_cli_puppet::KamuCliPuppet; +use opendatafabric::{AccountName, DatasetName}; use reqwest::Url; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_smart_push_pull_sequence(kamu_api_server_client: KamuApiServerClient) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + // 1. Grub a token let token = kamu_api_server_client.login_as_e2e_user().await; - let kamu_api_server_dataset_endpoint = { - let base_url = kamu_api_server_client.get_base_url(); + let kamu_api_server_dataset_endpoint = get_dataset_endpoint( + kamu_api_server_client.get_base_url(), + &dataset_name, + E2E_USER_ACCOUNT_NAME_STR, + ); + + // 2. Pushing the dataset to the API server + { + let kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; + + // 2.1. Add the dataset + { + kamu_in_push_workspace + .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + } + + // 2.1. Ingest data to the dataset + { + kamu_in_push_workspace + .ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; + } + + // 2.2. Login to the API server + kamu_in_push_workspace + .execute([ + "login", + kamu_api_server_client.get_base_url().as_str(), + "--access-token", + token.as_str(), + ]) + .await + .success(); + + // 2.3. Push the dataset to the API server + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ + "push", + dataset_name.as_str(), + "--to", + kamu_api_server_dataset_endpoint.as_str(), + ], + "1 dataset(s) pushed", + ) + .await; + } + + // 3. Pulling the dataset from the API server + { + let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec!["pull", kamu_api_server_dataset_endpoint.as_str()], + "1 dataset(s) updated", + ) + .await; + } +} - let mut dataset_endpoint = Url::parse("odf+http://host").unwrap(); +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - dataset_endpoint.set_host(base_url.host_str()).unwrap(); - dataset_endpoint.set_port(base_url.port()).unwrap(); +pub async fn test_smart_force_push_pull(kamu_api_server_client: KamuApiServerClient) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + // 1. Grub a token + let token = kamu_api_server_client.login_as_e2e_user().await; - dataset_endpoint - .join("e2e-user/player-scores") - .unwrap() - .to_string() - }; + let kamu_api_server_dataset_endpoint = get_dataset_endpoint( + kamu_api_server_client.get_base_url(), + &dataset_name, + E2E_USER_ACCOUNT_NAME_STR, + ); // 2. Pushing the dataset to the API server { @@ -37,75 +118,166 @@ pub async fn test_smart_push_pull_sequence(kamu_api_server_client: KamuApiServer // 2.1. Add the dataset { - let dataset_path = kamu_in_push_workspace - .workspace_path() - .join("player-scores.yaml"); - - std::fs::write( - dataset_path.clone(), - indoc::indoc!( - r#" - kind: DatasetSnapshot - version: 1 - content: - name: player-scores - kind: Root - metadata: - - kind: AddPushSource - sourceName: default - read: - kind: NdJson - schema: - - "match_time TIMESTAMP" - - "match_id BIGINT" - - "player_id STRING" - - "score BIGINT" - merge: - kind: Ledger - primaryKey: - - match_id - - player_id - - kind: SetVocab - eventTimeColumn: match_time - "# - ), - ) - .unwrap(); - kamu_in_push_workspace - .execute(["add", dataset_path.to_str().unwrap()]) + .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) .await .success(); } // 2.1. Ingest data to the dataset { - let dataset_data_path = kamu_in_push_workspace - .workspace_path() - .join("player-scores.data.ndjson"); - - std::fs::write( - dataset_data_path.clone(), - indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Bob", "score": 80} - "#, - ), + kamu_in_push_workspace + .ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; + } + + // 2.2. Login to the API server + kamu_in_push_workspace + .execute([ + "login", + kamu_api_server_client.get_base_url().as_str(), + "--access-token", + token.as_str(), + ]) + .await + .success(); + + // Initial dataset push + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ + "push", + dataset_name.as_str(), + "--to", + kamu_api_server_dataset_endpoint.as_str(), + ], + "1 dataset(s) pushed", + ) + .await; + + // Hard compact dataset + kamu_in_push_workspace + .execute([ + "--yes", + "system", + "compact", + dataset_name.as_str(), + "--hard", + "--keep-metadata-only", + ]) + .await + .success(); + + // Should fail without force flag + run_and_assert_command_failure( + &kamu_in_push_workspace, + vec![ + "push", + dataset_name.as_str(), + "--to", + kamu_api_server_dataset_endpoint.as_str(), + ], + "Failed to push 1 dataset(s)", + ) + .await; + + // Should successfully push with force flag + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ + "push", + dataset_name.as_str(), + "--to", + kamu_api_server_dataset_endpoint.as_str(), + "--force", + ], + "1 dataset(s) pushed", + ) + .await; + } + + // 3. Pulling the dataset from the API server + { + let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + + // Call with no-alias flag to avoid remote ingest checking in next step + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec![ + "pull", + kamu_api_server_dataset_endpoint.as_str(), + "--no-alias", + ], + "1 dataset(s) updated", + ) + .await; + + // Ingest data in pulled dataset + + kamu_in_pull_workspace + .ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, ) - .unwrap(); + .await; + + // Should fail without force flag + run_and_assert_command_failure( + &kamu_in_pull_workspace, + vec!["pull", kamu_api_server_dataset_endpoint.as_str()], + "Failed to update 1 dataset(s)", + ) + .await; + + // Should successfully pull with force flag + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec!["pull", kamu_api_server_dataset_endpoint.as_str(), "--force"], + "1 dataset(s) updated", + ) + .await; + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +pub async fn test_smart_push_pull_add_alias(kamu_api_server_client: KamuApiServerClient) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + // 1. Grub a token + let token = kamu_api_server_client.login_as_e2e_user().await; + + let kamu_api_server_dataset_endpoint = get_dataset_endpoint( + kamu_api_server_client.get_base_url(), + &dataset_name, + E2E_USER_ACCOUNT_NAME_STR, + ); + + // 2. Push command + { + let kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; + + // Add the dataset + { kamu_in_push_workspace - .execute([ - "ingest", - "player-scores", - dataset_data_path.to_str().unwrap(), - ]) + .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) .await .success(); } - // 2.2. Login to the API server + // Ingest data to the dataset + { + kamu_in_push_workspace + .ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; + } + + // Login to the API server kamu_in_push_workspace .execute([ "login", @@ -116,27 +288,1025 @@ pub async fn test_smart_push_pull_sequence(kamu_api_server_client: KamuApiServer .await .success(); - // 2.3. Push the dataset to the API server - kamu_in_push_workspace - .execute([ + // Dataset push without storing alias + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ + "push", + dataset_name.as_str(), + "--to", + kamu_api_server_dataset_endpoint.as_str(), + "--no-alias", + ], + "1 dataset(s) pushed", + ) + .await; + + // Check alias should be empty + let aliases = kamu_in_push_workspace + .get_list_of_repo_aliases(&opendatafabric::DatasetRef::from(dataset_name.clone())) + .await; + assert!(aliases.is_empty()); + + // Dataset push with storing alias + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ "push", - "player-scores", + dataset_name.as_str(), "--to", kamu_api_server_dataset_endpoint.as_str(), + ], + "1 dataset(s) up-to-date", + ) + .await; + + let aliases = kamu_in_push_workspace + .get_list_of_repo_aliases(&opendatafabric::DatasetRef::from(dataset_name.clone())) + .await; + let expected_aliases = vec![RepoAlias { + dataset: dataset_name.clone(), + kind: "Push".to_string(), + alias: kamu_api_server_dataset_endpoint.clone(), + }]; + pretty_assertions::assert_eq!(aliases, expected_aliases); + } + + // 3. Pull command + { + let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + + // Dataset pull without storing alias + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec![ + "pull", + kamu_api_server_dataset_endpoint.as_str(), + "--no-alias", + ], + "1 dataset(s) updated", + ) + .await; + + // Check alias should be empty + let aliases = kamu_in_pull_workspace + .get_list_of_repo_aliases(&opendatafabric::DatasetRef::from(dataset_name.clone())) + .await; + assert!(aliases.is_empty()); + + // Delete local dataset + kamu_in_pull_workspace + .execute(["--yes", "delete", dataset_name.as_str()]) + .await + .success(); + + // Dataset pull with storing alias + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec!["pull", kamu_api_server_dataset_endpoint.as_str()], + "1 dataset(s) updated", + ) + .await; + + let aliases = kamu_in_pull_workspace + .get_list_of_repo_aliases(&opendatafabric::DatasetRef::from(dataset_name.clone())) + .await; + let expected_aliases = vec![RepoAlias { + dataset: dataset_name.clone(), + kind: "Pull".to_string(), + alias: kamu_api_server_dataset_endpoint.clone(), + }]; + pretty_assertions::assert_eq!(aliases, expected_aliases); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_smart_pull_as(kamu_api_server_client: KamuApiServerClient) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + // 1. Grub a token + let token = kamu_api_server_client.login_as_e2e_user().await; + + kamu_api_server_client + .create_player_scores_dataset_with_data( + &token, + Some(AccountName::new_unchecked(E2E_USER_ACCOUNT_NAME_STR)), + ) + .await; + + let kamu_api_server_dataset_endpoint = get_dataset_endpoint( + kamu_api_server_client.get_base_url(), + &dataset_name, + E2E_USER_ACCOUNT_NAME_STR, + ); + + { + let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + + let new_dataset_name = DatasetName::new_unchecked("foo"); + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec![ + "pull", + kamu_api_server_dataset_endpoint.as_str(), + "--as", + new_dataset_name.as_str(), + ], + "1 dataset(s) updated", + ) + .await; + + let expected_dataset_list = kamu_in_pull_workspace + .list_datasets() + .await + .into_iter() + .map(|dataset| dataset.name) + .collect::>(); + + pretty_assertions::assert_eq!(vec![new_dataset_name], expected_dataset_list); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClient) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + let dataset_derivative_name = DatasetName::new_unchecked("leaderboard"); + + // 1. Grub a token + let token = kamu_api_server_client.login_as_e2e_user().await; + + let kamu_api_server_root_dataset_endpoint = get_dataset_endpoint( + kamu_api_server_client.get_base_url(), + &dataset_name, + E2E_USER_ACCOUNT_NAME_STR, + ); + let kamu_api_server_derivative_dataset_endpoint = get_dataset_endpoint( + kamu_api_server_client.get_base_url(), + &dataset_derivative_name, + E2E_USER_ACCOUNT_NAME_STR, + ); + + let mut kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; + + // 2. Pushing datasets to the API server + { + kamu_in_push_workspace + .set_system_time(Some(DateTime::from_str("2050-01-02T03:04:05Z").unwrap())); + + // 2.1. Add datasets + { + kamu_in_push_workspace + .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + kamu_in_push_workspace + .execute_with_input( + ["add", "--stdin"], + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + ) + .await + .success(); + } + + // 2.1. Ingest data to the dataset + { + kamu_in_push_workspace + .ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; + } + + // 2.2. Login to the API server + kamu_in_push_workspace + .execute([ + "login", + kamu_api_server_client.get_base_url().as_str(), + "--access-token", + token.as_str(), ]) .await .success(); + + // Push all datasets should fail + run_and_assert_command_failure( + &kamu_in_push_workspace, + vec!["push", "--all"], + "Pushing all datasets is not yet supported", + ) + .await; + + // Push datasets one by one + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ + "push", + dataset_name.as_str(), + "--to", + kamu_api_server_root_dataset_endpoint.as_str(), + ], + "1 dataset(s) pushed", + ) + .await; + + kamu_in_push_workspace + .execute(["pull", dataset_derivative_name.as_str()]) + .await + .success(); + + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ + "push", + dataset_derivative_name.as_str(), + "--to", + kamu_api_server_derivative_dataset_endpoint.as_str(), + ], + "1 dataset(s) pushed", + ) + .await; } - // 3. Pulling the dataset from the API server + // 3. Pulling datasets from the API server { let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + // Pull datasets one by one and check data + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec!["pull", kamu_api_server_root_dataset_endpoint.as_str()], + "1 dataset(s) updated", + ) + .await; + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec!["pull", kamu_api_server_derivative_dataset_endpoint.as_str()], + "1 dataset(s) updated", + ) + .await; + + let expected_schema = indoc::indoc!( + r#" + message arrow_schema { + REQUIRED INT64 offset; + REQUIRED INT32 op; + REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_id; + OPTIONAL BYTE_ARRAY player_id (STRING); + OPTIONAL INT64 score; + } + "# + ); + let expected_data = indoc::indoc!( + r#" + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + | offset | op | system_time | match_time | match_id | player_id | score | + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + "# + ); + let expected_derivative_schema = indoc::indoc!( + r#" + message arrow_schema { + OPTIONAL INT64 offset; + REQUIRED INT32 op; + REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 place; + OPTIONAL INT64 match_id; + OPTIONAL BYTE_ARRAY player_id (STRING); + OPTIONAL INT64 score; + } + "# + ); + let expected_derivative_data = indoc::indoc!( + r#" + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + "# + ); + + kamu_in_pull_workspace + .assert_last_data_slice(&dataset_name, expected_schema, expected_data) + .await; + kamu_in_pull_workspace + .assert_last_data_slice( + &dataset_derivative_name, + expected_derivative_schema, + expected_derivative_data, + ) + .await; + + // Update remote datasets + + kamu_in_push_workspace + .ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + ) + .await; + run_and_assert_command_success( + &kamu_in_push_workspace, + vec!["pull", dataset_derivative_name.as_str()], + "1 dataset(s) updated", + ) + .await; + + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ + "push", + dataset_name.as_str(), + "--to", + kamu_api_server_root_dataset_endpoint.as_str(), + ], + "1 dataset(s) pushed", + ) + .await; + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ + "push", + dataset_derivative_name.as_str(), + "--to", + kamu_api_server_derivative_dataset_endpoint.as_str(), + ], + "1 dataset(s) pushed", + ) + .await; + + // Pull all datasets + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec!["pull", "--all"], + "2 dataset(s) updated", + ) + .await; + + // Perform dataslices checks + let expected_data = indoc::indoc!( + r#" + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + | offset | op | system_time | match_time | match_id | player_id | score | + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + | 2 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | Alice | 70 | + | 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 2 | Charlie | 90 | + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + "# + ); + let expected_derivative_data = indoc::indoc!( + r#" + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | 2 | 1 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + | 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 2 | 2 | Charlie | 90 | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + "# + ); + + kamu_in_pull_workspace + .assert_last_data_slice(&dataset_name, expected_schema, expected_data) + .await; + kamu_in_pull_workspace + .assert_last_data_slice( + &dataset_derivative_name, + expected_derivative_schema, + expected_derivative_data, + ) + .await; + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServerClient) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + let dataset_derivative_name = DatasetName::new_unchecked("leaderboard"); + + // 1. Grub a token + let token = kamu_api_server_client.login_as_e2e_user().await; + + let kamu_api_server_root_dataset_endpoint = get_dataset_endpoint( + kamu_api_server_client.get_base_url(), + &dataset_name, + E2E_USER_ACCOUNT_NAME_STR, + ); + let mut kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; + + // 2. Pushing datasets to the API server + { + kamu_in_push_workspace + .set_system_time(Some(DateTime::from_str("2050-01-02T03:04:05Z").unwrap())); + + // 2.1. Add datasets + { + kamu_in_push_workspace + .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + } + + // 2.1. Ingest data to the dataset + { + kamu_in_push_workspace + .ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; + } + + // 2.2. Login to the API server + kamu_in_push_workspace + .execute([ + "login", + kamu_api_server_client.get_base_url().as_str(), + "--access-token", + token.as_str(), + ]) + .await + .success(); + + // Push all datasets should fail + run_and_assert_command_failure( + &kamu_in_push_workspace, + vec!["push", dataset_name.as_str(), "--recursive"], + "Recursive push is not yet supported", + ) + .await; + + // Push dataset + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ + "push", + dataset_name.as_str(), + "--to", + kamu_api_server_root_dataset_endpoint.as_str(), + ], + "1 dataset(s) pushed", + ) + .await; + } + + // 3. Pulling datasets from the API server + { + let mut kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + kamu_in_pull_workspace + .set_system_time(Some(DateTime::from_str("2050-01-02T03:04:05Z").unwrap())); + + // Pull datasets one by one and check data + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec!["pull", kamu_api_server_root_dataset_endpoint.as_str()], + "1 dataset(s) updated", + ) + .await; + kamu_in_pull_workspace - .execute(["pull", kamu_api_server_dataset_endpoint.as_str()]) + .execute_with_input( + ["add", "--stdin"], + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + ) .await .success(); + + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec!["pull", dataset_derivative_name.as_str()], + "1 dataset(s) updated", + ) + .await; + + let expected_schema = indoc::indoc!( + r#" + message arrow_schema { + REQUIRED INT64 offset; + REQUIRED INT32 op; + REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_id; + OPTIONAL BYTE_ARRAY player_id (STRING); + OPTIONAL INT64 score; + } + "# + ); + let expected_data = indoc::indoc!( + r#" + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + | offset | op | system_time | match_time | match_id | player_id | score | + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + "# + ); + let expected_derivative_schema = indoc::indoc!( + r#" + message arrow_schema { + OPTIONAL INT64 offset; + REQUIRED INT32 op; + REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 place; + OPTIONAL INT64 match_id; + OPTIONAL BYTE_ARRAY player_id (STRING); + OPTIONAL INT64 score; + } + "# + ); + let expected_derivative_data = indoc::indoc!( + r#" + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + "# + ); + + kamu_in_pull_workspace + .assert_last_data_slice(&dataset_name, expected_schema, expected_data) + .await; + kamu_in_pull_workspace + .assert_last_data_slice( + &dataset_derivative_name, + expected_derivative_schema, + expected_derivative_data, + ) + .await; + + // Update remote datasets + + kamu_in_push_workspace + .ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + ) + .await; + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ + "push", + dataset_name.as_str(), + "--to", + kamu_api_server_root_dataset_endpoint.as_str(), + ], + "1 dataset(s) pushed", + ) + .await; + + // Pull all datasets + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec!["pull", dataset_derivative_name.as_str(), "--recursive"], + "2 dataset(s) updated", + ) + .await; + + // Perform dataslices checks + let expected_data = indoc::indoc!( + r#" + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + | offset | op | system_time | match_time | match_id | player_id | score | + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + | 2 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | Alice | 70 | + | 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 2 | Charlie | 90 | + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + "# + ); + let expected_derivative_data = indoc::indoc!( + r#" + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | 2 | 1 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + | 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 2 | 2 | Charlie | 90 | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + "# + ); + + kamu_in_pull_workspace + .assert_last_data_slice(&dataset_name, expected_schema, expected_data) + .await; + kamu_in_pull_workspace + .assert_last_data_slice( + &dataset_derivative_name, + expected_derivative_schema, + expected_derivative_data, + ) + .await; } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_smart_pull_set_watermark(kamu: KamuCliPuppet) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + let assert = kamu + .execute([ + "pull", + dataset_name.as_str(), + "--set-watermark", + "2051-01-02T03:04:05Z", + ]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!(r#"Committed new block"#).trim()), + "Unexpected output:\n{stderr}", + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_smart_pull_reset_derivative(kamu: KamuCliPuppet) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + let dataset_derivative_name = DatasetName::new_unchecked("leaderboard"); + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + kamu.execute_with_input( + ["add", "--stdin"], + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + ) + .await + .success(); + + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; + + run_and_assert_command_success( + &kamu, + vec!["pull", dataset_derivative_name.as_str()], + "1 dataset(s) updated", + ) + .await; + + let expected_derivative_schema = indoc::indoc!( + r#" + message arrow_schema { + OPTIONAL INT64 offset; + REQUIRED INT32 op; + REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 place; + OPTIONAL INT64 match_id; + OPTIONAL BYTE_ARRAY player_id (STRING); + OPTIONAL INT64 score; + } + "# + ); + let expected_derivative_data = indoc::indoc!( + r#" + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + "# + ); + kamu.assert_last_data_slice( + &dataset_derivative_name, + expected_derivative_schema, + expected_derivative_data, + ) + .await; + + // Compact root dataset + kamu.execute([ + "--yes", + "system", + "compact", + dataset_name.as_str(), + "--hard", + "--keep-metadata-only", + ]) + .await + .success(); + + // Pull derivative should fail + run_and_assert_command_failure( + &kamu, + vec!["pull", dataset_derivative_name.as_str()], + "Failed to update 1 dataset(s)", + ) + .await; + + // Add new data to root dataset + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + ) + .await; + + run_and_assert_command_success( + &kamu, + vec![ + "pull", + dataset_derivative_name.as_str(), + "--reset-derivatives-on-diverged-input", + ], + "1 dataset(s) updated", + ) + .await; + + let expected_derivative_data = indoc::indoc!( + r#" + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | 2 | Alice | 70 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 1 | 2 | Charlie | 90 | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + "# + ); + kamu.assert_last_data_slice( + &dataset_derivative_name, + expected_derivative_schema, + expected_derivative_data, + ) + .await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_smart_push_visibility(kamu_api_server_client: KamuApiServerClient) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + + // 1. Grub a token + let token = kamu_api_server_client.login_as_e2e_user().await; + + let kamu_api_server_dataset_endpoint = get_dataset_endpoint( + kamu_api_server_client.get_base_url(), + &dataset_name, + E2E_USER_ACCOUNT_NAME_STR, + ); + + // 2. Pushing the dataset to the API server + { + let kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; + + // 2.1. Add the dataset + { + kamu_in_push_workspace + .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + } + + // 2.1. Ingest data to the dataset + { + kamu_in_push_workspace + .ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; + } + + // 2.2. Login to the API server + kamu_in_push_workspace + .execute([ + "login", + kamu_api_server_client.get_base_url().as_str(), + "--access-token", + token.as_str(), + ]) + .await + .success(); + + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ + "push", + dataset_name.as_str(), + "--to", + kamu_api_server_dataset_endpoint.as_str(), + "--visibility", + "private", + ], + "1 dataset(s) pushed", + ) + .await; + + // ToDo add visibility check + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_smart_push_pull_s3(kamu: KamuCliPuppet) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; + + let s3_server = LocalS3Server::new().await; + + let dataset_url = format!("{}/e2e-user/{dataset_name}", s3_server.url); + // Push dataset + run_and_assert_command_success( + &kamu, + vec!["push", dataset_name.as_str(), "--to", dataset_url.as_str()], + "1 dataset(s) pushed", + ) + .await; + + { + let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec!["pull", dataset_url.as_str()], + "1 dataset(s) updated", + ) + .await; + + let expected_schema = indoc::indoc!( + r#" + message arrow_schema { + REQUIRED INT64 offset; + REQUIRED INT32 op; + REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_id; + OPTIONAL BYTE_ARRAY player_id (STRING); + OPTIONAL INT64 score; + } + "# + ); + let expected_data = indoc::indoc!( + r#" + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + | offset | op | system_time | match_time | match_id | player_id | score | + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + "# + ); + kamu.assert_last_data_slice(&dataset_name, expected_schema, expected_data) + .await; + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_smart_pull_derivative(kamu: KamuCliPuppet) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + let dataset_derivative_name = DatasetName::new_unchecked("leaderboard"); + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + kamu.execute_with_input( + ["add", "--stdin"], + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + ) + .await + .success(); + + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; + + run_and_assert_command_failure( + &kamu, + vec![ + "tail", + dataset_derivative_name.as_str(), + "--output-format", + "table", + ], + "Error: Dataset schema is not yet available: leaderboard", + ) + .await; + + run_and_assert_command_success( + &kamu, + vec!["pull", dataset_derivative_name.as_str()], + "1 dataset(s) updated", + ) + .await; + + let expected_derivative_schema = indoc::indoc!( + r#" + message arrow_schema { + OPTIONAL INT64 offset; + REQUIRED INT32 op; + REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 place; + OPTIONAL INT64 match_id; + OPTIONAL BYTE_ARRAY player_id (STRING); + OPTIONAL INT64 score; + } + "# + ); + + let expected_derivative_data = indoc::indoc!( + r#" + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + "# + ); + kamu.assert_last_data_slice( + &dataset_derivative_name, + expected_derivative_schema, + expected_derivative_data, + ) + .await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Helpers +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +fn get_dataset_endpoint( + base_url: &Url, + dataset_name: &DatasetName, + account_name_str: &str, +) -> String { + let mut dataset_endpoint = Url::parse("odf+http://host").unwrap(); + + dataset_endpoint.set_host(base_url.host_str()).unwrap(); + dataset_endpoint.set_port(base_url.port()).unwrap(); + + dataset_endpoint + .join(format!("{account_name_str}/{dataset_name}").as_str()) + .unwrap() + .to_string() +} + +async fn run_and_assert_command_success( + kamu: &KamuCliPuppet, + args: Vec<&str>, + expected_message: &str, +) { + let assert = kamu.execute(args).await.success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(expected_message), + "Unexpected output:\n{stderr}", + ); +} + +async fn run_and_assert_command_failure( + kamu: &KamuCliPuppet, + args: Vec<&str>, + expected_message: &str, +) { + let assert = kamu.execute(args).await.failure(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(expected_message), + "Unexpected output:\n{stderr}", + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs index 8c9292bf7..bebf25b1e 100644 --- a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs +++ b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs @@ -14,15 +14,10 @@ use std::path::PathBuf; use async_trait::async_trait; use chrono::{DateTime, Utc}; use datafusion::prelude::{ParquetReadOptions, SessionContext}; -use opendatafabric::serde::yaml::{ - DatasetKindDef, - YamlDatasetSnapshotSerializer, - YamlMetadataBlockDeserializer, -}; +use opendatafabric::serde::yaml::{YamlDatasetSnapshotSerializer, YamlMetadataBlockDeserializer}; use opendatafabric::serde::{DatasetSnapshotSerializer, MetadataBlockDeserializer}; use opendatafabric::{ DatasetID, - DatasetKind, DatasetName, DatasetRef, DatasetSnapshot, @@ -41,14 +36,16 @@ pub trait KamuCliPuppetExt { async fn add_dataset(&self, dataset_snapshot: DatasetSnapshot); + async fn list_blocks(&self, dataset_name: &DatasetName) -> Vec; + + async fn ingest_data(&self, dataset_name: &DatasetName, data: &str); + async fn get_list_of_repo_aliases(&self, dataset_ref: &DatasetRef) -> Vec; async fn complete(&self, input: T, current: usize) -> Vec where T: Into + Send; - async fn list_blocks(&self, dataset_name: &DatasetName) -> Vec; - async fn start_api_server(self, e2e_data_file_path: PathBuf) -> ServerOutput; async fn assert_last_data_slice( @@ -227,6 +224,18 @@ impl KamuCliPuppetExt for KamuCliPuppet { kamu_data_utils::testing::assert_data_eq(df.clone(), expected_data).await; kamu_data_utils::testing::assert_schema_eq(df.schema(), expected_schema); } + + async fn ingest_data(&self, dataset_name: &DatasetName, data: &str) { + let dataset_data_path = self + .workspace_path() + .join(format!("{dataset_name}.data.ndjson")); + + std::fs::write(dataset_data_path.clone(), data).unwrap(); + + self.execute(["ingest", dataset_name, dataset_data_path.to_str().unwrap()]) + .await + .success(); + } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -244,8 +253,9 @@ pub struct DatasetRecord { #[serde(rename = "ID")] pub id: DatasetID, pub name: DatasetName, - #[serde(with = "DatasetKindDef")] - pub kind: DatasetKind, + // CLI returns regular ENUM DatasetKind(Root/Derivative) for local datasets + // but for remote it is Remote(DatasetKind) type + pub kind: String, pub head: Multihash, pub pulled: Option>, pub records: usize, From 1ad0fe093670449e2ed9b1f1eebb41d53cdcf6f4 Mon Sep 17 00:00:00 2001 From: Dima Pristupa Date: Mon, 14 Oct 2024 17:49:16 +0300 Subject: [PATCH 04/13] E2E: add Outbox processing middleware (#901) * E2E, e2e_middleware_fn(): implement * kamu-adapter-http: hide E2E things behind the "e2e" feature gate * query_handler_post_v2(): fix a typo * kamu-cli: integrate the E2E middleware * CHANGELOG.md: update --- CHANGELOG.md | 1 + Cargo.lock | 1 + src/adapter/http/Cargo.toml | 11 +++ src/adapter/http/src/e2e/e2e_middleware.rs | 105 +++++++++++++++++++++ src/adapter/http/src/e2e/mod.rs | 3 + src/adapter/http/src/lib.rs | 1 + src/app/cli/Cargo.toml | 2 +- src/app/cli/src/explore/api_server.rs | 17 +++- 8 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 src/adapter/http/src/e2e/e2e_middleware.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index dc906eb53..849eef4d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ Recommendation: for ease of reading, use the following order: - `kamu tail` command - `kamu login` command - `kamu logout` command +- E2E: HTTP middleware is implemented, which improves stability of E2E tests ### Changed - `kamu repo alias list`: added JSON output alongside with other formats mentioned in the command's help - Private Datasets, `DatasetEntry` integration that will allow us to build dataset indexing diff --git a/Cargo.lock b/Cargo.lock index 793fa976f..a3870a159 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5372,6 +5372,7 @@ dependencies = [ "futures", "headers", "http 1.1.0", + "http-body-util", "http-common", "hyper 1.4.1", "indoc 2.0.5", diff --git a/src/adapter/http/Cargo.toml b/src/adapter/http/Cargo.toml index ffa7d83b8..1bff300df 100644 --- a/src/adapter/http/Cargo.toml +++ b/src/adapter/http/Cargo.toml @@ -21,6 +21,12 @@ workspace = true doctest = false +[features] +default = [] + +e2e = ["dep:messaging-outbox", "dep:http-body-util"] + + [dependencies] database-common = { workspace = true } database-common-macros = { workspace = true } @@ -86,6 +92,11 @@ tracing = "0.1" url = { version = "2", features = ["serde"] } uuid = { version = "1", default-features = false, features = ["v4"] } +# Optional +messaging-outbox = { optional = true, workspace = true } + +http-body-util = { optional = true, version = "0.1" } + [dev-dependencies] container-runtime = { workspace = true } diff --git a/src/adapter/http/src/e2e/e2e_middleware.rs b/src/adapter/http/src/e2e/e2e_middleware.rs new file mode 100644 index 000000000..22ab277d2 --- /dev/null +++ b/src/adapter/http/src/e2e/e2e_middleware.rs @@ -0,0 +1,105 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use axum::body::{Body, Bytes}; +use axum::extract::Request; +use axum::middleware::Next; +use axum::response::Response; +use http::Method; +use http_common::ApiError; +use serde::Deserialize; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Middleware that invokes Outbox messages processing for mutable requests +pub async fn e2e_middleware_fn(request: Request, next: Next) -> Result { + let base_catalog = request + .extensions() + .get::() + .cloned() + .expect("Catalog not found in http server extensions"); + + let (is_mutable_request, request) = analyze_request_for_mutability(request).await?; + let response = next.run(request).await; + + if is_mutable_request && response.status().is_success() { + let outbox_executor = base_catalog + .get_one::() + .unwrap(); + + outbox_executor.run_while_has_tasks().await?; + } + + Ok(response) +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +async fn analyze_request_for_mutability(request: Request) -> Result<(bool, Request), ApiError> { + { + let is_not_modifying_requests = request.method() != Method::POST; + + if is_not_modifying_requests { + return Ok((false, request)); + } + } + { + let is_rest_api_post_request = request.uri().path() != "/graphql"; + + if is_rest_api_post_request { + return Ok((true, request)); + } + } + { + // In the case of GQL, we check whether the query is mutable or not + let (request_parts, request_body) = request.into_parts(); + let buffered_request_body = buffer_request_body(request_body).await?; + + let is_mutating_gql = if let Ok(body) = std::str::from_utf8(&buffered_request_body) { + let gql_request = serde_json::from_str::(body) + .map_err(ApiError::bad_request)?; + + gql_request.query.starts_with("mutation") + } else { + false + }; + + let request = Request::from_parts(request_parts, Body::from(buffered_request_body)); + + Ok((is_mutating_gql, request)) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +async fn buffer_request_body(request_body: B) -> Result +where + B: axum::body::HttpBody, + B::Error: std::error::Error + Send + Sync + 'static, +{ + use http_body_util::BodyExt; + + let body_bytes = match request_body.collect().await { + Ok(collected) => collected.to_bytes(), + Err(e) => { + return Err(ApiError::bad_request(e)); + } + }; + + Ok(body_bytes) +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, Deserialize)] +struct SimplifiedGqlRequest { + query: String, +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/adapter/http/src/e2e/mod.rs b/src/adapter/http/src/e2e/mod.rs index c18d4f36a..1aab87b15 100644 --- a/src/adapter/http/src/e2e/mod.rs +++ b/src/adapter/http/src/e2e/mod.rs @@ -7,5 +7,8 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +mod e2e_middleware; mod e2e_router; + +pub use e2e_middleware::*; pub use e2e_router::*; diff --git a/src/adapter/http/src/lib.rs b/src/adapter/http/src/lib.rs index 0a138e055..7ab28f685 100644 --- a/src/adapter/http/src/lib.rs +++ b/src/adapter/http/src/lib.rs @@ -18,6 +18,7 @@ mod access_token; pub use access_token::*; mod axum_utils; pub mod data; +#[cfg(feature = "e2e")] pub mod e2e; mod simple_protocol; pub mod smart_protocol; diff --git a/src/app/cli/Cargo.toml b/src/app/cli/Cargo.toml index a3d507bde..17d87241a 100644 --- a/src/app/cli/Cargo.toml +++ b/src/app/cli/Cargo.toml @@ -57,7 +57,7 @@ kamu-data-utils = { workspace = true } kamu-adapter-auth-oso = { workspace = true } kamu-adapter-flight-sql = { optional = true, workspace = true } kamu-adapter-graphql = { workspace = true } -kamu-adapter-http = { workspace = true } +kamu-adapter-http = { workspace = true, features = ["e2e"], default-features = false } kamu-adapter-oauth = { workspace = true } kamu-adapter-odata = { workspace = true } kamu-datafusion-cli = { workspace = true } diff --git a/src/app/cli/src/explore/api_server.rs b/src/app/cli/src/explore/api_server.rs index 3d238c5fc..510d81298 100644 --- a/src/app/cli/src/explore/api_server.rs +++ b/src/app/cli/src/explore/api_server.rs @@ -14,7 +14,7 @@ use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; -use axum::Extension; +use axum::{middleware, Extension}; use database_common_macros::transactional_handler; use dill::{Catalog, CatalogBuilder}; use http_common::ApiError; @@ -94,6 +94,8 @@ impl APIServer { let mut app = axum::Router::new() .route("/", axum::routing::get(root)) .route( + // IMPORTANT: The same name is used inside e2e_middleware_fn(). + // If there is a need to change, please update there too. "/graphql", axum::routing::get(graphql_playground_handler).post(graphql_handler), ) @@ -135,7 +137,17 @@ impl APIServer { .nest("/", kamu_adapter_http::data::dataset_router()), multi_tenant_workspace, ), - ) + ); + + let is_e2e_testing = e2e_output_data_path.is_some(); + + if is_e2e_testing { + app = app.layer(middleware::from_fn( + kamu_adapter_http::e2e::e2e_middleware_fn, + )); + } + + app = app .layer(kamu_adapter_http::AuthenticationLayer::new()) .layer( tower_http::cors::CorsLayer::new() @@ -157,7 +169,6 @@ impl APIServer { .layer(Extension(gql_schema)) .layer(Extension(api_server_catalog)); - let is_e2e_testing = e2e_output_data_path.is_some(); let maybe_shutdown_notify = if is_e2e_testing { let shutdown_notify = Arc::new(Notify::new()); From f6055b0584815fef42ae061d176d44512d3d78fb Mon Sep 17 00:00:00 2001 From: Dima Pristupa Date: Mon, 14 Oct 2024 17:52:27 +0300 Subject: [PATCH 05/13] Updates after rebasing --- CHANGELOG.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 849eef4d7..8d92afbbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,14 +11,8 @@ Recommendation: for ease of reading, use the following order: - Fixed --> -## [0.205.0] - 2024-10-15 -### Changed -- `kamu push ` command now can be called without `--to` reference and Alias or Remote dataset repository will be used as destination -- `kamu login` command now will store repository to Repository registry. Name can be provided with `--repo-name` flag and to skip creating repo can be used `--skip-add-repo` flag - -## [0.204.5] - 2024-10-08 +## [Unreleased] ### Added -- Postgres implementation for dataset entry and account Re-BAC repositories - Added (or expanded) E2E tests for: - `kamu config` command - `kamu init` command @@ -38,6 +32,15 @@ Recommendation: for ease of reading, use the following order: - `kamu login` command - `kamu logout` command - E2E: HTTP middleware is implemented, which improves stability of E2E tests + +## [0.205.0] - 2024-10-15 +### Changed +- `kamu push ` command now can be called without `--to` reference and Alias or Remote dataset repository will be used as destination +- `kamu login` command now will store repository to Repository registry. Name can be provided with `--repo-name` flag and to skip creating repo can be used `--skip-add-repo` flag + +## [0.204.5] - 2024-10-08 +### Added +- Postgres implementation for dataset entry and account Re-BAC repositories ### Changed - `kamu repo alias list`: added JSON output alongside with other formats mentioned in the command's help - Private Datasets, `DatasetEntry` integration that will allow us to build dataset indexing From b32149f453e3c9c3c3985c56972a2315ae034ead Mon Sep 17 00:00:00 2001 From: Dima Pristupa Date: Mon, 14 Oct 2024 17:52:58 +0300 Subject: [PATCH 06/13] CHANGELOG.md: add missed commands --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d92afbbe..48ba6ad90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,8 @@ Recommendation: for ease of reading, use the following order: - `kamu tail` command - `kamu login` command - `kamu logout` command + - `kamu push` command + - `kamu pull` command - E2E: HTTP middleware is implemented, which improves stability of E2E tests ## [0.205.0] - 2024-10-15 From 7ab99be12e2e7f2730c2b973f769942b7fc07b6d Mon Sep 17 00:00:00 2001 From: Dima Pristupa Date: Mon, 14 Oct 2024 17:56:04 +0300 Subject: [PATCH 07/13] CHANGELOG.md: add a note about a "kamu add" fix --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48ba6ad90..e88588776 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,9 @@ Recommendation: for ease of reading, use the following order: - `kamu logout` command - `kamu push` command - `kamu pull` command -- E2E: HTTP middleware is implemented, which improves stability of E2E tests +- E2E: HTTP middleware is implemented, which improves stability of E2E tests +### Fixed +- `kamu add`: fixed behavior when using `--stdin` and `--name` arguments ## [0.205.0] - 2024-10-15 ### Changed From cd021570f0735a29c74d624d23faaac842d4cd42 Mon Sep 17 00:00:00 2001 From: Dima Pristupa Date: Mon, 14 Oct 2024 17:58:59 +0300 Subject: [PATCH 08/13] KamuCliPuppetExt::ingest_data(): simplify --- src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs index bebf25b1e..1047e8df8 100644 --- a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs +++ b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs @@ -226,13 +226,7 @@ impl KamuCliPuppetExt for KamuCliPuppet { } async fn ingest_data(&self, dataset_name: &DatasetName, data: &str) { - let dataset_data_path = self - .workspace_path() - .join(format!("{dataset_name}.data.ndjson")); - - std::fs::write(dataset_data_path.clone(), data).unwrap(); - - self.execute(["ingest", dataset_name, dataset_data_path.to_str().unwrap()]) + self.execute_with_input(["ingest", dataset_name, "--stdin"], data) .await .success(); } From 02729541f3490763498eaf7530168be477c73da0 Mon Sep 17 00:00:00 2001 From: Dima Pristupa Date: Mon, 14 Oct 2024 18:07:16 +0300 Subject: [PATCH 09/13] E2E: activate new tests for SQLite --- .../cli/sqlite/tests/tests/commands/mod.rs | 20 ++++ .../tests/tests/commands/test_add_command.rs | 40 +++++++ .../tests/commands/test_compact_command.rs | 36 +++++++ .../tests/commands/test_config_command.rs | 33 ++++++ .../tests/commands/test_delete_command.rs | 33 ++++++ .../tests/commands/test_ingest_command.rs | 57 ++++++++++ .../tests/tests/commands/test_init_command.rs | 53 +++++++++ .../tests/commands/test_inspect_command.rs | 35 ++++++ .../tests/tests/commands/test_log_command.rs | 21 ++++ .../tests/commands/test_login_command.rs | 27 +++++ .../tests/tests/commands/test_new_command.rs | 26 +++++ .../tests/commands/test_rename_command.rs | 19 ++++ .../tests/tests/commands/test_repo_command.rs | 26 +++++ .../tests/commands/test_reset_command.rs | 20 ++++ .../tests/commands/test_search_command.rs | 40 +++++++ .../tests/tests/commands/test_sql_command.rs | 38 +++++++ .../commands/test_system_diagnose_command.rs | 19 ++++ .../tests/commands/test_system_gc_command.rs | 16 +++ .../commands/test_system_info_command.rs | 19 ++++ .../tests/tests/commands/test_tail_command.rs | 21 ++++ .../tests/commands/test_verify_command.rs | 36 +++++++ .../tests/test_smart_transfer_protocol.rs | 101 ++++++++++++++++++ 22 files changed, 736 insertions(+) create mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_add_command.rs create mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_compact_command.rs create mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_config_command.rs create mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_delete_command.rs create mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_ingest_command.rs create mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_init_command.rs create mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_inspect_command.rs create mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_log_command.rs create mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_login_command.rs create mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_new_command.rs create mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_rename_command.rs create mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_repo_command.rs create mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_reset_command.rs create mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_search_command.rs create mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_sql_command.rs create mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_system_diagnose_command.rs create mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_system_gc_command.rs create mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_system_info_command.rs create mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_tail_command.rs create mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_verify_command.rs diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/mod.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/mod.rs index b58f493b3..c3408f409 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/mod.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/mod.rs @@ -7,5 +7,25 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +mod test_add_command; +mod test_compact_command; +mod test_config_command; +mod test_delete_command; +mod test_ingest_command; +mod test_init_command; +mod test_inspect_command; +mod test_log_command; +mod test_login_command; +mod test_new_command; +mod test_rename_command; +mod test_repo_command; +mod test_reset_command; +mod test_search_command; +mod test_sql_command; mod test_system_api_server_gql_query; +mod test_system_diagnose_command; +mod test_system_gc_command; mod test_system_generate_token_command; +mod test_system_info_command; +mod test_tail_command; +mod test_verify_command; diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_add_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_add_command.rs new file mode 100644 index 000000000..6ba93dace --- /dev/null +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_add_command.rs @@ -0,0 +1,40 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_add_dataset_from_stdin +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_add_dataset_with_name +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_add_dataset_with_replace +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_add_recursive +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_compact_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_compact_command.rs new file mode 100644 index 000000000..fb2c2851f --- /dev/null +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_compact_command.rs @@ -0,0 +1,36 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_compact_hard + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_compact_keep_metadata_only + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_compact_verify + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_config_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_config_command.rs new file mode 100644 index 000000000..111611ad7 --- /dev/null +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_config_command.rs @@ -0,0 +1,33 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_config_set_value +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_config_reset_key +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_config_get_with_default +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_delete_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_delete_command.rs new file mode 100644 index 000000000..4166202a2 --- /dev/null +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_delete_command.rs @@ -0,0 +1,33 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_delete_dataset +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_delete_dataset_recursive +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_delete_dataset_all +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_ingest_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_ingest_command.rs new file mode 100644 index 000000000..2c74c7d44 --- /dev/null +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_ingest_command.rs @@ -0,0 +1,57 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_push_ingest_from_file_ledger, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_push_ingest_from_file_snapshot_with_event_time, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_ingest_from_stdin, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_ingest_recursive, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_ingest_with_source_name, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_init_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_init_command.rs new file mode 100644 index 000000000..a07699b0b --- /dev/null +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_init_command.rs @@ -0,0 +1,53 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_init_multi_tenant_creates_sqlite_database, + options = Options::default().with_no_workspace() +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = + kamu_cli_e2e_repo_tests::test_init_multi_tenant_with_exists_ok_flag_creates_sqlite_database, + options = Options::default().with_no_workspace() +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_init_exist_ok_st, + options = Options::default().with_no_workspace() +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_init_exist_ok_mt, + options = Options::default().with_no_workspace() +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_init_in_an_existing_workspace, + options = Options::default().with_no_workspace() +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_inspect_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_inspect_command.rs new file mode 100644 index 000000000..9ddfcd3d5 --- /dev/null +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_inspect_command.rs @@ -0,0 +1,35 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_inspect_lineage, +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_inspect_query, + options = Options::default().with_frozen_system_time() +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_inspect_schema, + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_log_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_log_command.rs new file mode 100644 index 000000000..54bc30e5a --- /dev/null +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_log_command.rs @@ -0,0 +1,21 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_log, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_login_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_login_command.rs new file mode 100644 index 000000000..eeeb6e2f0 --- /dev/null +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_login_command.rs @@ -0,0 +1,27 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_login_logout_password, +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_login_logout_oauth, + options = Options::default().with_multi_tenant() +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_new_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_new_command.rs new file mode 100644 index 000000000..d299df3d9 --- /dev/null +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_new_command.rs @@ -0,0 +1,26 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_new_root, +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_new_derivative, +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_rename_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_rename_command.rs new file mode 100644 index 000000000..889c54c2a --- /dev/null +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_rename_command.rs @@ -0,0 +1,19 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_rename_dataset +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_repo_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_repo_command.rs new file mode 100644 index 000000000..29b16d730 --- /dev/null +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_repo_command.rs @@ -0,0 +1,26 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_repository_pull_aliases_commands +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_repository_push_aliases_commands +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_reset_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_reset_command.rs new file mode 100644 index 000000000..08fcb2bc3 --- /dev/null +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_reset_command.rs @@ -0,0 +1,20 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_reset, + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_search_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_search_command.rs new file mode 100644 index 000000000..dbdf1ec9a --- /dev/null +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_search_command.rs @@ -0,0 +1,40 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_search_by_name + // We need synthetic time for the tests, but the third-party JWT code + // uses the current time. Assuming that the token lifetime is 24 hours, we will + // use the projected date (the current day) as a workaround. + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_search_by_repo + // We need synthetic time for the tests, but the third-party JWT code + // uses the current time. Assuming that the token lifetime is 24 hours, we will + // use the projected date (the current day) as a workaround. + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_sql_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_sql_command.rs new file mode 100644 index 000000000..8d21f99b9 --- /dev/null +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_sql_command.rs @@ -0,0 +1,38 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_datafusion_cli, + extra_test_groups = "engine, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_datafusion_cli_not_launched_in_root_ws, + options = Options::default().with_no_workspace(), + extra_test_groups = "engine, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_sql_command, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_diagnose_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_diagnose_command.rs new file mode 100644 index 000000000..df35fd6e5 --- /dev/null +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_diagnose_command.rs @@ -0,0 +1,19 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_system_diagnose +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_gc_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_gc_command.rs new file mode 100644 index 000000000..7a3c36657 --- /dev/null +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_gc_command.rs @@ -0,0 +1,16 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!(storage = sqlite, fixture = kamu_cli_e2e_repo_tests::test_gc); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_info_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_info_command.rs new file mode 100644 index 000000000..8f5781110 --- /dev/null +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_info_command.rs @@ -0,0 +1,19 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_system_info +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_tail_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_tail_command.rs new file mode 100644 index 000000000..ac92be7d6 --- /dev/null +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_tail_command.rs @@ -0,0 +1,21 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_tail, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_verify_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_verify_command.rs new file mode 100644 index 000000000..fde8f8389 --- /dev/null +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_verify_command.rs @@ -0,0 +1,36 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_verify_regular_dataset + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_verify_recursive + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_verify_integrity + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/test_smart_transfer_protocol.rs b/src/e2e/app/cli/sqlite/tests/tests/test_smart_transfer_protocol.rs index 326e42818..e7655e724 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/test_smart_transfer_protocol.rs @@ -24,3 +24,104 @@ kamu_cli_run_api_server_e2e_test!( ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_force_push_pull, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_add_alias, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_as, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_all, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_recursive, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_set_watermark, + options = Options::default().with_frozen_system_time(), +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_reset_derivative, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_visibility, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_s3, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_derivative, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// From a869e31aff5eb069c638239e890d60543f1d86f4 Mon Sep 17 00:00:00 2001 From: Dima Pristupa Date: Mon, 14 Oct 2024 19:01:35 +0300 Subject: [PATCH 10/13] E2E: activate new tests for PostgreSQL --- .../cli/postgres/tests/tests/commands/mod.rs | 20 ++++ .../tests/tests/commands/test_add_command.rs | 40 +++++++ .../tests/commands/test_compact_command.rs | 36 +++++++ .../tests/commands/test_config_command.rs | 33 ++++++ .../tests/commands/test_delete_command.rs | 33 ++++++ .../tests/commands/test_ingest_command.rs | 57 ++++++++++ .../tests/tests/commands/test_init_command.rs | 36 +++++++ .../tests/commands/test_inspect_command.rs | 35 ++++++ .../tests/tests/commands/test_log_command.rs | 21 ++++ .../tests/commands/test_login_command.rs | 27 +++++ .../tests/tests/commands/test_new_command.rs | 26 +++++ .../tests/commands/test_rename_command.rs | 19 ++++ .../tests/tests/commands/test_repo_command.rs | 26 +++++ .../tests/commands/test_reset_command.rs | 20 ++++ .../tests/commands/test_search_command.rs | 40 +++++++ .../tests/tests/commands/test_sql_command.rs | 38 +++++++ .../commands/test_system_diagnose_command.rs | 19 ++++ .../tests/commands/test_system_gc_command.rs | 19 ++++ .../commands/test_system_info_command.rs | 19 ++++ .../tests/tests/commands/test_tail_command.rs | 21 ++++ .../tests/commands/test_verify_command.rs | 36 +++++++ .../tests/test_smart_transfer_protocol.rs | 101 ++++++++++++++++++ 22 files changed, 722 insertions(+) create mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_add_command.rs create mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_compact_command.rs create mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_config_command.rs create mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_delete_command.rs create mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_ingest_command.rs create mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_init_command.rs create mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_inspect_command.rs create mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_log_command.rs create mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_login_command.rs create mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_new_command.rs create mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_rename_command.rs create mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_repo_command.rs create mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_reset_command.rs create mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_search_command.rs create mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_sql_command.rs create mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_system_diagnose_command.rs create mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_system_gc_command.rs create mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_system_info_command.rs create mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_tail_command.rs create mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_verify_command.rs diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/mod.rs b/src/e2e/app/cli/postgres/tests/tests/commands/mod.rs index b58f493b3..c3408f409 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/mod.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/mod.rs @@ -7,5 +7,25 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +mod test_add_command; +mod test_compact_command; +mod test_config_command; +mod test_delete_command; +mod test_ingest_command; +mod test_init_command; +mod test_inspect_command; +mod test_log_command; +mod test_login_command; +mod test_new_command; +mod test_rename_command; +mod test_repo_command; +mod test_reset_command; +mod test_search_command; +mod test_sql_command; mod test_system_api_server_gql_query; +mod test_system_diagnose_command; +mod test_system_gc_command; mod test_system_generate_token_command; +mod test_system_info_command; +mod test_tail_command; +mod test_verify_command; diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_add_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_add_command.rs new file mode 100644 index 000000000..407eb9cbc --- /dev/null +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_add_command.rs @@ -0,0 +1,40 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_add_dataset_from_stdin +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_add_dataset_with_name +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_add_dataset_with_replace +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_add_recursive +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_compact_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_compact_command.rs new file mode 100644 index 000000000..90d11dab3 --- /dev/null +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_compact_command.rs @@ -0,0 +1,36 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_compact_hard + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_compact_keep_metadata_only + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_compact_verify + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_config_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_config_command.rs new file mode 100644 index 000000000..0547a3e80 --- /dev/null +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_config_command.rs @@ -0,0 +1,33 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_config_set_value +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_config_reset_key +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_config_get_with_default +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_delete_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_delete_command.rs new file mode 100644 index 000000000..faa7a6a99 --- /dev/null +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_delete_command.rs @@ -0,0 +1,33 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_delete_dataset +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_delete_dataset_recursive +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_delete_dataset_all +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_ingest_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_ingest_command.rs new file mode 100644 index 000000000..a9fd603d7 --- /dev/null +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_ingest_command.rs @@ -0,0 +1,57 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_push_ingest_from_file_ledger, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_push_ingest_from_file_snapshot_with_event_time, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_ingest_from_stdin, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_ingest_recursive, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_ingest_with_source_name, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_init_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_init_command.rs new file mode 100644 index 000000000..0617e08c6 --- /dev/null +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_init_command.rs @@ -0,0 +1,36 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_init_exist_ok_st, + options = Options::default().with_no_workspace() +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_init_exist_ok_mt, + options = Options::default().with_no_workspace() +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_init_in_an_existing_workspace, + options = Options::default().with_no_workspace() +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_inspect_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_inspect_command.rs new file mode 100644 index 000000000..96239fd6c --- /dev/null +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_inspect_command.rs @@ -0,0 +1,35 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_inspect_lineage, +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_inspect_query, + options = Options::default().with_frozen_system_time() +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_inspect_schema, + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_log_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_log_command.rs new file mode 100644 index 000000000..c2449941d --- /dev/null +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_log_command.rs @@ -0,0 +1,21 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_log, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_login_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_login_command.rs new file mode 100644 index 000000000..98b7aecd3 --- /dev/null +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_login_command.rs @@ -0,0 +1,27 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_login_logout_password, +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_login_logout_oauth, + options = Options::default().with_multi_tenant() +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_new_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_new_command.rs new file mode 100644 index 000000000..3c65ebef4 --- /dev/null +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_new_command.rs @@ -0,0 +1,26 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_new_root, +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_new_derivative, +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_rename_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_rename_command.rs new file mode 100644 index 000000000..64a5565da --- /dev/null +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_rename_command.rs @@ -0,0 +1,19 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_rename_dataset +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_repo_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_repo_command.rs new file mode 100644 index 000000000..eeec2744e --- /dev/null +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_repo_command.rs @@ -0,0 +1,26 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_repository_pull_aliases_commands +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_repository_push_aliases_commands +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_reset_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_reset_command.rs new file mode 100644 index 000000000..1161e8059 --- /dev/null +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_reset_command.rs @@ -0,0 +1,20 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_reset, + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_search_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_search_command.rs new file mode 100644 index 000000000..135de4131 --- /dev/null +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_search_command.rs @@ -0,0 +1,40 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_search_by_name + // We need synthetic time for the tests, but the third-party JWT code + // uses the current time. Assuming that the token lifetime is 24 hours, we will + // use the projected date (the current day) as a workaround. + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_search_by_repo + // We need synthetic time for the tests, but the third-party JWT code + // uses the current time. Assuming that the token lifetime is 24 hours, we will + // use the projected date (the current day) as a workaround. + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_sql_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_sql_command.rs new file mode 100644 index 000000000..339bc6586 --- /dev/null +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_sql_command.rs @@ -0,0 +1,38 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_datafusion_cli, + extra_test_groups = "engine, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_datafusion_cli_not_launched_in_root_ws, + options = Options::default().with_no_workspace(), + extra_test_groups = "engine, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_sql_command, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_system_diagnose_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_system_diagnose_command.rs new file mode 100644 index 000000000..9d1968db8 --- /dev/null +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_system_diagnose_command.rs @@ -0,0 +1,19 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_system_diagnose +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_system_gc_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_system_gc_command.rs new file mode 100644 index 000000000..0a95452b5 --- /dev/null +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_system_gc_command.rs @@ -0,0 +1,19 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_gc +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_system_info_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_system_info_command.rs new file mode 100644 index 000000000..63a713ca8 --- /dev/null +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_system_info_command.rs @@ -0,0 +1,19 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_system_info +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_tail_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_tail_command.rs new file mode 100644 index 000000000..9950b16af --- /dev/null +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_tail_command.rs @@ -0,0 +1,21 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_tail, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_verify_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_verify_command.rs new file mode 100644 index 000000000..e2f423962 --- /dev/null +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_verify_command.rs @@ -0,0 +1,36 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_verify_regular_dataset + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_verify_recursive + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_verify_integrity + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/test_smart_transfer_protocol.rs b/src/e2e/app/cli/postgres/tests/tests/test_smart_transfer_protocol.rs index fd35d189c..d1faad338 100644 --- a/src/e2e/app/cli/postgres/tests/tests/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/postgres/tests/tests/test_smart_transfer_protocol.rs @@ -24,3 +24,104 @@ kamu_cli_run_api_server_e2e_test!( ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_force_push_pull, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_add_alias, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_as, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_all, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_recursive, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_set_watermark, + options = Options::default().with_frozen_system_time(), +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_reset_derivative, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_visibility, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_s3, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_derivative, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// From 2b9720337b6bd402ebffde9eacfacd998d4dac34 Mon Sep 17 00:00:00 2001 From: Dima Pristupa Date: Mon, 14 Oct 2024 19:09:47 +0300 Subject: [PATCH 11/13] E2E: remove "kamu config" tests for SQLite & PostgreSQL --- .../cli/postgres/tests/tests/commands/mod.rs | 1 - .../tests/commands/test_config_command.rs | 33 ------------------- .../cli/sqlite/tests/tests/commands/mod.rs | 1 - .../tests/commands/test_config_command.rs | 33 ------------------- 4 files changed, 68 deletions(-) delete mode 100644 src/e2e/app/cli/postgres/tests/tests/commands/test_config_command.rs delete mode 100644 src/e2e/app/cli/sqlite/tests/tests/commands/test_config_command.rs diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/mod.rs b/src/e2e/app/cli/postgres/tests/tests/commands/mod.rs index c3408f409..6c06ec61a 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/mod.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/mod.rs @@ -9,7 +9,6 @@ mod test_add_command; mod test_compact_command; -mod test_config_command; mod test_delete_command; mod test_ingest_command; mod test_init_command; diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_config_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_config_command.rs deleted file mode 100644 index 0547a3e80..000000000 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_config_command.rs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Kamu Data, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -use kamu_cli_e2e_common::prelude::*; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -kamu_cli_execute_command_e2e_test!( - storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_config_set_value -); - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -kamu_cli_execute_command_e2e_test!( - storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_config_reset_key -); - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -kamu_cli_execute_command_e2e_test!( - storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_config_get_with_default -); - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/mod.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/mod.rs index c3408f409..6c06ec61a 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/mod.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/mod.rs @@ -9,7 +9,6 @@ mod test_add_command; mod test_compact_command; -mod test_config_command; mod test_delete_command; mod test_ingest_command; mod test_init_command; diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_config_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_config_command.rs deleted file mode 100644 index 111611ad7..000000000 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_config_command.rs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Kamu Data, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -use kamu_cli_e2e_common::prelude::*; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -kamu_cli_execute_command_e2e_test!( - storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_config_set_value -); - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -kamu_cli_execute_command_e2e_test!( - storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_config_reset_key -); - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -kamu_cli_execute_command_e2e_test!( - storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_config_get_with_default -); - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// From a7f94e7d3005c2829cdc280a1a93aa6d5d38a5d7 Mon Sep 17 00:00:00 2001 From: Dima Pristupa Date: Tue, 15 Oct 2024 17:13:04 +0300 Subject: [PATCH 12/13] E2E: test_smart_pull_derivative(): use the tail command + stabilization (#903) * E2E: test_smart_pull_derivative(): use the tail command * E2E: run postgres tests one by one * E2E: test_smart_pull_derivative(): stabilize order * E2E: wait_for_flows_to_finish(): increase takes count * E2E, test_ingest_command: stabilize order without the "match_time" changing * E2E, test_log_command: stabilize order without the "match_time" changing * E2E, test_sql_command: stabilize order without the "match_time" changing * E2E, test_tail_command: stabilize order without the "match_time" changing * E2E, test_dataset_trigger_flow(): stabilize order without the "match_time" changing * E2E, test_smart_transfer_protocol: stabilize order without the "match_time" changing --- .config/nextest.toml | 6 + .../common/src/kamu_api_server_client_ext.rs | 12 +- .../src/commands/test_ingest_command.rs | 16 +- .../src/commands/test_log_command.rs | 20 +- .../src/commands/test_sql_command.rs | 14 +- .../src/commands/test_tail_command.rs | 12 +- src/e2e/app/cli/repo-tests/src/test_flow.rs | 6 +- .../src/test_smart_transfer_protocol.rs | 204 +++++++++--------- 8 files changed, 146 insertions(+), 144 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 090f4047e..280dc51c2 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -3,6 +3,7 @@ fail-fast = false [test-groups] mysql = { max-threads = 1 } # Explanation below +postgres = { max-threads = 1 } # Explanation below setup = { max-threads = 1 } # Serialize the setup steps containerized = { max-threads = 8 } # Don't use too much memory engine = { max-threads = 2 } # Engine tests are very memory-hungry @@ -21,6 +22,11 @@ filter = "test(::mysql::)" test-group = "mysql" retries = { count = 3, backoff = "exponential", delay = "3s" } +[[profile.default.overrides]] +filter = "test(::postgres::)" +test-group = "postgres" +retries = { count = 3, backoff = "exponential", delay = "3s" } + [[profile.default.overrides]] filter = "test(::setup::)" test-group = "setup" diff --git a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs index ef4fc862e..e9033fbe3 100644 --- a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs +++ b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs @@ -104,33 +104,27 @@ lazy_static! { }; } -/// NOTE: 1 millisecond for stable order within tests -/// /// pub const DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1: &str = indoc::indoc!( r#" {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} - {"match_time": "2000-01-01 00:00:00.001", "match_id": 1, "player_id": "Bob", "score": 80} + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Bob", "score": 80} "# ); -/// NOTE: 1 millisecond for stable order within tests -/// /// pub const DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2: &str = indoc::indoc!( r#" {"match_time": "2000-01-02", "match_id": 2, "player_id": "Alice", "score": 70} - {"match_time": "2000-01-02 00:00:00.001", "match_id": 2, "player_id": "Charlie", "score": 90} + {"match_time": "2000-01-02", "match_id": 2, "player_id": "Charlie", "score": 90} "# ); -/// NOTE: 1 millisecond for stable order within tests -/// /// pub const DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_3: &str = indoc::indoc!( r#" {"match_time": "2000-01-03", "match_id": 3, "player_id": "Bob", "score": 60} - {"match_time": "2000-01-03 00:00:00.001", "match_id": 3, "player_id": "Charlie", "score": 110} + {"match_time": "2000-01-03", "match_id": 3, "player_id": "Charlie", "score": 110} "# ); diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_ingest_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_ingest_command.rs index 04224bcaa..124a735c7 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_ingest_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_ingest_command.rs @@ -198,8 +198,8 @@ pub async fn test_ingest_from_stdin(kamu: KamuCliPuppet) { ┌────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ │ op │ system_time │ match_time │ match_id │ player_id │ score │ ├────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ - │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ └────┴──────────────────────┴──────────────────────┴──────────┴───────────┴───────┘ "# ), @@ -215,8 +215,8 @@ pub async fn test_ingest_from_stdin(kamu: KamuCliPuppet) { ┌────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ │ op │ system_time │ match_time │ match_id │ player_id │ score │ ├────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ - │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-02T00:00:00Z │ 2 │ Alice │ 70 │ │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-02T00:00:00Z │ 2 │ Charlie │ 90 │ └────┴──────────────────────┴──────────────────────┴──────────┴───────────┴───────┘ @@ -234,8 +234,8 @@ pub async fn test_ingest_from_stdin(kamu: KamuCliPuppet) { ┌────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ │ op │ system_time │ match_time │ match_id │ player_id │ score │ ├────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ - │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-02T00:00:00Z │ 2 │ Alice │ 70 │ │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-02T00:00:00Z │ 2 │ Charlie │ 90 │ │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-03T00:00:00Z │ 3 │ Bob │ 60 │ @@ -326,8 +326,8 @@ pub async fn test_ingest_with_source_name(kamu: KamuCliPuppet) { ┌────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ │ op │ system_time │ match_time │ match_id │ player_id │ score │ ├────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ - │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ └────┴──────────────────────┴──────────────────────┴──────────┴───────────┴───────┘ "# ), @@ -349,8 +349,8 @@ pub async fn test_ingest_with_source_name(kamu: KamuCliPuppet) { ┌────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ │ op │ system_time │ match_time │ match_id │ player_id │ score │ ├────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ - │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-02T00:00:00Z │ 2 │ Alice │ 70 │ │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-02T00:00:00Z │ 2 │ Charlie │ 90 │ └────┴──────────────────────┴──────────────────────┴──────────┴───────────┴───────┘ @@ -374,8 +374,8 @@ pub async fn test_ingest_with_source_name(kamu: KamuCliPuppet) { ┌────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ │ op │ system_time │ match_time │ match_id │ player_id │ score │ ├────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ - │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-02T00:00:00Z │ 2 │ Alice │ 70 │ │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-02T00:00:00Z │ 2 │ Charlie │ 90 │ │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-03T00:00:00Z │ 3 │ Bob │ 60 │ @@ -449,12 +449,12 @@ async fn assert_ingest_data_to_player_scores_from_stdio( r#" SELECT op, system_time, - DATE_TRUNC('second', match_time) as match_time, + match_time, match_id, player_id, score FROM "player-scores" - ORDER BY match_time; + ORDER BY match_id, score, player_id; "# ), "--output-format", diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_log_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_log_command.rs index d24bc9471..8ae5550a2 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_log_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_log_command.rs @@ -9,7 +9,7 @@ use std::assert_matches::assert_matches; -use chrono::{TimeZone, Timelike, Utc}; +use chrono::{TimeZone, Utc}; use kamu_cli_e2e_common::{ DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, @@ -172,16 +172,11 @@ pub async fn test_log(kamu: KamuCliPuppet) { OffsetInterval { start: 0, end: 1 }, actual_new_data.offset_interval ); - pretty_assertions::assert_eq!(1674, actual_new_data.size); + pretty_assertions::assert_eq!(1665, actual_new_data.size); pretty_assertions::assert_eq!(None, actual_add_data.new_checkpoint); pretty_assertions::assert_eq!( - Some( - Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0) - .unwrap() - .with_nanosecond(1_000_000) // 1 ms - .unwrap() - ), + Some(Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap()), actual_add_data.new_watermark ); pretty_assertions::assert_eq!(None, actual_add_data.new_source_state); @@ -202,16 +197,11 @@ pub async fn test_log(kamu: KamuCliPuppet) { OffsetInterval { start: 2, end: 3 }, actual_new_data.offset_interval ); - pretty_assertions::assert_eq!(1690, actual_new_data.size); + pretty_assertions::assert_eq!(1681, actual_new_data.size); pretty_assertions::assert_eq!(None, actual_add_data.new_checkpoint); pretty_assertions::assert_eq!( - Some( - Utc.with_ymd_and_hms(2000, 1, 2, 0, 0, 0) - .unwrap() - .with_nanosecond(1_000_000) // 1 ms - .unwrap() - ), + Some(Utc.with_ymd_and_hms(2000, 1, 2, 0, 0, 0).unwrap()), actual_add_data.new_watermark ); pretty_assertions::assert_eq!(None, actual_add_data.new_source_state); diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_sql_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_sql_command.rs index 24f3576ab..bca4c7928 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_sql_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_sql_command.rs @@ -137,7 +137,7 @@ pub async fn test_sql_command(kamu: KamuCliPuppet) { .execute([ "sql", "--command", - "SELECT * FROM \"player-scores\";", + "SELECT * FROM \"player-scores\" ORDER BY offset;", "--output-format", "table", ]) @@ -149,12 +149,12 @@ pub async fn test_sql_command(kamu: KamuCliPuppet) { pretty_assertions::assert_eq!( indoc::indoc!( r#" - ┌────────┬────┬──────────────────────┬──────────────────────────┬──────────┬───────────┬───────┐ - │ offset │ op │ system_time │ match_time │ match_id │ player_id │ score │ - ├────────┼────┼──────────────────────┼──────────────────────────┼──────────┼───────────┼───────┤ - │ 0 │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ - │ 1 │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00.001Z │ 1 │ Bob │ 80 │ - └────────┴────┴──────────────────────┴──────────────────────────┴──────────┴───────────┴───────┘ + ┌────────┬────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ + │ offset │ op │ system_time │ match_time │ match_id │ player_id │ score │ + ├────────┼────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ + │ 0 │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ + │ 1 │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ + └────────┴────┴──────────────────────┴──────────────────────┴──────────┴───────────┴───────┘ "# ), stdout diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_tail_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_tail_command.rs index 54242ad84..cd486052b 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_tail_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_tail_command.rs @@ -52,12 +52,12 @@ pub async fn test_tail(kamu: KamuCliPuppet) { pretty_assertions::assert_eq!( indoc::indoc!( r#" - ┌────────┬────┬──────────────────────┬──────────────────────────┬──────────┬───────────┬───────┐ - │ offset │ op │ system_time │ match_time │ match_id │ player_id │ score │ - ├────────┼────┼──────────────────────┼──────────────────────────┼──────────┼───────────┼───────┤ - │ 0 │ +A │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ - │ 1 │ +A │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00.001Z │ 1 │ Bob │ 80 │ - └────────┴────┴──────────────────────┴──────────────────────────┴──────────┴───────────┴───────┘ + ┌────────┬────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ + │ offset │ op │ system_time │ match_time │ match_id │ player_id │ score │ + ├────────┼────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ + │ 0 │ +A │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ + │ 1 │ +A │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ + └────────┴────┴──────────────────────┴──────────────────────┴──────────┴───────────┴───────┘ "# ), stdout diff --git a/src/e2e/app/cli/repo-tests/src/test_flow.rs b/src/e2e/app/cli/repo-tests/src/test_flow.rs index eb96c840a..d1ff857fb 100644 --- a/src/e2e/app/cli/repo-tests/src/test_flow.rs +++ b/src/e2e/app/cli/repo-tests/src/test_flow.rs @@ -615,7 +615,7 @@ pub async fn test_dataset_trigger_flow(kamu_api_server_client: KamuApiServerClie "__typename": "FlowDescriptionUpdateResultSuccess", "numBlocks": 2, "numRecords": 2, - "updatedWatermark": "2000-01-01T00:00:00.001+00:00" + "updatedWatermark": "2000-01-01T00:00:00+00:00" } }, "initiator": { @@ -649,7 +649,7 @@ pub async fn test_dataset_trigger_flow(kamu_api_server_client: KamuApiServerClie "__typename": "FlowDescriptionUpdateResultSuccess", "numBlocks": 2, "numRecords": 2, - "updatedWatermark": "2000-01-01T00:00:00.001+00:00" + "updatedWatermark": "2000-01-01T00:00:00+00:00" } }, "initiator": { @@ -1169,7 +1169,7 @@ async fn wait_for_flows_to_finish( dataset_id: &str, token: AccessToken, ) { - let retry_strategy = FixedInterval::from_millis(5_000).take(10); + let retry_strategy = FixedInterval::from_millis(5_000).take(18); // 1m 30s Retry::spawn(retry_strategy, || async { let response = kamu_api_server_client diff --git a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs index 5deee2f3e..34b036eb6 100644 --- a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs @@ -563,12 +563,12 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien ); let expected_data = indoc::indoc!( r#" - +--------+----+----------------------+--------------------------+----------+-----------+-------+ - | offset | op | system_time | match_time | match_id | player_id | score | - +--------+----+----------------------+--------------------------+----------+-----------+-------+ - | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | - | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | - +--------+----+----------------------+--------------------------+----------+-----------+-------+ + +--------+----+----------------------+----------------------+----------+-----------+-------+ + | offset | op | system_time | match_time | match_id | player_id | score | + +--------+----+----------------------+----------------------+----------+-----------+-------+ + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Bob | 80 | + +--------+----+----------------------+----------------------+----------+-----------+-------+ "# ); let expected_derivative_schema = indoc::indoc!( @@ -587,12 +587,12 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien ); let expected_derivative_data = indoc::indoc!( r#" - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | offset | op | system_time | match_time | place | match_id | player_id | score | - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | - | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 2 | 1 | Bob | 80 | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ "# ); @@ -656,22 +656,22 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien // Perform dataslices checks let expected_data = indoc::indoc!( r#" - +--------+----+----------------------+--------------------------+----------+-----------+-------+ - | offset | op | system_time | match_time | match_id | player_id | score | - +--------+----+----------------------+--------------------------+----------+-----------+-------+ - | 2 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | Alice | 70 | - | 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 2 | Charlie | 90 | - +--------+----+----------------------+--------------------------+----------+-----------+-------+ + +--------+----+----------------------+----------------------+----------+-----------+-------+ + | offset | op | system_time | match_time | match_id | player_id | score | + +--------+----+----------------------+----------------------+----------+-----------+-------+ + | 2 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | Charlie | 90 | + | 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | Alice | 70 | + +--------+----+----------------------+----------------------+----------+-----------+-------+ "# ); let expected_derivative_data = indoc::indoc!( r#" - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | offset | op | system_time | match_time | place | match_id | player_id | score | - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 2 | 1 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | - | 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 2 | 2 | Charlie | 90 | - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | 2 | 1 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 2 | 1 | Bob | 80 | + | 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | 2 | Charlie | 90 | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ "# ); @@ -804,12 +804,12 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe ); let expected_data = indoc::indoc!( r#" - +--------+----+----------------------+--------------------------+----------+-----------+-------+ - | offset | op | system_time | match_time | match_id | player_id | score | - +--------+----+----------------------+--------------------------+----------+-----------+-------+ - | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | - | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | - +--------+----+----------------------+--------------------------+----------+-----------+-------+ + +--------+----+----------------------+----------------------+----------+-----------+-------+ + | offset | op | system_time | match_time | match_id | player_id | score | + +--------+----+----------------------+----------------------+----------+-----------+-------+ + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Bob | 80 | + +--------+----+----------------------+----------------------+----------+-----------+-------+ "# ); let expected_derivative_schema = indoc::indoc!( @@ -828,12 +828,12 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe ); let expected_derivative_data = indoc::indoc!( r#" - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | offset | op | system_time | match_time | place | match_id | player_id | score | - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | - | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 2 | 1 | Bob | 80 | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ "# ); @@ -879,22 +879,22 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe // Perform dataslices checks let expected_data = indoc::indoc!( r#" - +--------+----+----------------------+--------------------------+----------+-----------+-------+ - | offset | op | system_time | match_time | match_id | player_id | score | - +--------+----+----------------------+--------------------------+----------+-----------+-------+ - | 2 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | Alice | 70 | - | 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 2 | Charlie | 90 | - +--------+----+----------------------+--------------------------+----------+-----------+-------+ + +--------+----+----------------------+----------------------+----------+-----------+-------+ + | offset | op | system_time | match_time | match_id | player_id | score | + +--------+----+----------------------+----------------------+----------+-----------+-------+ + | 2 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | Charlie | 90 | + | 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | Alice | 70 | + +--------+----+----------------------+----------------------+----------+-----------+-------+ "# ); let expected_derivative_data = indoc::indoc!( r#" - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | offset | op | system_time | match_time | place | match_id | player_id | score | - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 2 | 1 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | - | 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 2 | 2 | Charlie | 90 | - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | 2 | 1 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 2 | 1 | Bob | 80 | + | 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | 2 | Charlie | 90 | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ "# ); @@ -984,12 +984,12 @@ pub async fn test_smart_pull_reset_derivative(kamu: KamuCliPuppet) { ); let expected_derivative_data = indoc::indoc!( r#" - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | offset | op | system_time | match_time | place | match_id | player_id | score | - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | - | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 2 | 1 | Bob | 80 | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ "# ); kamu.assert_last_data_slice( @@ -1039,12 +1039,12 @@ pub async fn test_smart_pull_reset_derivative(kamu: KamuCliPuppet) { let expected_derivative_data = indoc::indoc!( r#" - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | offset | op | system_time | match_time | place | match_id | player_id | score | - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | 2 | Alice | 70 | - | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 1 | 2 | Charlie | 90 | - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | 2 | Alice | 70 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 1 | 2 | Charlie | 90 | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ "# ); kamu.assert_last_data_slice( @@ -1171,12 +1171,12 @@ pub async fn test_smart_push_pull_s3(kamu: KamuCliPuppet) { ); let expected_data = indoc::indoc!( r#" - +--------+----+----------------------+--------------------------+----------+-----------+-------+ - | offset | op | system_time | match_time | match_id | player_id | score | - +--------+----+----------------------+--------------------------+----------+-----------+-------+ - | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | - | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | - +--------+----+----------------------+--------------------------+----------+-----------+-------+ + +--------+----+----------------------+----------------------+----------+-----------+-------+ + | offset | op | system_time | match_time | match_id | player_id | score | + +--------+----+----------------------+----------------------+----------+-----------+-------+ + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Bob | 80 | + +--------+----+----------------------+----------------------+----------+-----------+-------+ "# ); kamu.assert_last_data_slice(&dataset_name, expected_schema, expected_data) @@ -1226,37 +1226,49 @@ pub async fn test_smart_pull_derivative(kamu: KamuCliPuppet) { ) .await; - let expected_derivative_schema = indoc::indoc!( - r#" - message arrow_schema { - OPTIONAL INT64 offset; - REQUIRED INT32 op; - REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); - OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); - OPTIONAL INT64 place; - OPTIONAL INT64 match_id; - OPTIONAL BYTE_ARRAY player_id (STRING); - OPTIONAL INT64 score; - } - "# - ); + { + let assert = kamu + .execute([ + "sql", + "--engine", + "datafusion", + "--command", + // Without unstable "offset" column. + // For a beautiful output, cut to seconds + indoc::indoc!( + r#" + SELECT op, + system_time, + DATE_TRUNC('second', match_time) as match_time, + match_id, + player_id, + score + FROM "player-scores" + ORDER BY match_time; + "# + ), + "--output-format", + "table", + ]) + .await + .success(); - let expected_derivative_data = indoc::indoc!( - r#" - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | offset | op | system_time | match_time | place | match_id | player_id | score | - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | - | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | - +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - "# - ); - kamu.assert_last_data_slice( - &dataset_derivative_name, - expected_derivative_schema, - expected_derivative_data, - ) - .await; + let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + pretty_assertions::assert_eq!( + indoc::indoc!( + r#" + ┌────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ + │ op │ system_time │ match_time │ match_id │ player_id │ score │ + ├────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ + └────┴──────────────────────┴──────────────────────┴──────────┴───────────┴───────┘ + "# + ), + stdout + ); + } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// From ef119fec716e3f5da318de93f4a2ddd877b0532a Mon Sep 17 00:00:00 2001 From: Dima Pristupa Date: Wed, 16 Oct 2024 00:23:14 +0300 Subject: [PATCH 13/13] KamuCliPuppetExt: absorb helper methods (#906) * E2E, test_rest_api_request_dataset_tail(): use a const instead of a hardcoded value * E2E, test_verify_command: use a consts instead of hardcoded values * E2E: KamuApiServerClient::get_node_url(): absorb * CI: mark postgres as flaky but run with 8 threads * Tests: add flaky "risingwave" group * E2E, test_dataset_trigger_flow(): add "risingwave" test-group * CI: go not specify threads for postgres * E2E, KamuCliPuppetExt::assert_player_scores_dataset_data(): extract * E2E, KamuCliPuppetExt::assert_success_command_execution(): absorb * E2E, KamuCliPuppetExt::assert_failure_command_execution(): absorb * E2E, assert_ingest_data_to_player_scores_from_stdio(): use assert_success_command_execution_with_input() * KamuCliPuppetExt::assert_player_scores_dataset_data(): use assert_success_command_execution() * E2E, test_config_command: use new methods * E2E, test_inspect_command: use new methods * E2E, test_search_command: use new methods * E2E, test_sql_command: use new methods * E2E, test_tail_command: use new methods * E2E, test_system_api_server_gql_query: use new methods * E2E, test_add_command: use new methods * KamuCliPuppetExt: use iter for maybe_expected_stderr * E2E, test_compact_command: use new methods * E2E, test_delete_command: use new methods * E2E, test_ingest_command: use new methods * E2E, test_init_command: use new methods * E2E, test_login_command: use new methods * E2E, test_new_command: use new methods * E2E, test_rename_command: use new methods * E2E, test_reset_command: use new methods * E2E, test_smart_transfer_protocol: use new methods * E2E, test_system_gc_command: use new methods * E2E, test_verify_command: use new methods * E2E, test_workspace_svc: use new methods * KamuCliPuppetExt::assert_failure_command_execution_with_input(): implement --- .config/nextest.toml | 18 +- Cargo.lock | 2 + src/app/cli/tests/tests/test_workspace_svc.rs | 18 +- .../cli/common/src/kamu_api_server_client.rs | 17 + .../tests/commands/test_verify_command.rs | 8 +- .../app/cli/inmem/tests/tests/test_flow.rs | 2 +- .../tests/test_smart_transfer_protocol.rs | 8 +- .../app/cli/mysql/tests/tests/test_flow.rs | 2 +- .../tests/commands/test_verify_command.rs | 8 +- .../app/cli/postgres/tests/tests/test_flow.rs | 2 +- .../tests/test_smart_transfer_protocol.rs | 8 +- .../src/commands/test_add_command.rs | 223 ++-- .../src/commands/test_compact_command.rs | 84 +- .../src/commands/test_config_command.rs | 332 +++--- .../src/commands/test_delete_command.rs | 66 +- .../src/commands/test_ingest_command.rs | 98 +- .../src/commands/test_init_command.rs | 28 +- .../src/commands/test_inspect_command.rs | 205 ++-- .../src/commands/test_login_command.rs | 213 ++-- .../src/commands/test_new_command.rs | 39 +- .../src/commands/test_rename_command.rs | 38 +- .../src/commands/test_reset_command.rs | 29 +- .../src/commands/test_search_command.rs | 142 ++- .../src/commands/test_sql_command.rs | 163 ++- .../test_system_api_server_gql_query.rs | 21 +- .../src/commands/test_system_gc_command.rs | 15 +- .../src/commands/test_tail_command.rs | 57 +- .../src/commands/test_verify_command.rs | 121 +-- .../app/cli/repo-tests/src/test_rest_api.rs | 9 +- .../src/test_smart_transfer_protocol.rs | 995 +++++++++--------- .../tests/commands/test_verify_command.rs | 8 +- .../app/cli/sqlite/tests/tests/test_flow.rs | 2 +- .../tests/test_smart_transfer_protocol.rs | 8 +- src/utils/kamu-cli-puppet/Cargo.toml | 4 + .../src/kamu_cli_puppet_ext.rs | 167 ++- 35 files changed, 1434 insertions(+), 1726 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 280dc51c2..283a80316 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -3,20 +3,20 @@ fail-fast = false [test-groups] mysql = { max-threads = 1 } # Explanation below -postgres = { max-threads = 1 } # Explanation below setup = { max-threads = 1 } # Serialize the setup steps containerized = { max-threads = 8 } # Don't use too much memory engine = { max-threads = 2 } # Engine tests are very memory-hungry database = { max-threads = 8 } # Don't use too much memory +# NOTE: --> There is an incompatibility between nextest and sqlx: +# - nextest implies multiprocessing, +# - while sqlx has a lock on cleanup within the current process +# (https://github.com/launchbadge/sqlx/pull/2640#issuecomment-1659455042). + # TODO: Delete this workaround when this PR is merged: # - Fix: nextest cleanup race condition by bonega # https://github.com/launchbadge/sqlx/pull/3334 # -# NOTE: There is an incompatibility between nextest and sqlx: -# - nextest implies multiprocessing, -# - while sqlx has a lock on cleanup within the current process -# (https://github.com/launchbadge/sqlx/pull/2640#issuecomment-1659455042). [[profile.default.overrides]] filter = "test(::mysql::)" test-group = "mysql" @@ -24,7 +24,13 @@ retries = { count = 3, backoff = "exponential", delay = "3s" } [[profile.default.overrides]] filter = "test(::postgres::)" -test-group = "postgres" +retries = { count = 3, backoff = "exponential", delay = "3s" } +# NOTE: <-- There is an incompatibility between nextest and sqlx + +# NOTE: Periodic missing rows when the system is under load +# https://github.com/kamu-data/kamu-engine-risingwave/issues/7 +[[profile.default.overrides]] +filter = "test(::risingwave::)" retries = { count = 3, backoff = "exponential", delay = "3s" } [[profile.default.overrides]] diff --git a/Cargo.lock b/Cargo.lock index a3870a159..e8b8a5fc2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5791,8 +5791,10 @@ dependencies = [ "async-trait", "chrono", "datafusion", + "indoc 2.0.5", "kamu-data-utils", "opendatafabric", + "pretty_assertions", "serde", "serde_json", "tempfile", diff --git a/src/app/cli/tests/tests/test_workspace_svc.rs b/src/app/cli/tests/tests/test_workspace_svc.rs index ae37a6ce1..417cd5b60 100644 --- a/src/app/cli/tests/tests/test_workspace_svc.rs +++ b/src/app/cli/tests/tests/test_workspace_svc.rs @@ -14,6 +14,7 @@ use kamu::domain::*; use kamu::testing::{MetadataFactory, ParquetWriterHelper}; use kamu::*; use kamu_cli::*; +use kamu_cli_puppet::extensions::KamuCliPuppetExt; use kamu_cli_puppet::KamuCliPuppet; use opendatafabric::serde::yaml::Manifest; use opendatafabric::*; @@ -46,16 +47,15 @@ async fn test_workspace_upgrade() { let kamu = KamuCliPuppet::new(temp_dir.path()); - let assert = kamu.execute(["list"]).await.failure(); - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains( + kamu.assert_failure_command_execution( + ["list"], + None, + Some([ "Error: Workspace needs to be upgraded before continuing - please run `kamu system \ - upgrade-workspace`" - ), - "Unexpected output:\n{stderr}", - ); + upgrade-workspace`", + ]), + ) + .await; // TODO: Restore this test upon the first upgrade post V5 breaking changes /* diff --git a/src/e2e/app/cli/common/src/kamu_api_server_client.rs b/src/e2e/app/cli/common/src/kamu_api_server_client.rs index 5cdfc97a0..dfee2940b 100644 --- a/src/e2e/app/cli/common/src/kamu_api_server_client.rs +++ b/src/e2e/app/cli/common/src/kamu_api_server_client.rs @@ -8,6 +8,7 @@ // by the Apache License, Version 2.0. use internal_error::{InternalError, ResultIntoInternal}; +use opendatafabric::DatasetAlias; use reqwest::{Method, Response, StatusCode, Url}; use serde::Deserialize; use tokio_retry::strategy::FixedInterval; @@ -79,6 +80,22 @@ impl KamuApiServerClient { &self.server_base_url } + pub fn get_node_url(&self) -> Url { + let mut node_url = Url::parse("odf+http://host").unwrap(); + let base_url = self.get_base_url(); + + node_url.set_host(base_url.host_str()).unwrap(); + node_url.set_port(base_url.port()).unwrap(); + + node_url + } + + pub fn get_dataset_endpoint(&self, dataset_alias: &DatasetAlias) -> Url { + let node_url = self.get_node_url(); + + node_url.join(format!("{dataset_alias}").as_str()).unwrap() + } + pub async fn rest_api_call_assert( &self, token: Option, diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_verify_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_verify_command.rs index 0a4b5598a..cd27f148b 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_verify_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_verify_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_verify_regular_dataset + fixture = kamu_cli_e2e_repo_tests::test_verify_regular_dataset, extra_test_groups = "engine, ingest, datafusion" ); @@ -21,15 +21,15 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_verify_recursive - extra_test_groups = "containerized, engine, ingest, datafusion" + fixture = kamu_cli_e2e_repo_tests::test_verify_recursive, + extra_test_groups = "containerized, engine, ingest, datafusion, risingwave" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_verify_integrity + fixture = kamu_cli_e2e_repo_tests::test_verify_integrity, extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/inmem/tests/tests/test_flow.rs b/src/e2e/app/cli/inmem/tests/tests/test_flow.rs index 665fa0e8e..e1684e70a 100644 --- a/src/e2e/app/cli/inmem/tests/tests/test_flow.rs +++ b/src/e2e/app/cli/inmem/tests/tests/test_flow.rs @@ -38,7 +38,7 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = inmem, fixture = kamu_cli_e2e_repo_tests::test_dataset_trigger_flow, - extra_test_groups = "containerized, engine, transform, datafusion" + extra_test_groups = "containerized, engine, transform, datafusion, risingwave" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs b/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs index abb5120d3..4dd8081a0 100644 --- a/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs @@ -64,7 +64,7 @@ kamu_cli_run_api_server_e2e_test!( options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, transform, datafusion" + extra_test_groups = "containerized, engine, ingest, transform, datafusion, risingwave" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -75,7 +75,7 @@ kamu_cli_run_api_server_e2e_test!( options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, transform, datafusion" + extra_test_groups = "containerized, engine, ingest, transform, datafusion, risingwave" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -92,7 +92,7 @@ kamu_cli_execute_command_e2e_test!( storage = inmem, fixture = kamu_cli_e2e_repo_tests::test_smart_pull_reset_derivative, options = Options::default().with_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, transform, datafusion" + extra_test_groups = "containerized, engine, ingest, transform, datafusion, risingwave" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -121,7 +121,7 @@ kamu_cli_execute_command_e2e_test!( storage = inmem, fixture = kamu_cli_e2e_repo_tests::test_smart_pull_derivative, options = Options::default().with_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, transform, datafusion" + extra_test_groups = "containerized, engine, ingest, transform, datafusion, risingwave" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/mysql/tests/tests/test_flow.rs b/src/e2e/app/cli/mysql/tests/tests/test_flow.rs index 50ebbb6e5..b7d1fe4c5 100644 --- a/src/e2e/app/cli/mysql/tests/tests/test_flow.rs +++ b/src/e2e/app/cli/mysql/tests/tests/test_flow.rs @@ -38,7 +38,7 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = mysql, fixture = kamu_cli_e2e_repo_tests::test_dataset_trigger_flow, - extra_test_groups = "containerized, engine, transform, datafusion" + extra_test_groups = "containerized, engine, transform, datafusion, risingwave" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_verify_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_verify_command.rs index e2f423962..ac98ac359 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_verify_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_verify_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_verify_regular_dataset + fixture = kamu_cli_e2e_repo_tests::test_verify_regular_dataset, extra_test_groups = "engine, ingest, datafusion" ); @@ -21,15 +21,15 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_verify_recursive - extra_test_groups = "containerized, engine, ingest, datafusion" + fixture = kamu_cli_e2e_repo_tests::test_verify_recursive, + extra_test_groups = "containerized, engine, ingest, datafusion, risingwave" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_verify_integrity + fixture = kamu_cli_e2e_repo_tests::test_verify_integrity, extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/postgres/tests/tests/test_flow.rs b/src/e2e/app/cli/postgres/tests/tests/test_flow.rs index d0872f75d..f81d9769d 100644 --- a/src/e2e/app/cli/postgres/tests/tests/test_flow.rs +++ b/src/e2e/app/cli/postgres/tests/tests/test_flow.rs @@ -38,7 +38,7 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = postgres, fixture = kamu_cli_e2e_repo_tests::test_dataset_trigger_flow, - extra_test_groups = "containerized, engine, transform, datafusion" + extra_test_groups = "containerized, engine, transform, datafusion, risingwave" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/test_smart_transfer_protocol.rs b/src/e2e/app/cli/postgres/tests/tests/test_smart_transfer_protocol.rs index d1faad338..b943affd1 100644 --- a/src/e2e/app/cli/postgres/tests/tests/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/postgres/tests/tests/test_smart_transfer_protocol.rs @@ -64,7 +64,7 @@ kamu_cli_run_api_server_e2e_test!( options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, transform, datafusion" + extra_test_groups = "containerized, engine, ingest, transform, datafusion, risingwave" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -75,7 +75,7 @@ kamu_cli_run_api_server_e2e_test!( options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, transform, datafusion" + extra_test_groups = "containerized, engine, ingest, transform, datafusion, risingwave" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -92,7 +92,7 @@ kamu_cli_execute_command_e2e_test!( storage = postgres, fixture = kamu_cli_e2e_repo_tests::test_smart_pull_reset_derivative, options = Options::default().with_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, transform, datafusion" + extra_test_groups = "containerized, engine, ingest, transform, datafusion, risingwave" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -121,7 +121,7 @@ kamu_cli_execute_command_e2e_test!( storage = postgres, fixture = kamu_cli_e2e_repo_tests::test_smart_pull_derivative, options = Options::default().with_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, transform, datafusion" + extra_test_groups = "containerized, engine, ingest, transform, datafusion, risingwave" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_add_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_add_command.rs index 8cba1ea12..022fd1ba5 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_add_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_add_command.rs @@ -16,22 +16,18 @@ use opendatafabric as odf; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_add_dataset_from_stdin(kamu: KamuCliPuppet) { - let assert = kamu - .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(indoc::indoc!( + kamu.assert_success_command_execution_with_input( + ["add", "--stdin"], + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, + None, + Some([indoc::indoc!( r#" Added: player-scores Added 1 dataset(s) "# - )), - "Unexpected output:\n{stderr}", - ); + )]), + ) + .await; let dataset_names = kamu .list_datasets() @@ -47,59 +43,44 @@ pub async fn test_add_dataset_from_stdin(kamu: KamuCliPuppet) { pub async fn test_add_dataset_with_name(kamu: KamuCliPuppet) { // Add from stdio - { - let assert = kamu - .execute_with_input( - ["add", "--stdin", "--name", "player-scores-1"], - DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, - ) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(indoc::indoc!( - r#" - Added: player-scores-1 - Added 1 dataset(s) - "# - )), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_success_command_execution_with_input( + ["add", "--stdin", "--name", "player-scores-1"], + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, + None, + Some([indoc::indoc!( + r#" + Added: player-scores-1 + Added 1 dataset(s) + "# + )]), + ) + .await; + // Add from a file - { - let snapshot_path = kamu.workspace_path().join("player-scores.yaml"); + let snapshot_path = kamu.workspace_path().join("player-scores.yaml"); - std::fs::write( - snapshot_path.clone(), - DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, - ) - .unwrap(); + std::fs::write( + snapshot_path.clone(), + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, + ) + .unwrap(); - let assert = kamu - .execute([ - "add", - "--name", - "player-scores-2", - snapshot_path.to_str().unwrap(), - ]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(indoc::indoc!( - r#" - Added: player-scores-2 - Added 1 dataset(s) - "# - )), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_success_command_execution( + [ + "add", + "--name", + "player-scores-2", + snapshot_path.to_str().unwrap(), + ], + None, + Some([indoc::indoc!( + r#" + Added: player-scores-2 + Added 1 dataset(s) + "# + )]), + ) + .await; let dataset_names = kamu .list_datasets() @@ -114,62 +95,43 @@ pub async fn test_add_dataset_with_name(kamu: KamuCliPuppet) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_add_dataset_with_replace(kamu: KamuCliPuppet) { - { - let assert = kamu - .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(indoc::indoc!( - r#" - Added: player-scores - "# - )), - "Unexpected output:\n{stderr}", - ); - } - { - let assert = kamu - .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(indoc::indoc!( - r#" - Skipped: player-scores: Already exists - Added 0 dataset(s) - "# - )), - "Unexpected output:\n{stderr}", - ); - } - { - let assert = kamu - .execute_with_input( - ["--yes", "add", "--stdin", "--replace"], - DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, - ) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(indoc::indoc!( - r#" - Added: player-scores - Added 1 dataset(s) - "# - )), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_success_command_execution_with_input( + ["add", "--stdin"], + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, + None, + Some([indoc::indoc!( + r#" + Added: player-scores + "# + )]), + ) + .await; + + kamu.assert_success_command_execution_with_input( + ["add", "--stdin"], + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, + None, + Some([indoc::indoc!( + r#" + Skipped: player-scores: Already exists + Added 0 dataset(s) + "# + )]), + ) + .await; + + kamu.assert_success_command_execution_with_input( + ["--yes", "add", "--stdin", "--replace"], + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, + None, + Some([indoc::indoc!( + r#" + Added: player-scores + Added 1 dataset(s) + "# + )]), + ) + .await; let dataset_names = kamu .list_datasets() @@ -229,14 +191,23 @@ pub async fn test_add_recursive(kamu: KamuCliPuppet) { ) .unwrap(); - kamu.execute([ - "-v", - "add", - "--recursive", - kamu.workspace_path().as_os_str().to_str().unwrap(), - ]) - .await - .success(); + kamu.assert_success_command_execution( + [ + "-v", + "add", + "--recursive", + kamu.workspace_path().as_os_str().to_str().unwrap(), + ], + None, + Some([indoc::indoc!( + r#" + Added: commented + Added: plain + Added 2 dataset(s) + "# + )]), + ) + .await; let dataset_names = kamu .list_datasets() diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_compact_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_compact_command.rs index 316359eb4..3747b77b9 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_compact_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_compact_command.rs @@ -40,27 +40,18 @@ pub async fn test_compact_hard(kamu: KamuCliPuppet) { let blocks_before_compacting = kamu.list_blocks(&dataset_name).await; - let assert = kamu - .execute([ + kamu.assert_success_command_execution( + [ "--yes", "system", "compact", dataset_name.as_str(), "--hard", - ]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(indoc::indoc!( - r#" - 1 dataset(s) were compacted - "# - )), - "Unexpected output:\n{stderr}", - ); + ], + None, + Some(["1 dataset(s) were compacted"]), + ) + .await; let blocks_after_compacting = kamu.list_blocks(&dataset_name).await; assert_eq!( @@ -95,28 +86,19 @@ pub async fn test_compact_keep_metadata_only(kamu: KamuCliPuppet) { let blocks_before_compacting = kamu.list_blocks(&dataset_name).await; - let assert = kamu - .execute([ + kamu.assert_success_command_execution( + [ "--yes", "system", "compact", dataset_name.as_str(), "--hard", "--keep-metadata-only", - ]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(indoc::indoc!( - r#" - 1 dataset(s) were compacted - "# - )), - "Unexpected output:\n{stderr}", - ); + ], + None, + Some(["1 dataset(s) were compacted"]), + ) + .await; let blocks_after_compacting = kamu.list_blocks(&dataset_name).await; assert_eq!( @@ -151,40 +133,22 @@ pub async fn test_compact_verify(kamu: KamuCliPuppet) { let blocks_before_compacting = kamu.list_blocks(&dataset_name).await; - let assert = kamu - .execute([ + kamu.assert_success_command_execution( + [ "--yes", "system", "compact", dataset_name.as_str(), "--hard", "--verify", - ]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains( - indoc::indoc!( - r#" - verify with dataset_ref: player-scores - "# - ) - .trim() - ), - "Unexpected output:\n{stderr}", - ); - - assert!( - stderr.contains(indoc::indoc!( - r#" - 1 dataset(s) were compacted - "# - )), - "Unexpected output:\n{stderr}", - ); + ], + None, + Some([ + "verify with dataset_ref: player-scores", + "1 dataset(s) were compacted", + ]), + ) + .await; let blocks_after_compacting = kamu.list_blocks(&dataset_name).await; assert_eq!( diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_config_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_config_command.rs index b1a3fcf19..8f5df7ddb 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_config_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_config_command.rs @@ -7,6 +7,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +use kamu_cli_puppet::extensions::KamuCliPuppetExt; use kamu_cli_puppet::KamuCliPuppet; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -14,223 +15,151 @@ use kamu_cli_puppet::KamuCliPuppet; pub async fn test_config_set_value(kamu: KamuCliPuppet) { // 0. CI sets container runtime to podman for some targets, so we simulate this // behavior for all others. - { - let assert = kamu - .execute(["config", "set", "engine.runtime", "podman"]) - .await - .success(); + kamu.assert_success_command_execution( + ["config", "set", "engine.runtime", "podman"], + None, + Some(["Set engine.runtime to podman in workspace scope"]), + ) + .await; - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + // 1. Set flow for the "engine.networkNs" key + kamu.assert_success_command_execution( + ["config", "list"], + Some(indoc::indoc!( + r#" + engine: + runtime: podman - assert!( - stderr.contains("Set engine.runtime to podman in workspace scope"), - "Unexpected output:\n{stderr}", - ); - } + "# + )), + None::>, + ) + .await; + + kamu.assert_success_command_execution( + ["config", "set", "engine.networkNs", "host"], + None, + Some(["Set engine.networkNs to host in workspace scope"]), + ) + .await; + + kamu.assert_success_command_execution( + ["config", "get", "engine.networkNs"], + Some(indoc::indoc!( + r#" + host + + "# + )), + None::>, + ) + .await; + + kamu.assert_success_command_execution( + ["config", "list"], + Some(indoc::indoc!( + r#" + engine: + runtime: podman + networkNs: host + + "# + )), + None::>, + ) + .await; - // 1. Set flow for the "engine.networkNs" key - { - let assert = kamu.execute(["config", "list"]).await.success(); - let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); - - pretty_assertions::assert_eq!( - indoc::indoc!( - r#" - engine: - runtime: podman - - "# - ), - stdout - ); - } - { - let assert = kamu - .execute(["config", "set", "engine.networkNs", "host"]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Set engine.networkNs to host in workspace scope"), - "Unexpected output:\n{stderr}", - ); - } - { - let assert = kamu - .execute(["config", "get", "engine.networkNs"]) - .await - .success(); - - let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); - - pretty_assertions::assert_eq!( - indoc::indoc!( - r#" - host - - "# - ), - stdout - ); - } - { - let assert = kamu.execute(["config", "list"]).await.success(); - let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); - - pretty_assertions::assert_eq!( - indoc::indoc!( - r#" - engine: - runtime: podman - networkNs: host - - "# - ), - stdout - ); - } // 2. Set flow for the "uploads.maxFileSizeInMb" key - { - let assert = kamu - .execute(["config", "set", "uploads.maxFileSizeInMb", "42"]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Set uploads.maxFileSizeInMb to 42 in workspace scope"), - "Unexpected output:\n{stderr}", - ); - } - { - let assert = kamu - .execute(["config", "get", "uploads.maxFileSizeInMb"]) - .await - .success(); - - let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); - - pretty_assertions::assert_eq!( - indoc::indoc!( - r#" - 42 - - "# - ), - stdout - ); - } - { - let assert = kamu.execute(["config", "list"]).await.success(); - let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); - - pretty_assertions::assert_eq!( - indoc::indoc!( - r#" - engine: - runtime: podman - networkNs: host - uploads: - maxFileSizeInMb: 42 - - "# - ), - stdout - ); - } + kamu.assert_success_command_execution( + ["config", "set", "uploads.maxFileSizeInMb", "42"], + None, + Some(["Set uploads.maxFileSizeInMb to 42 in workspace scope"]), + ) + .await; + + kamu.assert_success_command_execution( + ["config", "get", "uploads.maxFileSizeInMb"], + Some(indoc::indoc!( + r#" + 42 + + "# + )), + None::>, + ) + .await; + + kamu.assert_success_command_execution( + ["config", "list"], + Some(indoc::indoc!( + r#" + engine: + runtime: podman + networkNs: host + uploads: + maxFileSizeInMb: 42 + + "# + )), + None::>, + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_config_reset_key(kamu: KamuCliPuppet) { - { - let assert = kamu - .execute(["config", "set", "engine.networkNs", "host"]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Set engine.networkNs to host in workspace scope"), - "Unexpected output:\n{stderr}", - ); - } - { - let assert = kamu - .execute(["config", "set", "engine.networkNs"]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Removed engine.networkNs from workspace scope"), - "Unexpected output:\n{stderr}", - ); - } - { - let assert = kamu - .execute(["config", "get", "engine.networkNs"]) - .await - .failure(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Error: Key engine.networkNs not found"), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_success_command_execution( + ["config", "set", "engine.networkNs", "host"], + None, + Some(["Set engine.networkNs to host in workspace scope"]), + ) + .await; + + kamu.assert_success_command_execution( + ["config", "set", "engine.networkNs"], + None, + Some(["Removed engine.networkNs from workspace scope"]), + ) + .await; + + kamu.assert_failure_command_execution( + ["config", "get", "engine.networkNs"], + None, + Some(["Error: Key engine.networkNs not found"]), + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_config_get_with_default(kamu: KamuCliPuppet) { - { - let assert = kamu - .execute(["config", "get", "engine.networkNs"]) - .await - .failure(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Error: Key engine.networkNs not found"), - "Unexpected output:\n{stderr}", - ); - } - { - let assert = kamu - .execute(["config", "get", "engine.networkNs", "--with-defaults"]) - .await - .success(); - - let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); - - pretty_assertions::assert_eq!( - indoc::indoc!( - r#" - private - - "# - ), - stdout - ); - } + kamu.assert_failure_command_execution( + ["config", "get", "engine.networkNs"], + None, + Some(["Error: Key engine.networkNs not found"]), + ) + .await; + + kamu.assert_success_command_execution( + ["config", "get", "engine.networkNs", "--with-defaults"], + Some(indoc::indoc!( + r#" + private + + "# + )), + None::>, + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_config_get_from_config(kamu: KamuCliPuppet) { - let assert = kamu.execute(["config", "list"]).await.success(); - let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); - - pretty_assertions::assert_eq!( - indoc::indoc!( + kamu.assert_success_command_execution( + ["config", "list"], + Some(indoc::indoc!( r#" engine: runtime: podman @@ -238,9 +167,10 @@ pub async fn test_config_get_from_config(kamu: KamuCliPuppet) { maxFileSizeInMb: 42 "# - ), - stdout - ); + )), + None::>, + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_delete_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_delete_command.rs index 8555ff0af..0d6cae2e0 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_delete_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_delete_command.rs @@ -17,34 +17,23 @@ use kamu_cli_puppet::KamuCliPuppet; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_delete_dataset(kamu: KamuCliPuppet) { - { - let assert = kamu.execute(["delete", "player-scores"]).await.failure(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Error: Dataset not found: player-scores"), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_failure_command_execution( + ["delete", "player-scores"], + None, + Some(["Error: Dataset not found: player-scores"]), + ) + .await; kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) .await .success(); - { - let assert = kamu - .execute(["--yes", "delete", "player-scores"]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Deleted 1 dataset(s)"), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_success_command_execution( + ["--yes", "delete", "player-scores"], + None, + Some(["Deleted 1 dataset(s)"]), + ) + .await; let dataset_names = kamu .list_datasets() @@ -96,19 +85,14 @@ pub async fn test_delete_dataset_recursive(kamu: KamuCliPuppet) { ["another-root", "leaderboard", "player-scores"] ); } - { - let assert = kamu - .execute(["--yes", "delete", "player-scores", "--recursive"]) - .await - .success(); - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + kamu.assert_success_command_execution( + ["--yes", "delete", "player-scores", "--recursive"], + None, + Some(["Deleted 2 dataset(s)"]), + ) + .await; - assert!( - stderr.contains("Deleted 2 dataset(s)"), - "Unexpected output:\n{stderr}", - ); - } { let dataset_names = kamu .list_datasets() @@ -158,16 +142,14 @@ pub async fn test_delete_dataset_all(kamu: KamuCliPuppet) { ["another-root", "leaderboard", "player-scores"] ); } - { - let assert = kamu.execute(["--yes", "delete", "--all"]).await.success(); - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + kamu.assert_success_command_execution( + ["--yes", "delete", "--all"], + None, + Some(["Deleted 3 dataset(s)"]), + ) + .await; - assert!( - stderr.contains("Deleted 3 dataset(s)"), - "Unexpected output:\n{stderr}", - ); - } { let dataset_names = kamu .list_datasets() diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_ingest_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_ingest_command.rs index 124a735c7..7f20b585f 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_ingest_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_ingest_command.rs @@ -262,19 +262,12 @@ pub async fn test_ingest_recursive(kamu: KamuCliPuppet) { .await .success(); - { - let assert = kamu - .execute(["tail", "leaderboard", "--output-format", "table"]) - .await - .failure(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Error: Dataset schema is not yet available: leaderboard"), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_failure_command_execution( + ["tail", "leaderboard", "--output-format", "table"], + None, + Some(["Error: Dataset schema is not yet available: leaderboard"]), + ) + .await; // TODO: `kamu ingest`: implement `--recursive` mode // https://github.com/kamu-data/kamu-cli/issues/886 @@ -403,70 +396,31 @@ async fn assert_ingest_data_to_player_scores_from_stdio( ingest_data: T, expected_tail_table: &str, ) where - I: IntoIterator + Clone, + I: IntoIterator + Send + Clone, S: AsRef, - T: Into> + Clone, + T: Into> + Send + Clone, { // Ingest - { - let assert = kamu - .execute_with_input(ingest_cmd.clone(), ingest_data.clone()) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Dataset updated"), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_success_command_execution_with_input( + ingest_cmd.clone(), + ingest_data.clone(), + None, + Some(["Dataset updated"]), + ) + .await; + // Trying to ingest the same data - { - let assert = kamu - .execute_with_input(ingest_cmd, ingest_data) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Dataset up-to-date"), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_success_command_execution_with_input( + ingest_cmd, + ingest_data, + None, + Some(["Dataset up-to-date"]), + ) + .await; + // Assert ingested data - { - let assert = kamu - .execute([ - "sql", - "--engine", - "datafusion", - "--command", - // Without unstable "offset" column. - // For a beautiful output, cut to seconds - indoc::indoc!( - r#" - SELECT op, - system_time, - match_time, - match_id, - player_id, - score - FROM "player-scores" - ORDER BY match_id, score, player_id; - "# - ), - "--output-format", - "table", - ]) - .await - .success(); - - let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); - - pretty_assertions::assert_eq!(expected_tail_table, stdout); - } + kamu.assert_player_scores_dataset_data(expected_tail_table) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_init_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_init_command.rs index 7b2c1e3a2..8f8c48319 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_init_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_init_command.rs @@ -8,6 +8,7 @@ // by the Apache License, Version 2.0. use kamu_cli::{DEFAULT_MULTI_TENANT_SQLITE_DATABASE_NAME, KAMU_WORKSPACE_DIR_NAME}; +use kamu_cli_puppet::extensions::KamuCliPuppetExt; use kamu_cli_puppet::KamuCliPuppet; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -90,24 +91,15 @@ pub async fn test_init_exist_ok_mt(mut kamu: KamuCliPuppet) { pub async fn test_init_in_an_existing_workspace(mut kamu: KamuCliPuppet) { kamu.set_workspace_path_in_tmp_dir(); - { - let assert = kamu.execute(["init"]).await.success(); - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Initialized an empty workspace"), - "Unexpected output:\n{stderr}", - ); - } - { - let assert = kamu.execute(["init"]).await.failure(); - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Error: Directory is already a kamu workspace"), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_success_command_execution(["init"], None, Some(["Initialized an empty workspace"])) + .await; + + kamu.assert_failure_command_execution( + ["init"], + None, + Some(["Error: Directory is already a kamu workspace"]), + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_inspect_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_inspect_command.rs index c849f4a7c..d2c12670e 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_inspect_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_inspect_command.rs @@ -30,25 +30,18 @@ pub async fn test_inspect_lineage(kamu: KamuCliPuppet) { .await .success(); - { - let assert = kamu - .execute(["inspect", "lineage", "--output-format", "shell"]) - .await - .success(); - - let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); - - pretty_assertions::assert_eq!( - stdout, - indoc::indoc!( - r#" - leaderboard: Derivative - └── player-scores: Root - player-scores: Root - "# - ) - ); - } + kamu.assert_success_command_execution( + ["inspect", "lineage", "--output-format", "shell"], + Some(indoc::indoc!( + r#" + leaderboard: Derivative + └── player-scores: Root + player-scores: Root + "# + )), + None::>, + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -91,25 +84,16 @@ pub async fn test_inspect_query(kamu: KamuCliPuppet) { }) .unwrap(); - { - let assert = kamu - .execute(["inspect", "query", "player-scores"]) - .await - .success(); - - let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); - - pretty_assertions::assert_eq!(stdout, ""); - } - { - let assert = kamu - .execute(["inspect", "query", "leaderboard"]) - .await - .success(); - - let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + kamu.assert_success_command_execution( + ["inspect", "query", "player-scores"], + Some(""), + None::>, + ) + .await; - pretty_assertions::assert_eq!( + kamu.assert_success_command_execution( + ["inspect", "query", "leaderboard"], + Some( indoc::formatdoc!( r#" Transform: {leaderboard_transform_block_hash} @@ -134,10 +118,12 @@ pub async fn test_inspect_query(kamu: KamuCliPuppet) { Query: leaderboard select * from leaderboard "# - ), - stdout - ); - } + ) + .as_str(), + ), + None::>, + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -147,19 +133,12 @@ pub async fn test_inspect_schema(kamu: KamuCliPuppet) { .await .success(); - { - let assert = kamu - .execute(["inspect", "schema", "player-scores"]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Warning: Dataset schema is not yet available: player-scores"), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_success_command_execution( + ["inspect", "schema", "player-scores"], + None, + Some(["Warning: Dataset schema is not yet available: player-scores"]), + ) + .await; kamu.execute_with_input( ["add", "--stdin"], @@ -168,25 +147,18 @@ pub async fn test_inspect_schema(kamu: KamuCliPuppet) { .await .success(); - { - let assert = kamu - .execute([ - "inspect", - "schema", - "leaderboard", - "--output-format", - "parquet", - ]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Warning: Dataset schema is not yet available: leaderboard"), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_success_command_execution( + [ + "inspect", + "schema", + "leaderboard", + "--output-format", + "parquet", + ], + None, + Some(["Warning: Dataset schema is not yet available: leaderboard"]), + ) + .await; kamu.execute_with_input( ["ingest", "player-scores", "--stdin"], @@ -195,56 +167,43 @@ pub async fn test_inspect_schema(kamu: KamuCliPuppet) { .await .success(); - { - let assert = kamu - .execute([ - "inspect", - "schema", - "player-scores", - "--output-format", - "parquet", - ]) - .await - .success(); - - let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); - - pretty_assertions::assert_eq!( - indoc::indoc!( - r#" - message arrow_schema { - REQUIRED INT64 offset; - REQUIRED INT32 op; - REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); - OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); - OPTIONAL INT64 match_id; - OPTIONAL BYTE_ARRAY player_id (STRING); - OPTIONAL INT64 score; - } - "# - ), - stdout - ); - } - { - let assert = kamu - .execute([ - "inspect", - "schema", - "leaderboard", - "--output-format", - "parquet", - ]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Warning: Dataset schema is not yet available: leaderboard"), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_success_command_execution( + [ + "inspect", + "schema", + "player-scores", + "--output-format", + "parquet", + ], + Some(indoc::indoc!( + r#" + message arrow_schema { + REQUIRED INT64 offset; + REQUIRED INT32 op; + REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_id; + OPTIONAL BYTE_ARRAY player_id (STRING); + OPTIONAL INT64 score; + } + "# + )), + None::>, + ) + .await; + + kamu.assert_success_command_execution( + [ + "inspect", + "schema", + "leaderboard", + "--output-format", + "parquet", + ], + None, + Some(["Warning: Dataset schema is not yet available: leaderboard"]), + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_login_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_login_command.rs index 79f3c005a..d0f2a80dc 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_login_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_login_command.rs @@ -12,6 +12,7 @@ use kamu_cli_e2e_common::{ KamuApiServerClientExt, DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, }; +use kamu_cli_puppet::extensions::KamuCliPuppetExt; use kamu_cli_puppet::KamuCliPuppet; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -20,90 +21,57 @@ pub async fn test_login_logout_password(kamu_node_api_client: KamuApiServerClien let kamu_node_url = kamu_node_api_client.get_base_url().as_str(); let kamu = KamuCliPuppet::new_workspace_tmp().await; - { - let assert = kamu.execute(["logout", kamu_node_url]).await.success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(format!("Not logged in to {kamu_node_url}").as_str()), - "Unexpected output:\n{stderr}", - ); - } - { - let assert = kamu - .execute(["login", kamu_node_url, "--check"]) - .await - .failure(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(format!("Error: No access token found for: {kamu_node_url}").as_str()), - "Unexpected output:\n{stderr}", - ); - } - - { - let assert = kamu - .execute(["login", "password", "kamu", "kamu", kamu_node_url]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(format!("Login successful: {kamu_node_url}").as_str()), - "Unexpected output:\n{stderr}", - ); - } - { - let assert = kamu - .execute(["login", kamu_node_url, "--check"]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(format!("Access token valid: {kamu_node_url}").as_str()), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_success_command_execution( + ["logout", kamu_node_url], + None, + Some([format!("Not logged in to {kamu_node_url}").as_str()]), + ) + .await; + + kamu.assert_failure_command_execution( + ["login", kamu_node_url, "--check"], + None, + Some([format!("Error: No access token found for: {kamu_node_url}").as_str()]), + ) + .await; + + kamu.assert_success_command_execution( + ["login", "password", "kamu", "kamu", kamu_node_url], + None, + Some([format!("Login successful: {kamu_node_url}").as_str()]), + ) + .await; + + kamu.assert_success_command_execution( + ["login", kamu_node_url, "--check"], + None, + Some([format!("Access token valid: {kamu_node_url}").as_str()]), + ) + .await; kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) .await .success(); // Token validation, via an API call that requires authorization - { - let assert = kamu - .execute([ - "push", - "player-scores", - "--to", - &format!("odf+{kamu_node_url}player-scores"), - ]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("1 dataset(s) pushed"), - "Unexpected output:\n{stderr}", - ); - } - { - let assert = kamu.execute(["logout", kamu_node_url]).await.success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(format!("Logged out of {kamu_node_url}").as_str()), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_success_command_execution( + [ + "push", + "player-scores", + "--to", + &format!("odf+{kamu_node_url}player-scores"), + ], + None, + Some(["1 dataset(s) pushed"]), + ) + .await; + + kamu.assert_success_command_execution( + ["logout", kamu_node_url], + None, + Some([format!("Logged out of {kamu_node_url}").as_str()]), + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -112,74 +80,47 @@ pub async fn test_login_logout_oauth(kamu_node_api_client: KamuApiServerClient) let kamu_node_url = kamu_node_api_client.get_base_url().as_str(); let kamu = KamuCliPuppet::new_workspace_tmp().await; - { - let assert = kamu.execute(["logout", kamu_node_url]).await.success(); + kamu.assert_success_command_execution( + ["logout", kamu_node_url], + None, + Some([format!("Not logged in to {kamu_node_url}").as_str()]), + ) + .await; - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(format!("Not logged in to {kamu_node_url}").as_str()), - "Unexpected output:\n{stderr}", - ); - } - { - let assert = kamu - .execute(["login", kamu_node_url, "--check"]) - .await - .failure(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(format!("Error: No access token found for: {kamu_node_url}").as_str()), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_failure_command_execution( + ["login", kamu_node_url, "--check"], + None, + Some([format!("Error: No access token found for: {kamu_node_url}").as_str()]), + ) + .await; let oauth_token = kamu_node_api_client.login_as_e2e_user().await; - { - let assert = kamu - .execute(["login", "oauth", "github", &oauth_token, kamu_node_url]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(format!("Login successful: {kamu_node_url}").as_str()), - "Unexpected output:\n{stderr}", - ); - } - { - let assert = kamu - .execute(["login", kamu_node_url, "--check"]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(format!("Access token valid: {kamu_node_url}").as_str()), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_success_command_execution( + ["login", "oauth", "github", &oauth_token, kamu_node_url], + None, + Some([format!("Login successful: {kamu_node_url}").as_str()]), + ) + .await; + + kamu.assert_success_command_execution( + ["login", kamu_node_url, "--check"], + None, + Some([format!("Access token valid: {kamu_node_url}").as_str()]), + ) + .await; // Token validation, via an API call that requires authorization kamu_node_api_client .create_player_scores_dataset(&oauth_token) .await; - { - let assert = kamu.execute(["logout", kamu_node_url]).await.success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(format!("Logged out of {kamu_node_url}").as_str()), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_success_command_execution( + ["logout", kamu_node_url], + None, + Some([format!("Logged out of {kamu_node_url}").as_str()]), + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_new_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_new_command.rs index 7d9db6442..82bd99080 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_new_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_new_command.rs @@ -7,27 +7,23 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +use kamu_cli_puppet::extensions::KamuCliPuppetExt; use kamu_cli_puppet::KamuCliPuppet; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_new_root(kamu: KamuCliPuppet) { - let assert = kamu - .execute(["new", "--root", "test-dataset"]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(indoc::indoc!( + kamu.assert_success_command_execution( + ["new", "--root", "test-dataset"], + None, + Some([indoc::indoc!( r#" Written new manifest template to: test-dataset.yaml Follow directions in the file's comments and use `kamu add test-dataset.yaml` when ready. "# - )), - "Unexpected output:\n{stderr}", - ); + )]), + ) + .await; // TODO: After solving this issue, add `kamu add` calls and populate with // data @@ -40,22 +36,17 @@ pub async fn test_new_root(kamu: KamuCliPuppet) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_new_derivative(kamu: KamuCliPuppet) { - let assert = kamu - .execute(["new", "--derivative", "test-dataset"]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(indoc::indoc!( + kamu.assert_success_command_execution( + ["new", "--derivative", "test-dataset"], + None, + Some([indoc::indoc!( r#" Written new manifest template to: test-dataset.yaml Follow directions in the file's comments and use `kamu add test-dataset.yaml` when ready. "# - )), - "Unexpected output:\n{stderr}", - ); + )]), + ) + .await; // TODO: After solving this issue, add `kamu add` calls and populate with // data diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_rename_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_rename_command.rs index 987a6aba8..e05386b2c 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_rename_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_rename_command.rs @@ -14,37 +14,23 @@ use kamu_cli_puppet::KamuCliPuppet; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_rename_dataset(kamu: KamuCliPuppet) { - { - let assert = kamu - .execute(["rename", "player-scores", "top-player-scores"]) - .await - .failure(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Error: Dataset not found: player-scores"), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_failure_command_execution( + ["rename", "player-scores", "top-player-scores"], + None, + Some(["Error: Dataset not found: player-scores"]), + ) + .await; kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) .await .success(); - { - let assert = kamu - .execute(["rename", "player-scores", "top-player-scores"]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Dataset renamed"), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_success_command_execution( + ["rename", "player-scores", "top-player-scores"], + None, + Some(["Dataset renamed"]), + ) + .await; let dataset_names = kamu .list_datasets() diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_reset_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_reset_command.rs index de1f02d0a..9ae74eeab 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_reset_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_reset_command.rs @@ -58,24 +58,17 @@ pub async fn test_reset(kamu: KamuCliPuppet) { .as_multibase() .to_stack_string(); - { - let assert = kamu - .execute([ - "--yes", - "reset", - "player-scores", - set_vocab_block_hash.as_str(), - ]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Dataset was reset"), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_success_command_execution( + [ + "--yes", + "reset", + "player-scores", + set_vocab_block_hash.as_str(), + ], + None, + Some(["Dataset was reset"]), + ) + .await; let block_records_after_resetting = kamu .list_blocks(&DatasetName::new_unchecked("player-scores")) diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_search_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_search_command.rs index 8be9d1df4..0f78b7211 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_search_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_search_command.rs @@ -13,9 +13,9 @@ use kamu_cli_e2e_common::{ RequestBody, DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, }; +use kamu_cli_puppet::extensions::KamuCliPuppetExt; use kamu_cli_puppet::KamuCliPuppet; use opendatafabric::*; -use reqwest::Url; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -24,10 +24,9 @@ pub async fn test_search_multi_user(kamu_node_api_client: KamuApiServerClient) { add_repo_to_workspace(&kamu_node_api_client, &kamu, "kamu-node").await; - assert_search( - &kamu, + kamu.assert_success_command_execution( ["search", "player", "--output-format", "table"], - indoc::indoc!( + Some(indoc::indoc!( r#" ┌───────┬──────┬─────────────┬────────┬─────────┬──────┐ │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ @@ -35,7 +34,8 @@ pub async fn test_search_multi_user(kamu_node_api_client: KamuApiServerClient) { │ │ │ │ │ │ │ └───────┴──────┴─────────────┴────────┴─────────┴──────┘ "# - ), + )), + None::>, ) .await; @@ -45,10 +45,9 @@ pub async fn test_search_multi_user(kamu_node_api_client: KamuApiServerClient) { .create_player_scores_dataset(&e2e_user_token) .await; - assert_search( - &kamu, + kamu.assert_success_command_execution( ["search", "player", "--output-format", "table"], - indoc::indoc!( + Some(indoc::indoc!( r#" ┌──────────────────────────────────┬──────┬─────────────┬────────┬─────────┬──────┐ │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ @@ -56,7 +55,8 @@ pub async fn test_search_multi_user(kamu_node_api_client: KamuApiServerClient) { │ kamu-node/e2e-user/player-scores │ Root │ - │ 3 │ - │ - │ └──────────────────────────────────┴──────┴─────────────┴────────┴─────────┴──────┘ "# - ), + )), + None::>, ) .await; @@ -73,10 +73,9 @@ pub async fn test_search_multi_user(kamu_node_api_client: KamuApiServerClient) { ) .await; - assert_search( - &kamu, + kamu.assert_success_command_execution( ["search", "player", "--output-format", "table"], - indoc::indoc!( + Some(indoc::indoc!( r#" ┌──────────────────────────────────┬──────┬─────────────┬────────┬─────────┬──────────┐ │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ @@ -84,7 +83,8 @@ pub async fn test_search_multi_user(kamu_node_api_client: KamuApiServerClient) { │ kamu-node/e2e-user/player-scores │ Root │ - │ 5 │ 2 │ 1.63 KiB │ └──────────────────────────────────┴──────┴─────────────┴────────┴─────────┴──────────┘ "# - ), + )), + None::>, ) .await; @@ -137,10 +137,9 @@ pub async fn test_search_multi_user(kamu_node_api_client: KamuApiServerClient) { ) .await; - assert_search( - &kamu, + kamu.assert_success_command_execution( ["search", "player", "--output-format", "table"], - indoc::indoc!( + Some(indoc::indoc!( r#" ┌───────────────────────────────────────┬────────────┬─────────────┬────────┬─────────┬──────────┐ │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ @@ -149,7 +148,8 @@ pub async fn test_search_multi_user(kamu_node_api_client: KamuApiServerClient) { │ kamu-node/e2e-user/player-scores │ Root │ - │ 5 │ 2 │ 1.63 KiB │ └───────────────────────────────────────┴────────────┴─────────────┴────────┴─────────┴──────────┘ "# - ), + )), + None::>, ) .await; @@ -159,10 +159,9 @@ pub async fn test_search_multi_user(kamu_node_api_client: KamuApiServerClient) { .create_player_scores_dataset(&kamu_token) .await; - assert_search( - &kamu, + kamu.assert_success_command_execution( ["search", "player", "--output-format", "table"], - indoc::indoc!( + Some(indoc::indoc!( r#" ┌───────────────────────────────────────┬────────────┬─────────────┬────────┬─────────┬──────────┐ │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ @@ -172,7 +171,8 @@ pub async fn test_search_multi_user(kamu_node_api_client: KamuApiServerClient) { │ kamu-node/kamu/player-scores │ Root │ - │ 3 │ - │ - │ └───────────────────────────────────────┴────────────┴─────────────┴────────┴─────────┴──────────┘ "# - ), + )), + None::>, ) .await; } @@ -194,10 +194,9 @@ pub async fn test_search_by_name(kamu_node_api_client: KamuApiServerClient) { .create_leaderboard(&e2e_user_token) .await; - assert_search( - &kamu, + kamu.assert_success_command_execution( ["search", "player", "--output-format", "table"], - indoc::indoc!( + Some(indoc::indoc!( r#" ┌──────────────────────────────────┬──────┬─────────────┬────────┬─────────┬──────┐ │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ @@ -205,14 +204,14 @@ pub async fn test_search_by_name(kamu_node_api_client: KamuApiServerClient) { │ kamu-node/e2e-user/player-scores │ Root │ - │ 3 │ - │ - │ └──────────────────────────────────┴──────┴─────────────┴────────┴─────────┴──────┘ "# - ), + )), + None::>, ) .await; - assert_search( - &kamu, + kamu.assert_success_command_execution( ["search", "scores", "--output-format", "table"], - indoc::indoc!( + Some(indoc::indoc!( r#" ┌──────────────────────────────────┬──────┬─────────────┬────────┬─────────┬──────┐ │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ @@ -220,14 +219,14 @@ pub async fn test_search_by_name(kamu_node_api_client: KamuApiServerClient) { │ kamu-node/e2e-user/player-scores │ Root │ - │ 3 │ - │ - │ └──────────────────────────────────┴──────┴─────────────┴────────┴─────────┴──────┘ "# - ), + )), + None::>, ) .await; - assert_search( - &kamu, + kamu.assert_success_command_execution( ["search", "not-relevant-query", "--output-format", "table"], - indoc::indoc!( + Some(indoc::indoc!( r#" ┌───────┬──────┬─────────────┬────────┬─────────┬──────┐ │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ @@ -235,14 +234,14 @@ pub async fn test_search_by_name(kamu_node_api_client: KamuApiServerClient) { │ │ │ │ │ │ │ └───────┴──────┴─────────────┴────────┴─────────┴──────┘ "# - ), + )), + None::>, ) .await; - assert_search( - &kamu, + kamu.assert_success_command_execution( ["search", "lead", "--output-format", "table"], - indoc::indoc!( + Some(indoc::indoc!( r#" ┌────────────────────────────────┬────────────┬─────────────┬────────┬─────────┬──────┐ │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ @@ -250,7 +249,8 @@ pub async fn test_search_by_name(kamu_node_api_client: KamuApiServerClient) { │ kamu-node/e2e-user/leaderboard │ Derivative │ - │ 3 │ - │ - │ └────────────────────────────────┴────────────┴─────────────┴────────┴─────────┴──────┘ "# - ), + )), + None::>, ) .await; } @@ -274,10 +274,9 @@ pub async fn test_search_by_repo(kamu_node_api_client: KamuApiServerClient) { .create_leaderboard(&e2e_user_token) .await; - assert_search( - &kamu, + kamu.assert_success_command_execution( ["search", "player", "--output-format", "table"], - indoc::indoc!( + Some(indoc::indoc!( r#" ┌──────────────────────────────────────┬──────┬─────────────┬────────┬─────────┬──────┐ │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ @@ -286,12 +285,12 @@ pub async fn test_search_by_repo(kamu_node_api_client: KamuApiServerClient) { │ kamu-node/e2e-user/player-scores │ Root │ - │ 3 │ - │ - │ └──────────────────────────────────────┴──────┴─────────────┴────────┴─────────┴──────┘ "# - ), + )), + None::>, ) .await; - assert_search( - &kamu, + kamu.assert_success_command_execution( [ "search", "player", @@ -300,7 +299,7 @@ pub async fn test_search_by_repo(kamu_node_api_client: KamuApiServerClient) { "--output-format", "table", ], - indoc::indoc!( + Some(indoc::indoc!( r#" ┌──────────────────────────────────────┬──────┬─────────────┬────────┬─────────┬──────┐ │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ @@ -308,12 +307,12 @@ pub async fn test_search_by_repo(kamu_node_api_client: KamuApiServerClient) { │ acme-org-node/e2e-user/player-scores │ Root │ - │ 3 │ - │ - │ └──────────────────────────────────────┴──────┴─────────────┴────────┴─────────┴──────┘ "# - ), + )), + None::>, ) .await; - assert_search( - &kamu, + kamu.assert_success_command_execution( [ "search", "player", @@ -322,7 +321,7 @@ pub async fn test_search_by_repo(kamu_node_api_client: KamuApiServerClient) { "--output-format", "table", ], - indoc::indoc!( + Some(indoc::indoc!( r#" ┌──────────────────────────────────┬──────┬─────────────┬────────┬─────────┬──────┐ │ Alias │ Kind │ Description │ Blocks │ Records │ Size │ @@ -330,7 +329,8 @@ pub async fn test_search_by_repo(kamu_node_api_client: KamuApiServerClient) { │ kamu-node/e2e-user/player-scores │ Root │ - │ 3 │ - │ - │ └──────────────────────────────────┴──────┴─────────────┴────────┴─────────┴──────┘ "# - ), + )), + None::>, ) .await; } @@ -344,39 +344,17 @@ async fn add_repo_to_workspace( kamu: &KamuCliPuppet, repo_name: &str, ) { - let http_repo = { - let mut url = Url::parse("odf+http://host").unwrap(); - let base_url = kamu_node_api_client.get_base_url(); - url.set_host(base_url.host_str()).unwrap(); - url.set_port(base_url.port()).unwrap(); - url - }; - - let assert = kamu - .execute(["repo", "add", repo_name, http_repo.as_str()]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(format!("Added: {repo_name}").as_str()), - "Unexpected output:\n{stderr}", - ); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -async fn assert_search(kamu: &KamuCliPuppet, search_cmd: I, expected_table_output: &str) -where - I: IntoIterator, - S: AsRef, -{ - let assert = kamu.execute(search_cmd).await.success(); - - let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); - - pretty_assertions::assert_eq!(stdout, expected_table_output); + kamu.assert_success_command_execution( + [ + "repo", + "add", + repo_name, + kamu_node_api_client.get_node_url().as_str(), + ], + None, + Some([format!("Added: {repo_name}").as_str()]), + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_sql_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_sql_command.rs index bca4c7928..5718a97f0 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_sql_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_sql_command.rs @@ -7,11 +7,11 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use indoc::indoc; use kamu_cli_e2e_common::{ DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, }; +use kamu_cli_puppet::extensions::KamuCliPuppetExt; use kamu_cli_puppet::KamuCliPuppet; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -26,7 +26,7 @@ pub async fn test_datafusion_cli(kamu: KamuCliPuppet) { assert!( stdout.contains( - indoc!( + indoc::indoc!( r#" +----------+ | Int64(1) | @@ -50,80 +50,62 @@ pub async fn test_datafusion_cli_not_launched_in_root_ws(kamu: KamuCliPuppet) { // so there is no problem that the process working directory is one of the // subdirectories (kamu-cli/src/e2e/app/cli/inmem) - { - let assert = kamu.execute(["list"]).await.failure(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Error: Directory is not a kamu workspace"), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_failure_command_execution( + ["list"], + None, + Some(["Error: Directory is not a kamu workspace"]), + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_sql_command(kamu: KamuCliPuppet) { - { - let assert = kamu - .execute([ - "sql", - "--command", - "SELECT 42 as answer;", - "--output-format", - "table", - ]) - .await - .success(); - - let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); - - pretty_assertions::assert_eq!( - indoc::indoc!( - r#" - ┌────────┐ - │ answer │ - ├────────┤ - │ 42 │ - └────────┘ - "# - ), - stdout - ); - } + kamu.assert_success_command_execution( + [ + "sql", + "--command", + "SELECT 42 as answer;", + "--output-format", + "table", + ], + Some(indoc::indoc!( + r#" + ┌────────┐ + │ answer │ + ├────────┤ + │ 42 │ + └────────┘ + "# + )), + None::>, + ) + .await; kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) .await .success(); - { - let assert = kamu - .execute([ - "sql", - "--command", - "SELECT * FROM \"player-scores\";", - "--output-format", - "table", - ]) - .await - .success(); - - let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); - - pretty_assertions::assert_eq!( - indoc::indoc!( - r#" - ┌┐ - ││ - ├┤ - ││ - └┘ - "# - ), - stdout - ); - } + kamu.assert_success_command_execution( + [ + "sql", + "--command", + "SELECT * FROM \"player-scores\";", + "--output-format", + "table", + ], + Some(indoc::indoc!( + r#" + ┌┐ + ││ + ├┤ + ││ + └┘ + "# + )), + None::>, + ) + .await; kamu.execute_with_input( ["ingest", "player-scores", "--stdin"], @@ -132,34 +114,27 @@ pub async fn test_sql_command(kamu: KamuCliPuppet) { .await .success(); - { - let assert = kamu - .execute([ - "sql", - "--command", - "SELECT * FROM \"player-scores\" ORDER BY offset;", - "--output-format", - "table", - ]) - .await - .success(); - - let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); - - pretty_assertions::assert_eq!( - indoc::indoc!( - r#" - ┌────────┬────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ - │ offset │ op │ system_time │ match_time │ match_id │ player_id │ score │ - ├────────┼────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ - │ 0 │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ - │ 1 │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ - └────────┴────┴──────────────────────┴──────────────────────┴──────────┴───────────┴───────┘ - "# - ), - stdout - ); - } + kamu.assert_success_command_execution( + [ + "sql", + "--command", + "SELECT * FROM \"player-scores\" ORDER BY offset;", + "--output-format", + "table", + ], + Some(indoc::indoc!( + r#" + ┌────────┬────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ + │ offset │ op │ system_time │ match_time │ match_id │ player_id │ score │ + ├────────┼────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ + │ 0 │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ + │ 1 │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ + └────────┴────┴──────────────────────┴──────────────────────┴──────────┴───────────┴───────┘ + "# + )), + None::>, + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_system_api_server_gql_query.rs b/src/e2e/app/cli/repo-tests/src/commands/test_system_api_server_gql_query.rs index 7b9a77c28..5556d7f64 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_system_api_server_gql_query.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_system_api_server_gql_query.rs @@ -7,32 +7,29 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +use kamu_cli_puppet::extensions::KamuCliPuppetExt; use kamu_cli_puppet::KamuCliPuppet; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_gql_query_api_version(kamu: KamuCliPuppet) { - let assert = kamu - .execute([ + kamu.assert_success_command_execution( + [ "system", "api-server", "gql-query", "{apiVersion}".escape_default().to_string().as_str(), - ]) - .await - .success(); - let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); - - assert_eq!( - stdout, - indoc::indoc!( + ], + Some(indoc::indoc!( r#" { "apiVersion": "0.1" } "# - ) - ); + )), + None::>, + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_system_gc_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_system_gc_command.rs index 862f7c92f..3c82b5b6a 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_system_gc_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_system_gc_command.rs @@ -7,19 +7,18 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +use kamu_cli_puppet::extensions::KamuCliPuppetExt; use kamu_cli_puppet::KamuCliPuppet; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_gc(kamu: KamuCliPuppet) { - let assert = kamu.execute(["system", "gc"]).await.success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Cleaning cache...") && stderr.contains("Workspace is already clean"), - "Unexpected output:\n{stderr}", - ); + kamu.assert_success_command_execution( + ["system", "gc"], + None, + Some(["Cleaning cache...", "Workspace is already clean"]), + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_tail_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_tail_command.rs index cd486052b..1f84696db 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_tail_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_tail_command.rs @@ -11,6 +11,7 @@ use kamu_cli_e2e_common::{ DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, }; +use kamu_cli_puppet::extensions::KamuCliPuppetExt; use kamu_cli_puppet::KamuCliPuppet; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -20,19 +21,12 @@ pub async fn test_tail(kamu: KamuCliPuppet) { .await .success(); - { - let assert = kamu - .execute(["tail", "player-scores", "--output-format", "table"]) - .await - .failure(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains("Error: Dataset schema is not yet available: player-scores"), - "Unexpected output:\n{stderr}", - ); - } + kamu.assert_failure_command_execution( + ["tail", "player-scores", "--output-format", "table"], + None, + Some(["Error: Dataset schema is not yet available: player-scores"]), + ) + .await; kamu.execute_with_input( ["ingest", "player-scores", "--stdin"], @@ -41,28 +35,21 @@ pub async fn test_tail(kamu: KamuCliPuppet) { .await .success(); - { - let assert = kamu - .execute(["tail", "player-scores", "--output-format", "table"]) - .await - .success(); - - let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); - - pretty_assertions::assert_eq!( - indoc::indoc!( - r#" - ┌────────┬────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ - │ offset │ op │ system_time │ match_time │ match_id │ player_id │ score │ - ├────────┼────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ - │ 0 │ +A │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ - │ 1 │ +A │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ - └────────┴────┴──────────────────────┴──────────────────────┴──────────┴───────────┴───────┘ - "# - ), - stdout - ); - } + kamu.assert_success_command_execution( + ["tail", "player-scores", "--output-format", "table"], + Some(indoc::indoc!( + r#" + ┌────────┬────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ + │ offset │ op │ system_time │ match_time │ match_id │ player_id │ score │ + ├────────┼────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ + │ 0 │ +A │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ + │ 1 │ +A │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ + └────────┴────┴──────────────────────┴──────────────────────┴──────────┴───────────┴───────┘ + "# + )), + None::>, + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_verify_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_verify_command.rs index 088c6e1b2..eaac2950e 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_verify_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_verify_command.rs @@ -9,6 +9,7 @@ use kamu_cli_e2e_common::{ DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, }; use kamu_cli_puppet::extensions::KamuCliPuppetExt; @@ -24,29 +25,18 @@ pub async fn test_verify_regular_dataset(kamu: KamuCliPuppet) { .await .success(); - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90} - "#, - ); - - kamu.ingest_data(&dataset_name, data).await; - - let assert = kamu - .execute(["verify", dataset_name.as_str()]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; - assert!( - stderr.contains(indoc::indoc!( - r#" - 1 dataset(s) are valid - "# - )), - "Unexpected output:\n{stderr}", - ); + kamu.assert_success_command_execution( + ["verify", dataset_name.as_str()], + None, + Some(["1 dataset(s) are valid"]), + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -66,51 +56,31 @@ pub async fn test_verify_recursive(kamu: KamuCliPuppet) { .await .success(); - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90} - "#, - ); - - kamu.ingest_data(&dataset_name, data).await; + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; kamu.execute(["pull", dataset_derivative_name.as_str()]) .await .success(); // Call verify without recursive flag - let assert = kamu - .execute(["verify", dataset_derivative_name.as_str()]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(indoc::indoc!( - r#" - 1 dataset(s) are valid - "# - )), - "Unexpected output:\n{stderr}", - ); + kamu.assert_success_command_execution( + ["verify", dataset_derivative_name.as_str()], + None, + Some(["1 dataset(s) are valid"]), + ) + .await; // Call verify wit recursive flag - let assert = kamu - .execute(["verify", dataset_derivative_name.as_str(), "--recursive"]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(indoc::indoc!( - r#" - 2 dataset(s) are valid - "# - )), - "Unexpected output:\n{stderr}", - ); + kamu.assert_success_command_execution( + ["verify", dataset_derivative_name.as_str(), "--recursive"], + None, + Some(["2 dataset(s) are valid"]), + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -122,29 +92,18 @@ pub async fn test_verify_integrity(kamu: KamuCliPuppet) { .await .success(); - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90} - "#, - ); - - kamu.ingest_data(&dataset_name, data).await; - - let assert = kamu - .execute(["verify", dataset_name.as_str(), "--integrity"]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; - assert!( - stderr.contains(indoc::indoc!( - r#" - 1 dataset(s) are valid - "# - )), - "Unexpected output:\n{stderr}", - ); + kamu.assert_success_command_execution( + ["verify", dataset_name.as_str(), "--integrity"], + None, + Some(["1 dataset(s) are valid"]), + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/test_rest_api.rs b/src/e2e/app/cli/repo-tests/src/test_rest_api.rs index 2cbf04a80..d023596f1 100644 --- a/src/e2e/app/cli/repo-tests/src/test_rest_api.rs +++ b/src/e2e/app/cli/repo-tests/src/test_rest_api.rs @@ -13,6 +13,7 @@ use kamu_cli_e2e_common::{ KamuApiServerClient, KamuApiServerClientExt, RequestBody, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, }; use reqwest::{Method, StatusCode}; @@ -46,13 +47,7 @@ pub async fn test_rest_api_request_dataset_tail(kamu_api_server_client: KamuApiS Method::POST, "player-scores/ingest", Some(RequestBody::NdJson( - indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Bob", "score": 80} - "#, - ) - .into(), + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1.into(), )), StatusCode::OK, None, diff --git a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs index 34b036eb6..c8d47cab9 100644 --- a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs @@ -22,23 +22,21 @@ use kamu_cli_e2e_common::{ }; use kamu_cli_puppet::extensions::{KamuCliPuppetExt, RepoAlias}; use kamu_cli_puppet::KamuCliPuppet; -use opendatafabric::{AccountName, DatasetName}; -use reqwest::Url; +use opendatafabric::{AccountName, DatasetAlias, DatasetName}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_smart_push_pull_sequence(kamu_api_server_client: KamuApiServerClient) { - let dataset_name = DatasetName::new_unchecked("player-scores"); + let dataset_alias = DatasetAlias::new( + Some(AccountName::new_unchecked("e2e-user")), + DatasetName::new_unchecked("player-scores"), + ); + let kamu_api_server_dataset_endpoint = + kamu_api_server_client.get_dataset_endpoint(&dataset_alias); // 1. Grub a token let token = kamu_api_server_client.login_as_e2e_user().await; - let kamu_api_server_dataset_endpoint = get_dataset_endpoint( - kamu_api_server_client.get_base_url(), - &dataset_name, - E2E_USER_ACCOUNT_NAME_STR, - ); - // 2. Pushing the dataset to the API server { let kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; @@ -55,7 +53,7 @@ pub async fn test_smart_push_pull_sequence(kamu_api_server_client: KamuApiServer { kamu_in_push_workspace .ingest_data( - &dataset_name, + &dataset_alias.dataset_name, DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, ) .await; @@ -73,66 +71,64 @@ pub async fn test_smart_push_pull_sequence(kamu_api_server_client: KamuApiServer .success(); // 2.3. Push the dataset to the API server - run_and_assert_command_success( - &kamu_in_push_workspace, - vec![ - "push", - dataset_name.as_str(), - "--to", - kamu_api_server_dataset_endpoint.as_str(), - ], - "1 dataset(s) pushed", - ) - .await; + kamu_in_push_workspace + .assert_success_command_execution( + [ + "push", + dataset_alias.dataset_name.as_str(), + "--to", + kamu_api_server_dataset_endpoint.as_str(), + ], + None, + Some(["1 dataset(s) pushed"]), + ) + .await; } // 3. Pulling the dataset from the API server { let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; - run_and_assert_command_success( - &kamu_in_pull_workspace, - vec!["pull", kamu_api_server_dataset_endpoint.as_str()], - "1 dataset(s) updated", - ) - .await; + kamu_in_pull_workspace + .assert_success_command_execution( + ["pull", kamu_api_server_dataset_endpoint.as_str()], + None, + Some(["1 dataset(s) updated"]), + ) + .await; } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_smart_force_push_pull(kamu_api_server_client: KamuApiServerClient) { - let dataset_name = DatasetName::new_unchecked("player-scores"); + let dataset_alias = DatasetAlias::new( + Some(AccountName::new_unchecked("e2e-user")), + DatasetName::new_unchecked("player-scores"), + ); + let kamu_api_server_dataset_endpoint = + kamu_api_server_client.get_dataset_endpoint(&dataset_alias); + // 1. Grub a token let token = kamu_api_server_client.login_as_e2e_user().await; - let kamu_api_server_dataset_endpoint = get_dataset_endpoint( - kamu_api_server_client.get_base_url(), - &dataset_name, - E2E_USER_ACCOUNT_NAME_STR, - ); - // 2. Pushing the dataset to the API server { let kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; // 2.1. Add the dataset - { - kamu_in_push_workspace - .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) - .await - .success(); - } + kamu_in_push_workspace + .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); // 2.1. Ingest data to the dataset - { - kamu_in_push_workspace - .ingest_data( - &dataset_name, - DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, - ) - .await; - } + kamu_in_push_workspace + .ingest_data( + &dataset_alias.dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; // 2.2. Login to the API server kamu_in_push_workspace @@ -146,17 +142,18 @@ pub async fn test_smart_force_push_pull(kamu_api_server_client: KamuApiServerCli .success(); // Initial dataset push - run_and_assert_command_success( - &kamu_in_push_workspace, - vec![ - "push", - dataset_name.as_str(), - "--to", - kamu_api_server_dataset_endpoint.as_str(), - ], - "1 dataset(s) pushed", - ) - .await; + kamu_in_push_workspace + .assert_success_command_execution( + [ + "push", + dataset_alias.dataset_name.as_str(), + "--to", + kamu_api_server_dataset_endpoint.as_str(), + ], + None, + Some(["1 dataset(s) pushed"]), + ) + .await; // Hard compact dataset kamu_in_push_workspace @@ -164,7 +161,7 @@ pub async fn test_smart_force_push_pull(kamu_api_server_client: KamuApiServerCli "--yes", "system", "compact", - dataset_name.as_str(), + dataset_alias.dataset_name.as_str(), "--hard", "--keep-metadata-only", ]) @@ -172,31 +169,33 @@ pub async fn test_smart_force_push_pull(kamu_api_server_client: KamuApiServerCli .success(); // Should fail without force flag - run_and_assert_command_failure( - &kamu_in_push_workspace, - vec![ - "push", - dataset_name.as_str(), - "--to", - kamu_api_server_dataset_endpoint.as_str(), - ], - "Failed to push 1 dataset(s)", - ) - .await; + kamu_in_push_workspace + .assert_failure_command_execution( + [ + "push", + dataset_alias.dataset_name.as_str(), + "--to", + kamu_api_server_dataset_endpoint.as_str(), + ], + None, + Some(["Failed to push 1 dataset(s)"]), + ) + .await; // Should successfully push with force flag - run_and_assert_command_success( - &kamu_in_push_workspace, - vec![ - "push", - dataset_name.as_str(), - "--to", - kamu_api_server_dataset_endpoint.as_str(), - "--force", - ], - "1 dataset(s) pushed", - ) - .await; + kamu_in_push_workspace + .assert_success_command_execution( + [ + "push", + dataset_alias.dataset_name.as_str(), + "--to", + kamu_api_server_dataset_endpoint.as_str(), + "--force", + ], + None, + Some(["1 dataset(s) pushed"]), + ) + .await; } // 3. Pulling the dataset from the API server @@ -204,78 +203,77 @@ pub async fn test_smart_force_push_pull(kamu_api_server_client: KamuApiServerCli let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; // Call with no-alias flag to avoid remote ingest checking in next step - run_and_assert_command_success( - &kamu_in_pull_workspace, - vec![ - "pull", - kamu_api_server_dataset_endpoint.as_str(), - "--no-alias", - ], - "1 dataset(s) updated", - ) - .await; + kamu_in_pull_workspace + .assert_success_command_execution( + [ + "pull", + kamu_api_server_dataset_endpoint.as_str(), + "--no-alias", + ], + None, + Some(["1 dataset(s) updated"]), + ) + .await; // Ingest data in pulled dataset kamu_in_pull_workspace .ingest_data( - &dataset_name, + &dataset_alias.dataset_name, DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, ) .await; // Should fail without force flag - run_and_assert_command_failure( - &kamu_in_pull_workspace, - vec!["pull", kamu_api_server_dataset_endpoint.as_str()], - "Failed to update 1 dataset(s)", - ) - .await; + kamu_in_pull_workspace + .assert_failure_command_execution( + ["pull", kamu_api_server_dataset_endpoint.as_str()], + None, + Some(["Failed to update 1 dataset(s)"]), + ) + .await; // Should successfully pull with force flag - run_and_assert_command_success( - &kamu_in_pull_workspace, - vec!["pull", kamu_api_server_dataset_endpoint.as_str(), "--force"], - "1 dataset(s) updated", - ) - .await; + kamu_in_pull_workspace + .assert_success_command_execution( + ["pull", kamu_api_server_dataset_endpoint.as_str(), "--force"], + None, + Some(["1 dataset(s) updated"]), + ) + .await; } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_smart_push_pull_add_alias(kamu_api_server_client: KamuApiServerClient) { - let dataset_name = DatasetName::new_unchecked("player-scores"); + let dataset_alias = DatasetAlias::new( + Some(AccountName::new_unchecked("e2e-user")), + DatasetName::new_unchecked("player-scores"), + ); + let kamu_api_server_dataset_endpoint = + kamu_api_server_client.get_dataset_endpoint(&dataset_alias); + // 1. Grub a token let token = kamu_api_server_client.login_as_e2e_user().await; - let kamu_api_server_dataset_endpoint = get_dataset_endpoint( - kamu_api_server_client.get_base_url(), - &dataset_name, - E2E_USER_ACCOUNT_NAME_STR, - ); - // 2. Push command { let kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; // Add the dataset - { - kamu_in_push_workspace - .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) - .await - .success(); - } + kamu_in_push_workspace + .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); // Ingest data to the dataset - { - kamu_in_push_workspace - .ingest_data( - &dataset_name, - DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, - ) - .await; - } + kamu_in_push_workspace + .ingest_data( + &dataset_alias.dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; // Login to the API server kamu_in_push_workspace @@ -289,45 +287,47 @@ pub async fn test_smart_push_pull_add_alias(kamu_api_server_client: KamuApiServe .success(); // Dataset push without storing alias - run_and_assert_command_success( - &kamu_in_push_workspace, - vec![ - "push", - dataset_name.as_str(), - "--to", - kamu_api_server_dataset_endpoint.as_str(), - "--no-alias", - ], - "1 dataset(s) pushed", - ) - .await; + kamu_in_push_workspace + .assert_success_command_execution( + [ + "push", + dataset_alias.dataset_name.as_str(), + "--to", + kamu_api_server_dataset_endpoint.as_str(), + "--no-alias", + ], + None, + Some(["1 dataset(s) pushed"]), + ) + .await; // Check alias should be empty let aliases = kamu_in_push_workspace - .get_list_of_repo_aliases(&opendatafabric::DatasetRef::from(dataset_name.clone())) + .get_list_of_repo_aliases(&dataset_alias.dataset_name.clone().into()) .await; assert!(aliases.is_empty()); // Dataset push with storing alias - run_and_assert_command_success( - &kamu_in_push_workspace, - vec![ - "push", - dataset_name.as_str(), - "--to", - kamu_api_server_dataset_endpoint.as_str(), - ], - "1 dataset(s) up-to-date", - ) - .await; + kamu_in_push_workspace + .assert_success_command_execution( + [ + "push", + dataset_alias.dataset_name.as_str(), + "--to", + kamu_api_server_dataset_endpoint.as_str(), + ], + None, + Some(["1 dataset(s) up-to-date"]), + ) + .await; let aliases = kamu_in_push_workspace - .get_list_of_repo_aliases(&opendatafabric::DatasetRef::from(dataset_name.clone())) + .get_list_of_repo_aliases(&dataset_alias.dataset_name.clone().into()) .await; let expected_aliases = vec![RepoAlias { - dataset: dataset_name.clone(), + dataset: dataset_alias.dataset_name.clone(), kind: "Push".to_string(), - alias: kamu_api_server_dataset_endpoint.clone(), + alias: kamu_api_server_dataset_endpoint.to_string(), }]; pretty_assertions::assert_eq!(aliases, expected_aliases); } @@ -337,44 +337,46 @@ pub async fn test_smart_push_pull_add_alias(kamu_api_server_client: KamuApiServe let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; // Dataset pull without storing alias - run_and_assert_command_success( - &kamu_in_pull_workspace, - vec![ - "pull", - kamu_api_server_dataset_endpoint.as_str(), - "--no-alias", - ], - "1 dataset(s) updated", - ) - .await; + kamu_in_pull_workspace + .assert_success_command_execution( + [ + "pull", + kamu_api_server_dataset_endpoint.as_str(), + "--no-alias", + ], + None, + Some(["1 dataset(s) updated"]), + ) + .await; // Check alias should be empty let aliases = kamu_in_pull_workspace - .get_list_of_repo_aliases(&opendatafabric::DatasetRef::from(dataset_name.clone())) + .get_list_of_repo_aliases(&dataset_alias.dataset_name.clone().into()) .await; assert!(aliases.is_empty()); // Delete local dataset kamu_in_pull_workspace - .execute(["--yes", "delete", dataset_name.as_str()]) + .execute(["--yes", "delete", dataset_alias.dataset_name.as_str()]) .await .success(); // Dataset pull with storing alias - run_and_assert_command_success( - &kamu_in_pull_workspace, - vec!["pull", kamu_api_server_dataset_endpoint.as_str()], - "1 dataset(s) updated", - ) - .await; + kamu_in_pull_workspace + .assert_success_command_execution( + ["pull", kamu_api_server_dataset_endpoint.as_str()], + None, + Some(["1 dataset(s) updated"]), + ) + .await; let aliases = kamu_in_pull_workspace - .get_list_of_repo_aliases(&opendatafabric::DatasetRef::from(dataset_name.clone())) + .get_list_of_repo_aliases(&dataset_alias.dataset_name.clone().into()) .await; let expected_aliases = vec![RepoAlias { - dataset: dataset_name.clone(), + dataset: dataset_alias.dataset_name.clone(), kind: "Pull".to_string(), - alias: kamu_api_server_dataset_endpoint.clone(), + alias: kamu_api_server_dataset_endpoint.to_string(), }]; pretty_assertions::assert_eq!(aliases, expected_aliases); } @@ -383,7 +385,13 @@ pub async fn test_smart_push_pull_add_alias(kamu_api_server_client: KamuApiServe //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_smart_pull_as(kamu_api_server_client: KamuApiServerClient) { - let dataset_name = DatasetName::new_unchecked("player-scores"); + let dataset_alias = DatasetAlias::new( + Some(AccountName::new_unchecked("e2e-user")), + DatasetName::new_unchecked("player-scores"), + ); + let kamu_api_server_dataset_endpoint = + kamu_api_server_client.get_dataset_endpoint(&dataset_alias); + // 1. Grub a token let token = kamu_api_server_client.login_as_e2e_user().await; @@ -394,27 +402,22 @@ pub async fn test_smart_pull_as(kamu_api_server_client: KamuApiServerClient) { ) .await; - let kamu_api_server_dataset_endpoint = get_dataset_endpoint( - kamu_api_server_client.get_base_url(), - &dataset_name, - E2E_USER_ACCOUNT_NAME_STR, - ); - { let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; - let new_dataset_name = DatasetName::new_unchecked("foo"); - run_and_assert_command_success( - &kamu_in_pull_workspace, - vec![ - "pull", - kamu_api_server_dataset_endpoint.as_str(), - "--as", - new_dataset_name.as_str(), - ], - "1 dataset(s) updated", - ) - .await; + + kamu_in_pull_workspace + .assert_success_command_execution( + [ + "pull", + kamu_api_server_dataset_endpoint.as_str(), + "--as", + new_dataset_name.as_str(), + ], + None, + Some(["1 dataset(s) updated"]), + ) + .await; let expected_dataset_list = kamu_in_pull_workspace .list_datasets() @@ -430,23 +433,23 @@ pub async fn test_smart_pull_as(kamu_api_server_client: KamuApiServerClient) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClient) { - let dataset_name = DatasetName::new_unchecked("player-scores"); - let dataset_derivative_name = DatasetName::new_unchecked("leaderboard"); + let root_dataset_alias = DatasetAlias::new( + Some(AccountName::new_unchecked("e2e-user")), + DatasetName::new_unchecked("player-scores"), + ); + let kamu_api_server_root_dataset_endpoint = + kamu_api_server_client.get_dataset_endpoint(&root_dataset_alias); + + let derivative_dataset_alias = DatasetAlias::new( + Some(AccountName::new_unchecked("e2e-user")), + DatasetName::new_unchecked("leaderboard"), + ); + let kamu_api_server_derivative_dataset_endpoint = + kamu_api_server_client.get_dataset_endpoint(&derivative_dataset_alias); // 1. Grub a token let token = kamu_api_server_client.login_as_e2e_user().await; - let kamu_api_server_root_dataset_endpoint = get_dataset_endpoint( - kamu_api_server_client.get_base_url(), - &dataset_name, - E2E_USER_ACCOUNT_NAME_STR, - ); - let kamu_api_server_derivative_dataset_endpoint = get_dataset_endpoint( - kamu_api_server_client.get_base_url(), - &dataset_derivative_name, - E2E_USER_ACCOUNT_NAME_STR, - ); - let mut kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; // 2. Pushing datasets to the API server @@ -474,7 +477,7 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien { kamu_in_push_workspace .ingest_data( - &dataset_name, + &root_dataset_alias.dataset_name, DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, ) .await; @@ -492,42 +495,45 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien .success(); // Push all datasets should fail - run_and_assert_command_failure( - &kamu_in_push_workspace, - vec!["push", "--all"], - "Pushing all datasets is not yet supported", - ) - .await; + kamu_in_push_workspace + .assert_failure_command_execution( + ["push", "--all"], + None, + Some(["Pushing all datasets is not yet supported"]), + ) + .await; // Push datasets one by one - run_and_assert_command_success( - &kamu_in_push_workspace, - vec![ - "push", - dataset_name.as_str(), - "--to", - kamu_api_server_root_dataset_endpoint.as_str(), - ], - "1 dataset(s) pushed", - ) - .await; + kamu_in_push_workspace + .assert_success_command_execution( + [ + "push", + root_dataset_alias.dataset_name.as_str(), + "--to", + kamu_api_server_root_dataset_endpoint.as_str(), + ], + None, + Some(["1 dataset(s) pushed"]), + ) + .await; kamu_in_push_workspace - .execute(["pull", dataset_derivative_name.as_str()]) + .execute(["pull", derivative_dataset_alias.dataset_name.as_str()]) .await .success(); - run_and_assert_command_success( - &kamu_in_push_workspace, - vec![ - "push", - dataset_derivative_name.as_str(), - "--to", - kamu_api_server_derivative_dataset_endpoint.as_str(), - ], - "1 dataset(s) pushed", - ) - .await; + kamu_in_push_workspace + .assert_success_command_execution( + [ + "push", + derivative_dataset_alias.dataset_name.as_str(), + "--to", + kamu_api_server_derivative_dataset_endpoint.as_str(), + ], + None, + Some(["1 dataset(s) pushed"]), + ) + .await; } // 3. Pulling datasets from the API server @@ -535,18 +541,21 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; // Pull datasets one by one and check data - run_and_assert_command_success( - &kamu_in_pull_workspace, - vec!["pull", kamu_api_server_root_dataset_endpoint.as_str()], - "1 dataset(s) updated", - ) - .await; - run_and_assert_command_success( - &kamu_in_pull_workspace, - vec!["pull", kamu_api_server_derivative_dataset_endpoint.as_str()], - "1 dataset(s) updated", - ) - .await; + kamu_in_pull_workspace + .assert_success_command_execution( + ["pull", kamu_api_server_root_dataset_endpoint.as_str()], + None, + Some(["1 dataset(s) updated"]), + ) + .await; + + kamu_in_pull_workspace + .assert_success_command_execution( + ["pull", kamu_api_server_derivative_dataset_endpoint.as_str()], + None, + Some(["1 dataset(s) updated"]), + ) + .await; let expected_schema = indoc::indoc!( r#" @@ -597,11 +606,15 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien ); kamu_in_pull_workspace - .assert_last_data_slice(&dataset_name, expected_schema, expected_data) + .assert_last_data_slice( + &root_dataset_alias.dataset_name, + expected_schema, + expected_data, + ) .await; kamu_in_pull_workspace .assert_last_data_slice( - &dataset_derivative_name, + &derivative_dataset_alias.dataset_name, expected_derivative_schema, expected_derivative_data, ) @@ -611,47 +624,53 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien kamu_in_push_workspace .ingest_data( - &dataset_name, + &root_dataset_alias.dataset_name, DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, ) .await; - run_and_assert_command_success( - &kamu_in_push_workspace, - vec!["pull", dataset_derivative_name.as_str()], - "1 dataset(s) updated", - ) - .await; - run_and_assert_command_success( - &kamu_in_push_workspace, - vec![ - "push", - dataset_name.as_str(), - "--to", - kamu_api_server_root_dataset_endpoint.as_str(), - ], - "1 dataset(s) pushed", - ) - .await; - run_and_assert_command_success( - &kamu_in_push_workspace, - vec![ - "push", - dataset_derivative_name.as_str(), - "--to", - kamu_api_server_derivative_dataset_endpoint.as_str(), - ], - "1 dataset(s) pushed", - ) - .await; + kamu_in_push_workspace + .assert_success_command_execution( + ["pull", derivative_dataset_alias.dataset_name.as_str()], + None, + Some(["1 dataset(s) updated"]), + ) + .await; + + kamu_in_push_workspace + .assert_success_command_execution( + [ + "push", + root_dataset_alias.dataset_name.as_str(), + "--to", + kamu_api_server_root_dataset_endpoint.as_str(), + ], + None, + Some(["1 dataset(s) pushed"]), + ) + .await; + + kamu_in_push_workspace + .assert_success_command_execution( + [ + "push", + derivative_dataset_alias.dataset_name.as_str(), + "--to", + kamu_api_server_derivative_dataset_endpoint.as_str(), + ], + None, + Some(["1 dataset(s) pushed"]), + ) + .await; // Pull all datasets - run_and_assert_command_success( - &kamu_in_pull_workspace, - vec!["pull", "--all"], - "2 dataset(s) updated", - ) - .await; + kamu_in_pull_workspace + .assert_success_command_execution( + ["pull", "--all"], + None, + Some(["2 dataset(s) updated"]), + ) + .await; // Perform dataslices checks let expected_data = indoc::indoc!( @@ -676,11 +695,15 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien ); kamu_in_pull_workspace - .assert_last_data_slice(&dataset_name, expected_schema, expected_data) + .assert_last_data_slice( + &root_dataset_alias.dataset_name, + expected_schema, + expected_data, + ) .await; kamu_in_pull_workspace .assert_last_data_slice( - &dataset_derivative_name, + &derivative_dataset_alias.dataset_name, expected_derivative_schema, expected_derivative_data, ) @@ -691,17 +714,21 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServerClient) { - let dataset_name = DatasetName::new_unchecked("player-scores"); - let dataset_derivative_name = DatasetName::new_unchecked("leaderboard"); + let root_dataset_alias = DatasetAlias::new( + Some(AccountName::new_unchecked("e2e-user")), + DatasetName::new_unchecked("player-scores"), + ); + let kamu_api_server_root_dataset_endpoint = + kamu_api_server_client.get_dataset_endpoint(&root_dataset_alias); + + let derivative_dataset_alias = DatasetAlias::new( + Some(AccountName::new_unchecked("e2e-user")), + DatasetName::new_unchecked("leaderboard"), + ); // 1. Grub a token let token = kamu_api_server_client.login_as_e2e_user().await; - let kamu_api_server_root_dataset_endpoint = get_dataset_endpoint( - kamu_api_server_client.get_base_url(), - &dataset_name, - E2E_USER_ACCOUNT_NAME_STR, - ); let mut kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; // 2. Pushing datasets to the API server @@ -721,7 +748,7 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe { kamu_in_push_workspace .ingest_data( - &dataset_name, + &root_dataset_alias.dataset_name, DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, ) .await; @@ -739,40 +766,48 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe .success(); // Push all datasets should fail - run_and_assert_command_failure( - &kamu_in_push_workspace, - vec!["push", dataset_name.as_str(), "--recursive"], - "Recursive push is not yet supported", - ) - .await; + kamu_in_push_workspace + .assert_failure_command_execution( + [ + "push", + root_dataset_alias.dataset_name.as_str(), + "--recursive", + ], + None, + Some(["Recursive push is not yet supported"]), + ) + .await; // Push dataset - run_and_assert_command_success( - &kamu_in_push_workspace, - vec![ - "push", - dataset_name.as_str(), - "--to", - kamu_api_server_root_dataset_endpoint.as_str(), - ], - "1 dataset(s) pushed", - ) - .await; + kamu_in_push_workspace + .assert_success_command_execution( + [ + "push", + root_dataset_alias.dataset_name.as_str(), + "--to", + kamu_api_server_root_dataset_endpoint.as_str(), + ], + None, + Some(["1 dataset(s) pushed"]), + ) + .await; } // 3. Pulling datasets from the API server { let mut kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + kamu_in_pull_workspace .set_system_time(Some(DateTime::from_str("2050-01-02T03:04:05Z").unwrap())); // Pull datasets one by one and check data - run_and_assert_command_success( - &kamu_in_pull_workspace, - vec!["pull", kamu_api_server_root_dataset_endpoint.as_str()], - "1 dataset(s) updated", - ) - .await; + kamu_in_pull_workspace + .assert_success_command_execution( + ["pull", kamu_api_server_root_dataset_endpoint.as_str()], + None, + Some(["1 dataset(s) updated"]), + ) + .await; kamu_in_pull_workspace .execute_with_input( @@ -782,12 +817,13 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe .await .success(); - run_and_assert_command_success( - &kamu_in_pull_workspace, - vec!["pull", dataset_derivative_name.as_str()], - "1 dataset(s) updated", - ) - .await; + kamu_in_pull_workspace + .assert_success_command_execution( + ["pull", derivative_dataset_alias.dataset_name.as_str()], + None, + Some(["1 dataset(s) updated"]), + ) + .await; let expected_schema = indoc::indoc!( r#" @@ -838,11 +874,15 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe ); kamu_in_pull_workspace - .assert_last_data_slice(&dataset_name, expected_schema, expected_data) + .assert_last_data_slice( + &root_dataset_alias.dataset_name, + expected_schema, + expected_data, + ) .await; kamu_in_pull_workspace .assert_last_data_slice( - &dataset_derivative_name, + &derivative_dataset_alias.dataset_name, expected_derivative_schema, expected_derivative_data, ) @@ -852,29 +892,36 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe kamu_in_push_workspace .ingest_data( - &dataset_name, + &root_dataset_alias.dataset_name, DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, ) .await; - run_and_assert_command_success( - &kamu_in_push_workspace, - vec![ - "push", - dataset_name.as_str(), - "--to", - kamu_api_server_root_dataset_endpoint.as_str(), - ], - "1 dataset(s) pushed", - ) - .await; + + kamu_in_push_workspace + .assert_success_command_execution( + [ + "push", + root_dataset_alias.dataset_name.as_str(), + "--to", + kamu_api_server_root_dataset_endpoint.as_str(), + ], + None, + Some(["1 dataset(s) pushed"]), + ) + .await; // Pull all datasets - run_and_assert_command_success( - &kamu_in_pull_workspace, - vec!["pull", dataset_derivative_name.as_str(), "--recursive"], - "2 dataset(s) updated", - ) - .await; + kamu_in_pull_workspace + .assert_success_command_execution( + [ + "pull", + derivative_dataset_alias.dataset_name.as_str(), + "--recursive", + ], + None, + Some(["2 dataset(s) updated"]), + ) + .await; // Perform dataslices checks let expected_data = indoc::indoc!( @@ -899,11 +946,15 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe ); kamu_in_pull_workspace - .assert_last_data_slice(&dataset_name, expected_schema, expected_data) + .assert_last_data_slice( + &root_dataset_alias.dataset_name, + expected_schema, + expected_data, + ) .await; kamu_in_pull_workspace .assert_last_data_slice( - &dataset_derivative_name, + &derivative_dataset_alias.dataset_name, expected_derivative_schema, expected_derivative_data, ) @@ -920,22 +971,17 @@ pub async fn test_smart_pull_set_watermark(kamu: KamuCliPuppet) { .await .success(); - let assert = kamu - .execute([ + kamu.assert_success_command_execution( + [ "pull", dataset_name.as_str(), "--set-watermark", "2051-01-02T03:04:05Z", - ]) - .await - .success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(indoc::indoc!(r#"Committed new block"#).trim()), - "Unexpected output:\n{stderr}", - ); + ], + None, + Some(["Committed new block"]), + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -961,10 +1007,10 @@ pub async fn test_smart_pull_reset_derivative(kamu: KamuCliPuppet) { ) .await; - run_and_assert_command_success( - &kamu, - vec!["pull", dataset_derivative_name.as_str()], - "1 dataset(s) updated", + kamu.assert_success_command_execution( + ["pull", dataset_derivative_name.as_str()], + None, + Some(["1 dataset(s) updated"]), ) .await; @@ -1012,10 +1058,10 @@ pub async fn test_smart_pull_reset_derivative(kamu: KamuCliPuppet) { .success(); // Pull derivative should fail - run_and_assert_command_failure( - &kamu, - vec!["pull", dataset_derivative_name.as_str()], - "Failed to update 1 dataset(s)", + kamu.assert_failure_command_execution( + ["pull", dataset_derivative_name.as_str()], + None, + Some(["Failed to update 1 dataset(s)"]), ) .await; @@ -1026,14 +1072,14 @@ pub async fn test_smart_pull_reset_derivative(kamu: KamuCliPuppet) { ) .await; - run_and_assert_command_success( - &kamu, - vec![ + kamu.assert_success_command_execution( + [ "pull", dataset_derivative_name.as_str(), "--reset-derivatives-on-diverged-input", ], - "1 dataset(s) updated", + None, + Some(["1 dataset(s) updated"]), ) .await; @@ -1058,38 +1104,33 @@ pub async fn test_smart_pull_reset_derivative(kamu: KamuCliPuppet) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_smart_push_visibility(kamu_api_server_client: KamuApiServerClient) { - let dataset_name = DatasetName::new_unchecked("player-scores"); + let dataset_alias = DatasetAlias::new( + Some(AccountName::new_unchecked("e2e-user")), + DatasetName::new_unchecked("player-scores"), + ); + let kamu_api_server_dataset_endpoint = + kamu_api_server_client.get_dataset_endpoint(&dataset_alias); // 1. Grub a token let token = kamu_api_server_client.login_as_e2e_user().await; - let kamu_api_server_dataset_endpoint = get_dataset_endpoint( - kamu_api_server_client.get_base_url(), - &dataset_name, - E2E_USER_ACCOUNT_NAME_STR, - ); - // 2. Pushing the dataset to the API server { let kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; // 2.1. Add the dataset - { - kamu_in_push_workspace - .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) - .await - .success(); - } + kamu_in_push_workspace + .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); // 2.1. Ingest data to the dataset - { - kamu_in_push_workspace - .ingest_data( - &dataset_name, - DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, - ) - .await; - } + kamu_in_push_workspace + .ingest_data( + &dataset_alias.dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; // 2.2. Login to the API server kamu_in_push_workspace @@ -1102,19 +1143,20 @@ pub async fn test_smart_push_visibility(kamu_api_server_client: KamuApiServerCli .await .success(); - run_and_assert_command_success( - &kamu_in_push_workspace, - vec![ - "push", - dataset_name.as_str(), - "--to", - kamu_api_server_dataset_endpoint.as_str(), - "--visibility", - "private", - ], - "1 dataset(s) pushed", - ) - .await; + kamu_in_push_workspace + .assert_success_command_execution( + [ + "push", + dataset_alias.dataset_name.as_str(), + "--to", + kamu_api_server_dataset_endpoint.as_str(), + "--visibility", + "private", + ], + None, + Some(["1 dataset(s) pushed"]), + ) + .await; // ToDo add visibility check } @@ -1136,25 +1178,26 @@ pub async fn test_smart_push_pull_s3(kamu: KamuCliPuppet) { .await; let s3_server = LocalS3Server::new().await; - let dataset_url = format!("{}/e2e-user/{dataset_name}", s3_server.url); + // Push dataset - run_and_assert_command_success( - &kamu, - vec!["push", dataset_name.as_str(), "--to", dataset_url.as_str()], - "1 dataset(s) pushed", + kamu.assert_success_command_execution( + ["push", dataset_name.as_str(), "--to", dataset_url.as_str()], + None, + Some(["1 dataset(s) pushed"]), ) .await; { let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; - run_and_assert_command_success( - &kamu_in_pull_workspace, - vec!["pull", dataset_url.as_str()], - "1 dataset(s) updated", - ) - .await; + kamu_in_pull_workspace + .assert_success_command_execution( + ["pull", dataset_url.as_str()], + None, + Some(["1 dataset(s) updated"]), + ) + .await; let expected_schema = indoc::indoc!( r#" @@ -1207,118 +1250,36 @@ pub async fn test_smart_pull_derivative(kamu: KamuCliPuppet) { ) .await; - run_and_assert_command_failure( - &kamu, - vec![ + kamu.assert_failure_command_execution( + [ "tail", dataset_derivative_name.as_str(), "--output-format", "table", ], - "Error: Dataset schema is not yet available: leaderboard", + None, + Some(["Error: Dataset schema is not yet available: leaderboard"]), ) .await; - run_and_assert_command_success( - &kamu, - vec!["pull", dataset_derivative_name.as_str()], - "1 dataset(s) updated", + kamu.assert_success_command_execution( + ["pull", dataset_derivative_name.as_str()], + None, + Some(["1 dataset(s) updated"]), ) .await; - { - let assert = kamu - .execute([ - "sql", - "--engine", - "datafusion", - "--command", - // Without unstable "offset" column. - // For a beautiful output, cut to seconds - indoc::indoc!( - r#" - SELECT op, - system_time, - DATE_TRUNC('second', match_time) as match_time, - match_id, - player_id, - score - FROM "player-scores" - ORDER BY match_time; - "# - ), - "--output-format", - "table", - ]) - .await - .success(); - - let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); - - pretty_assertions::assert_eq!( - indoc::indoc!( - r#" - ┌────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ - │ op │ system_time │ match_time │ match_id │ player_id │ score │ - ├────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ - │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ - │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ - └────┴──────────────────────┴──────────────────────┴──────────┴───────────┴───────┘ - "# - ), - stdout - ); - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// Helpers -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -fn get_dataset_endpoint( - base_url: &Url, - dataset_name: &DatasetName, - account_name_str: &str, -) -> String { - let mut dataset_endpoint = Url::parse("odf+http://host").unwrap(); - - dataset_endpoint.set_host(base_url.host_str()).unwrap(); - dataset_endpoint.set_port(base_url.port()).unwrap(); - - dataset_endpoint - .join(format!("{account_name_str}/{dataset_name}").as_str()) - .unwrap() - .to_string() -} - -async fn run_and_assert_command_success( - kamu: &KamuCliPuppet, - args: Vec<&str>, - expected_message: &str, -) { - let assert = kamu.execute(args).await.success(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(expected_message), - "Unexpected output:\n{stderr}", - ); -} - -async fn run_and_assert_command_failure( - kamu: &KamuCliPuppet, - args: Vec<&str>, - expected_message: &str, -) { - let assert = kamu.execute(args).await.failure(); - - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - - assert!( - stderr.contains(expected_message), - "Unexpected output:\n{stderr}", - ); + kamu.assert_player_scores_dataset_data(indoc::indoc!( + r#" + ┌────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ + │ op │ system_time │ match_time │ match_id │ player_id │ score │ + ├────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ + │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ + └────┴──────────────────────┴──────────────────────┴──────────┴───────────┴───────┘ + "# + )) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_verify_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_verify_command.rs index fde8f8389..e025ab033 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_verify_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_verify_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_verify_regular_dataset + fixture = kamu_cli_e2e_repo_tests::test_verify_regular_dataset, extra_test_groups = "engine, ingest, datafusion" ); @@ -21,15 +21,15 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_verify_recursive - extra_test_groups = "containerized, engine, ingest, datafusion" + fixture = kamu_cli_e2e_repo_tests::test_verify_recursive, + extra_test_groups = "containerized, engine, ingest, datafusion, risingwave" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_verify_integrity + fixture = kamu_cli_e2e_repo_tests::test_verify_integrity, extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/sqlite/tests/tests/test_flow.rs b/src/e2e/app/cli/sqlite/tests/tests/test_flow.rs index 9ec764030..a0a38ec16 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/test_flow.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/test_flow.rs @@ -38,7 +38,7 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = sqlite, fixture = kamu_cli_e2e_repo_tests::test_dataset_trigger_flow, - extra_test_groups = "containerized, engine, transform, datafusion" + extra_test_groups = "containerized, engine, transform, datafusion, risingwave" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/test_smart_transfer_protocol.rs b/src/e2e/app/cli/sqlite/tests/tests/test_smart_transfer_protocol.rs index e7655e724..a77a86dd3 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/test_smart_transfer_protocol.rs @@ -64,7 +64,7 @@ kamu_cli_run_api_server_e2e_test!( options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, transform, datafusion" + extra_test_groups = "containerized, engine, ingest, transform, datafusion, risingwave" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -75,7 +75,7 @@ kamu_cli_run_api_server_e2e_test!( options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, transform, datafusion" + extra_test_groups = "containerized, engine, ingest, transform, datafusion, risingwave" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -92,7 +92,7 @@ kamu_cli_execute_command_e2e_test!( storage = sqlite, fixture = kamu_cli_e2e_repo_tests::test_smart_pull_reset_derivative, options = Options::default().with_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, transform, datafusion" + extra_test_groups = "containerized, engine, ingest, transform, datafusion, risingwave" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -121,7 +121,7 @@ kamu_cli_execute_command_e2e_test!( storage = sqlite, fixture = kamu_cli_e2e_repo_tests::test_smart_pull_derivative, options = Options::default().with_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, transform, datafusion" + extra_test_groups = "containerized, engine, ingest, transform, datafusion, risingwave" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/kamu-cli-puppet/Cargo.toml b/src/utils/kamu-cli-puppet/Cargo.toml index 43620908c..1285e6659 100644 --- a/src/utils/kamu-cli-puppet/Cargo.toml +++ b/src/utils/kamu-cli-puppet/Cargo.toml @@ -29,6 +29,8 @@ extensions = [ # External "dep:async-trait", "dep:datafusion", + "dep:indoc", + "dep:pretty_assertions", "dep:serde", "dep:serde_json", ] @@ -47,6 +49,8 @@ opendatafabric = { optional = true, workspace = true } async-trait = { optional = true, version = "0.1" } datafusion = { optional = true, version = "42", default-features = false } +indoc = { optional = true, version = "2" } +pretty_assertions = { optional = true, version = "1" } serde = { optional = true, version = "1", default-features = false, features = [ "derive", ] } diff --git a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs index 1047e8df8..5ba68f52c 100644 --- a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs +++ b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs @@ -26,12 +26,52 @@ use opendatafabric::{ }; use serde::Deserialize; -use crate::KamuCliPuppet; +use crate::{ExecuteCommandResult, KamuCliPuppet}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait] pub trait KamuCliPuppetExt { + async fn assert_success_command_execution( + &self, + cmd: I, + maybe_expected_stdout: Option<&str>, + maybe_expected_stderr: Option + Send>, + ) where + I: IntoIterator + Send, + S: AsRef; + + async fn assert_success_command_execution_with_input( + &self, + cmd: I, + input: T, + maybe_expected_stdout: Option<&str>, + maybe_expected_stderr: Option + Send>, + ) where + I: IntoIterator + Send, + S: AsRef, + T: Into> + Send; + + async fn assert_failure_command_execution( + &self, + cmd: I, + maybe_expected_stdout: Option<&str>, + maybe_expected_stderr: Option + Send>, + ) where + I: IntoIterator + Send, + S: AsRef; + + async fn assert_failure_command_execution_with_input( + &self, + cmd: I, + input: T, + maybe_expected_stdout: Option<&str>, + maybe_expected_stderr: Option + Send>, + ) where + I: IntoIterator + Send, + S: AsRef, + T: Into> + Send; + async fn list_datasets(&self) -> Vec; async fn add_dataset(&self, dataset_snapshot: DatasetSnapshot); @@ -48,6 +88,8 @@ pub trait KamuCliPuppetExt { async fn start_api_server(self, e2e_data_file_path: PathBuf) -> ServerOutput; + async fn assert_player_scores_dataset_data(&self, expected_player_scores_table: &str); + async fn assert_last_data_slice( &self, dataset_name: &DatasetName, @@ -184,6 +226,36 @@ impl KamuCliPuppetExt for KamuCliPuppet { ServerOutput { stdout, stderr } } + async fn assert_player_scores_dataset_data(&self, expected_player_scores_table: &str) { + self.assert_success_command_execution( + [ + "sql", + "--engine", + "datafusion", + "--command", + // Without unstable "offset" column. + // For a beautiful output, cut to seconds + indoc::indoc!( + r#" + SELECT op, + system_time, + match_time, + match_id, + player_id, + score + FROM "player-scores" + ORDER BY match_id, score, player_id; + "# + ), + "--output-format", + "table", + ], + Some(expected_player_scores_table), + None::>, + ) + .await; + } + async fn assert_last_data_slice( &self, dataset_name: &DatasetName, @@ -230,6 +302,74 @@ impl KamuCliPuppetExt for KamuCliPuppet { .await .success(); } + + async fn assert_success_command_execution( + &self, + cmd: I, + maybe_expected_stdout: Option<&str>, + maybe_expected_stderr: Option + Send>, + ) where + I: IntoIterator + Send, + S: AsRef, + { + assert_execute_command_result( + &self.execute(cmd).await.success(), + maybe_expected_stdout, + maybe_expected_stderr, + ); + } + + async fn assert_success_command_execution_with_input( + &self, + cmd: I, + input: T, + maybe_expected_stdout: Option<&str>, + maybe_expected_stderr: Option + Send>, + ) where + I: IntoIterator + Send, + S: AsRef, + T: Into> + Send, + { + assert_execute_command_result( + &self.execute_with_input(cmd, input).await.success(), + maybe_expected_stdout, + maybe_expected_stderr, + ); + } + + async fn assert_failure_command_execution( + &self, + cmd: I, + maybe_expected_stdout: Option<&str>, + maybe_expected_stderr: Option + Send>, + ) where + I: IntoIterator + Send, + S: AsRef, + { + assert_execute_command_result( + &self.execute(cmd).await.failure(), + maybe_expected_stdout, + maybe_expected_stderr, + ); + } + + async fn assert_failure_command_execution_with_input( + &self, + cmd: I, + input: T, + maybe_expected_stdout: Option<&str>, + maybe_expected_stderr: Option + Send>, + ) where + I: IntoIterator + Send, + S: AsRef, + T: Into> + Send, + { + assert_execute_command_result( + &self.execute_with_input(cmd, input).await.failure(), + maybe_expected_stdout, + maybe_expected_stderr, + ); + } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -273,3 +413,28 @@ pub struct BlockRecord { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +fn assert_execute_command_result<'a>( + command_result: &ExecuteCommandResult, + maybe_expected_stdout: Option<&str>, + maybe_expected_stderr: Option>, +) { + let actual_stdout = std::str::from_utf8(&command_result.get_output().stdout).unwrap(); + + if let Some(expected_stdout) = maybe_expected_stdout { + pretty_assertions::assert_eq!(expected_stdout, actual_stdout); + } + + if let Some(expected_stderr_items) = maybe_expected_stderr { + let stderr = std::str::from_utf8(&command_result.get_output().stderr).unwrap(); + + for expected_stderr_item in expected_stderr_items { + assert!( + stderr.contains(expected_stderr_item), + "Unexpected output:\n{stderr}", + ); + } + } +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////