Push, pull, compact, verify commands e2e tests (#893)

* Add query commitments example

* Release (patch) `0.204.5`: fix `--yes / -y` flag: fixed when working from a TTY (#881)

* Interact: fix if is_tty

* CHANGELOG.md: update

* Release (patch): 0.204.5

* Updated yanked crate (futures-util to 0.3.31)

* CI: Fixes `kamu-base-with-data-mt` image builds (#885)

* Images, kamu-base-with-data-mt: add "kamu" to predefined users

* Images, kamu-base-with-data-mt: init-workspace.py use .kamuconfig

* CHANGELOG.md: update

* Add push pull tests

* Add s3 tests

* Remove unused code

* Test transform engine error

* Revert check

* Add container extra groups

* Add verify and compact commands

* Fix grammar

* Fix review commets

* Revert "Merge branch 'master' into chore/push-and-pul-command-e2e-tests"

This reverts commit a4ec39c, reversing
changes made to 1130fa5.

* Clean code

* Fix review comments. Iter 2

* Revert ToDo comment

---------

Co-authored-by: Sergii Mikhtoniuk <mikhtoniuk@gmail.com>
Co-authored-by: Dima Pristupa <dima@pristupa.dev>

src/app/cli/src/commands/verify_command.rs

@@ -152,7 +152,6 @@ impl VerifyCommand {
 
         Ok(self
             .verification_svc
-            .clone()
             .verify_multi(filtered_requests, options, listener)
             .await)
     }
@@ -193,14 +192,14 @@ impl VerifyCommand {
 
                 let mut current_missed_dependencies = vec![];
 
-                for dependecy in summary.dependencies {
+                for dependency in summary.dependencies {
                     if self
                         .dataset_repo
-                        .resolve_dataset_ref(&DatasetRef::ID(dependecy.clone()))
+                        .resolve_dataset_ref(&DatasetRef::ID(dependency.clone()))
                         .await
                         .is_err()
                     {
-                        current_missed_dependencies.push(dependecy.to_string());
+                        current_missed_dependencies.push(dependency.to_string());
                     }
                 }
                 if !current_missed_dependencies.is_empty() {

src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs

@@ -9,7 +9,7 @@
 
 use async_trait::async_trait;
 use lazy_static::lazy_static;
-use opendatafabric::{DatasetAlias, DatasetName};
+use opendatafabric::{AccountName, DatasetAlias, DatasetName};
 use reqwest::{Method, StatusCode};
 
 use crate::{KamuApiServerClient, RequestBody};
@@ -134,6 +134,8 @@ pub const DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_3: &str = indoc::i
     "#
 );
 
+pub const E2E_USER_ACCOUNT_NAME_STR: &str = "e2e-user";
+
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 pub type AccessToken = String;
@@ -153,8 +155,11 @@ pub trait KamuApiServerClientExt {
 
     async fn create_player_scores_dataset(&self, token: &AccessToken) -> DatasetId;
 
-    /// NOTE: only for single-tenant workspaces
-    async fn create_player_scores_dataset_with_data(&self, token: &AccessToken) -> DatasetId;
+    async fn create_player_scores_dataset_with_data(
+        &self,
+        token: &AccessToken,
+        account_name_maybe: Option<AccountName>,
+    ) -> DatasetId;
 
     async fn create_leaderboard(&self, token: &AccessToken) -> DatasetId;
 
@@ -249,14 +254,19 @@ impl KamuApiServerClientExt for KamuApiServerClient {
             .await
     }
 
-    async fn create_player_scores_dataset_with_data(&self, token: &AccessToken) -> DatasetId {
+    async fn create_player_scores_dataset_with_data(
+        &self,
+        token: &AccessToken,
+        account_name_maybe: Option<AccountName>,
+    ) -> DatasetId {
         let dataset_id = self.create_player_scores_dataset(token).await;
 
         // TODO: Use the alias from the reply, after fixing the bug:
         //       https://github.com/kamu-data/kamu-cli/issues/891
-
-        // At the moment, only single-tenant
-        let dataset_alias = DatasetAlias::new(None, DatasetName::new_unchecked("player-scores"));
+        let dataset_alias = DatasetAlias::new(
+            account_name_maybe,
+            DatasetName::new_unchecked("player-scores"),
+        );
 
         self.ingest_data(
             &dataset_alias,

src/e2e/app/cli/inmem/tests/tests/commands/mod.rs

@@ -8,6 +8,7 @@
 // by the Apache License, Version 2.0.
 
 mod test_add_command;
+mod test_compact_command;
 mod test_complete_command;
 mod test_config_command;
 mod test_delete_command;
@@ -28,3 +29,4 @@ mod test_system_gc_command;
 mod test_system_generate_token_command;
 mod test_system_info_command;
 mod test_tail_command;
+mod test_verify_command;

src/e2e/app/cli/inmem/tests/tests/commands/test_compact_command.rs

@@ -0,0 +1,36 @@
+// Copyright Kamu Data, Inc. and contributors. All rights reserved.
+//
+// Use of this software is governed by the Business Source License
+// included in the LICENSE file.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0.
+
+use kamu_cli_e2e_common::prelude::*;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_execute_command_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_compact_hard
+    extra_test_groups = "engine, ingest, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_execute_command_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_compact_keep_metadata_only
+    extra_test_groups = "engine, ingest, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_execute_command_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_compact_verify
+    extra_test_groups = "engine, ingest, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

src/e2e/app/cli/inmem/tests/tests/commands/test_verify_command.rs

@@ -0,0 +1,36 @@
+// Copyright Kamu Data, Inc. and contributors. All rights reserved.
+//
+// Use of this software is governed by the Business Source License
+// included in the LICENSE file.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0.
+
+use kamu_cli_e2e_common::prelude::*;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_execute_command_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_verify_regular_dataset
+    extra_test_groups = "engine, ingest, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_execute_command_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_verify_recursive
+    extra_test_groups = "containerized, engine, ingest, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_execute_command_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_verify_integrity
+    extra_test_groups = "engine, ingest, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs

@@ -24,3 +24,104 @@ kamu_cli_run_api_server_e2e_test!(
 );
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_run_api_server_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_smart_force_push_pull,
+    options = Options::default()
+        .with_multi_tenant()
+        .with_today_as_frozen_system_time(),
+    extra_test_groups = "engine, ingest, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_run_api_server_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_add_alias,
+    options = Options::default()
+        .with_multi_tenant()
+        .with_today_as_frozen_system_time(),
+    extra_test_groups = "engine, ingest, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_run_api_server_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_smart_pull_as,
+    options = Options::default()
+        .with_multi_tenant()
+        .with_today_as_frozen_system_time(),
+    extra_test_groups = "engine, ingest, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_run_api_server_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_all,
+    options = Options::default()
+        .with_multi_tenant()
+        .with_today_as_frozen_system_time(),
+    extra_test_groups = "containerized, engine, ingest, transform, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_run_api_server_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_recursive,
+    options = Options::default()
+        .with_multi_tenant()
+        .with_today_as_frozen_system_time(),
+    extra_test_groups = "containerized, engine, ingest, transform, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_execute_command_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_smart_pull_set_watermark,
+    options = Options::default().with_frozen_system_time(),
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_execute_command_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_smart_pull_reset_derivative,
+    options = Options::default().with_frozen_system_time(),
+    extra_test_groups = "containerized, engine, ingest, transform, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_run_api_server_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_smart_push_visibility,
+    options = Options::default()
+        .with_multi_tenant()
+        .with_today_as_frozen_system_time(),
+    extra_test_groups = "engine, ingest, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_execute_command_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_s3,
+    options = Options::default().with_frozen_system_time(),
+    extra_test_groups = "containerized, engine, ingest, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_execute_command_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_smart_pull_derivative,
+    options = Options::default().with_frozen_system_time(),
+    extra_test_groups = "containerized, engine, ingest, transform, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

src/e2e/app/cli/repo-tests/src/commands/mod.rs

@@ -8,6 +8,7 @@
 // by the Apache License, Version 2.0.
 
 mod test_add_command;
+mod test_compact_command;
 mod test_complete_command;
 mod test_config_command;
 mod test_delete_command;
@@ -28,8 +29,10 @@ mod test_system_generate_token_command;
 mod test_system_info_command;
 mod test_system_info_diagnose;
 mod test_tail_command;
+mod test_verify_command;
 
 pub use test_add_command::*;
+pub use test_compact_command::*;
 pub use test_complete_command::*;
 pub use test_config_command::*;
 pub use test_delete_command::*;
@@ -50,3 +53,4 @@ pub use test_system_generate_token_command::*;
 pub use test_system_info_command::*;
 pub use test_system_info_diagnose::*;
 pub use test_tail_command::*;
+pub use test_verify_command::*;