Skip to content

Commit

Permalink
REST API tests for default bucket prefix (cvat-ai#7079)
Browse files Browse the repository at this point in the history 
This PR contains REST API tests for cvat-ai#6943
  • Loading branch information
@Marishka17
Marishka17 committed Nov 2, 2023
1 parent d2b5f3d commit 1f8d5d3
Show file tree
Hide file tree
Showing 4 changed files with 200 additions and 3 deletions.
2 changes: 1 addition & 1 deletion cvat/apps/engine/cloud_provider.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,7 @@ def list_files_on_one_page(

search_prefix = prefix
if self.prefix and (len(prefix) < len(self.prefix)):
if '/' in self.prefix[len(prefix):]:
if prefix and '/' in self.prefix[len(prefix):]:
next_layer_and_tail = self.prefix[prefix.find('/') + 1:].split(
"/", maxsplit=1
)
Expand Down
150 changes: 149 additions & 1 deletion tests/python/rest_api/test_cloud_storages.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -445,6 +445,7 @@ def test_org_user_get_cloud_storage_preview(
self._test_cannot_see(username, storage_id)


@pytest.mark.usefixtures("restore_db_per_function")
class TestGetCloudStorageContent:
USER = "admin1"

Expand Down Expand Up @@ -477,27 +478,30 @@ def _test_get_cloud_storage_content(

@pytest.mark.parametrize("cloud_storage_id", [2])
@pytest.mark.parametrize(
"version, manifest, prefix, page_size, expected_content",
"version, manifest, prefix, default_bucket_prefix, page_size, expected_content",
[
(
SUPPORTED_VERSIONS.V1, # [v1] list all bucket content
"sub/manifest.jsonl",
None,
None,
None,
["sub/image_case_65_1.png", "sub/image_case_65_2.png"],
),
(
SUPPORTED_VERSIONS.V2, # [v2] list the top level of bucket with based on manifest
"sub/manifest.jsonl",
None,
None,
None,
[FileInfo(mime_type="DIR", name="sub", type="DIR")],
),
(
SUPPORTED_VERSIONS.V2, # [v2] search by some prefix in bucket content based on manifest
"sub/manifest.jsonl",
"sub/image_case_65_1",
None,
None,
[
FileInfo(mime_type="image", name="image_case_65_1.png", type="REG"),
],
Expand All @@ -507,6 +511,7 @@ def _test_get_cloud_storage_content(
"sub/manifest.jsonl",
"sub/",
None,
None,
[
FileInfo(mime_type="image", name="image_case_65_1.png", type="REG"),
FileInfo(mime_type="image", name="image_case_65_2.png", type="REG"),
Expand All @@ -517,12 +522,14 @@ def _test_get_cloud_storage_content(
None,
None,
None,
None,
[FileInfo(mime_type="DIR", name="sub", type="DIR")],
),
(
SUPPORTED_VERSIONS.V2, # [v2] list the second layer (directory "sub") of real bucket content
None,
"sub/",
None,
2,
[
FileInfo(mime_type="unknown", name="demo_manifest.jsonl", type="REG"),
Expand All @@ -534,6 +541,83 @@ def _test_get_cloud_storage_content(
None,
"/sub/", # cover case: API is identical to share point API
None,
None,
[
FileInfo(mime_type="unknown", name="demo_manifest.jsonl", type="REG"),
FileInfo(mime_type="image", name="image_case_65_1.png", type="REG"),
FileInfo(mime_type="image", name="image_case_65_2.png", type="REG"),
FileInfo(mime_type="unknown", name="manifest.jsonl", type="REG"),
FileInfo(mime_type="unknown", name="manifest_1.jsonl", type="REG"),
FileInfo(mime_type="unknown", name="manifest_2.jsonl", type="REG"),
],
),
(
SUPPORTED_VERSIONS.V2, # [v2] list bucket content based on manifest when default bucket prefix is set to directory
"sub/manifest.jsonl",
None,
"sub/",
None,
[
FileInfo(mime_type="image", name="image_case_65_1.png", type="REG"),
FileInfo(mime_type="image", name="image_case_65_2.png", type="REG"),
],
),
(
# [v2] list bucket content based on manifest when default bucket prefix
# is set to template from which the files should start
SUPPORTED_VERSIONS.V2,
"sub/manifest.jsonl",
None,
"sub/image_case_65_1",
None,
[
FileInfo(mime_type="image", name="image_case_65_1.png", type="REG"),
],
),
(
SUPPORTED_VERSIONS.V2, # [v2] list bucket content based on manifest when specified prefix is stricter than default bucket prefix
"sub/manifest.jsonl",
"sub/image_case_65_1",
"sub/image_case",
None,
[
FileInfo(mime_type="image", name="image_case_65_1.png", type="REG"),
],
),
(
SUPPORTED_VERSIONS.V2, # [v2] list bucket content based on manifest when default bucket prefix is stricter than specified prefix
"sub/manifest.jsonl",
"sub/image_case",
"sub/image_case_65_1",
None,
[
FileInfo(mime_type="image", name="image_case_65_1.png", type="REG"),
],
),
(
SUPPORTED_VERSIONS.V2, # [v2] list bucket content based on manifest when default bucket prefix and specified prefix have no intersection
"sub/manifest.jsonl",
"sub/image_case_65_1",
"sub/image_case_65_2",
None,
[],
),
(
SUPPORTED_VERSIONS.V2, # [v2] list bucket content based on manifest when default bucket prefix contains dirs and prefix starts with it
"sub/manifest.jsonl",
"s",
"sub/",
None,
[
FileInfo(mime_type="DIR", name="sub", type="DIR"),
],
),
(
SUPPORTED_VERSIONS.V2, # [v2] list real bucket content when default bucket prefix is set to directory
None,
None,
"sub/",
None,
[
FileInfo(mime_type="unknown", name="demo_manifest.jsonl", type="REG"),
FileInfo(mime_type="image", name="image_case_65_1.png", type="REG"),
Expand All @@ -543,6 +627,56 @@ def _test_get_cloud_storage_content(
FileInfo(mime_type="unknown", name="manifest_2.jsonl", type="REG"),
],
),
(
# [v2] list real bucket content when default bucket prefix
# is set to template from which the files should start
SUPPORTED_VERSIONS.V2,
None,
None,
"sub/demo",
None,
[
FileInfo(mime_type="unknown", name="demo_manifest.jsonl", type="REG"),
],
),
(
SUPPORTED_VERSIONS.V2, # [v2] list real bucket content when specified prefix is stricter than default bucket prefix
None,
"sub/image_case_65_1",
"sub/image_case",
None,
[
FileInfo(mime_type="image", name="image_case_65_1.png", type="REG"),
],
),
(
SUPPORTED_VERSIONS.V2, # [v2] list real bucket content when default bucket prefix is stricter than specified prefix
None,
"sub/image_case",
"sub/image_case_65_1",
None,
[
FileInfo(mime_type="image", name="image_case_65_1.png", type="REG"),
],
),
(
SUPPORTED_VERSIONS.V2, # [v2] list real bucket content when default bucket prefix and specified prefix have no intersection
None,
"sub/image_case_65_1",
"sub/image_case_65_2",
None,
[],
),
(
SUPPORTED_VERSIONS.V2, # [v2] list real bucket content when default bucket prefix contains dirs and prefix starts with it
None,
"s",
"sub/",
None,
[
FileInfo(mime_type="DIR", name="sub", type="DIR"),
],
),
],
)
def test_get_cloud_storage_content(
Expand All @@ -551,9 +685,23 @@ def test_get_cloud_storage_content(
version: SUPPORTED_VERSIONS,
manifest: Optional[str],
prefix: Optional[str],
default_bucket_prefix: Optional[str],
page_size: Optional[int],
expected_content: Optional[Any],
cloud_storages,
):
if default_bucket_prefix:
cloud_storage = cloud_storages[cloud_storage_id]

with make_api_client(self.USER) as api_client:
(_, response) = api_client.cloudstorages_api.partial_update(
cloud_storage_id,
patched_cloud_storage_write_request={
"specific_attributes": f'{cloud_storage["specific_attributes"]}&prefix={default_bucket_prefix}'
},
)
assert response.status == HTTPStatus.OK

result = self._test_get_cloud_storage_content(
cloud_storage_id, version, manifest, prefix=prefix, page_size=page_size
)
Expand Down
49 changes: 49 additions & 0 deletions tests/python/rest_api/test_tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1620,6 +1620,55 @@ def test_cannot_create_task_with_same_skeleton_sublabels(self):
response = get_method(self._USERNAME, "tasks")
assert response.status_code == HTTPStatus.OK

@pytest.mark.with_external_services
@pytest.mark.parametrize("cloud_storage_id", [2])
@pytest.mark.parametrize("use_manifest", [True, False])
@pytest.mark.parametrize("server_files", [["test/"]])
@pytest.mark.parametrize(
"default_prefix, expected_task_size",
[
(
"test/sub_1/img_0",
1,
),
(
"test/sub_1/",
3,
),
],
)
@pytest.mark.parametrize("org", [""])
def test_create_task_with_cloud_storage_directories_and_default_bucket_prefix(
self,
cloud_storage_id: int,
use_manifest: bool,
server_files: List[str],
default_prefix: str,
expected_task_size: int,
org: str,
cloud_storages,
request,
):
cloud_storage = cloud_storages[cloud_storage_id]

with make_api_client(self._USERNAME) as api_client:
(_, response) = api_client.cloudstorages_api.partial_update(
cloud_storage_id,
patched_cloud_storage_write_request={
"specific_attributes": f'{cloud_storage["specific_attributes"]}&prefix={default_prefix}'
},
)
assert response.status == HTTPStatus.OK

task_id, _ = self._create_task_with_cloud_data(
request, cloud_storage, use_manifest, server_files, org=org
)

with make_api_client(self._USERNAME) as api_client:
(task, response) = api_client.tasks_api.retrieve(task_id)
assert response.status == HTTPStatus.OK
assert task.size == expected_task_size


@pytest.mark.usefixtures("restore_db_per_function")
class TestPatchTaskLabel:
Expand Down
2 changes: 1 addition & 1 deletion utils/dataset_manifest/core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -622,7 +622,7 @@ def emulate_hierarchical_structure(

search_prefix = prefix
if default_prefix and (len(prefix) < len(default_prefix)):
if '/' in self.prefix[len(prefix):]:
if prefix and '/' in default_prefix[len(prefix):]:
next_layer_and_tail = default_prefix[prefix.find('/') + 1:].split(
"/", maxsplit=1
)
Expand Down

0 comments on commit 1f8d5d3

Please sign in to comment.