Skip to content

Commit

Permalink
update tree comparison for tests to be order independent for children (
Browse files Browse the repository at this point in the history
…#22832)

following the introduction of pytest 8, the order in which children were
listed changed. Since the order is not important, this updates the tests
to make the tests not consider order of children when comparing actual
and expected outcomes of test runs.
  • Loading branch information
eleanorjboyd authored Feb 8, 2024
1 parent 4030717 commit c4c05a6
Show file tree
Hide file tree
Showing 9 changed files with 56 additions and 36 deletions.
22 changes: 11 additions & 11 deletions pythonFiles/tests/pytestadapter/expected_discovery_test_output.py
Original file line number Diff line number Diff line change
Expand Up @@ -317,7 +317,7 @@
# └── test_top_folder.py
# └── test_top_function_t
# └── test_top_function_f
# └── z_nested_folder_one
# └── nested_folder_one
# └── test_bottom_folder.py
# └── test_bottom_function_t
# └── test_bottom_function_f
Expand All @@ -326,14 +326,14 @@
TEST_DATA_PATH / "dual_level_nested_folder" / "test_top_folder.py"
)

test_z_nested_folder_one_path = (
TEST_DATA_PATH / "dual_level_nested_folder" / "z_nested_folder_one"
test_nested_folder_one_path = (
TEST_DATA_PATH / "dual_level_nested_folder" / "nested_folder_one"
)

test_bottom_folder_path = (
TEST_DATA_PATH
/ "dual_level_nested_folder"
/ "z_nested_folder_one"
/ "nested_folder_one"
/ "test_bottom_folder.py"
)

Expand Down Expand Up @@ -392,10 +392,10 @@
],
},
{
"name": "z_nested_folder_one",
"path": os.fspath(test_z_nested_folder_one_path),
"name": "nested_folder_one",
"path": os.fspath(test_nested_folder_one_path),
"type_": "folder",
"id_": os.fspath(test_z_nested_folder_one_path),
"id_": os.fspath(test_nested_folder_one_path),
"children": [
{
"name": "test_bottom_folder.py",
Expand All @@ -412,11 +412,11 @@
),
"type_": "test",
"id_": get_absolute_test_id(
"dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py::test_bottom_function_t",
"dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_t",
test_bottom_folder_path,
),
"runID": get_absolute_test_id(
"dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py::test_bottom_function_t",
"dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_t",
test_bottom_folder_path,
),
},
Expand All @@ -429,11 +429,11 @@
),
"type_": "test",
"id_": get_absolute_test_id(
"dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py::test_bottom_function_f",
"dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_f",
test_bottom_folder_path,
),
"runID": get_absolute_test_id(
"dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py::test_bottom_function_f",
"dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_f",
test_bottom_folder_path,
),
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@
# └── test_top_folder.py
# └── test_top_function_t: success
# └── test_top_function_f: failure
# └── z_nested_folder_one
# └── nested_folder_one
# └── test_bottom_folder.py
# └── test_bottom_function_t: success
# └── test_bottom_function_f: failure
Expand All @@ -318,7 +318,7 @@
dual_level_nested_folder_bottom_path = (
TEST_DATA_PATH
/ "dual_level_nested_folder"
/ "z_nested_folder_one"
/ "nested_folder_one"
/ "test_bottom_folder.py"
)
dual_level_nested_folder_execution_expected_output = {
Expand All @@ -345,11 +345,11 @@
"subtest": None,
},
get_absolute_test_id(
"z_nested_folder_one/test_bottom_folder.py::test_bottom_function_t",
"nested_folder_one/test_bottom_folder.py::test_bottom_function_t",
dual_level_nested_folder_bottom_path,
): {
"test": get_absolute_test_id(
"z_nested_folder_one/test_bottom_folder.py::test_bottom_function_t",
"nested_folder_one/test_bottom_folder.py::test_bottom_function_t",
dual_level_nested_folder_bottom_path,
),
"outcome": "success",
Expand All @@ -358,11 +358,11 @@
"subtest": None,
},
get_absolute_test_id(
"z_nested_folder_one/test_bottom_folder.py::test_bottom_function_f",
"nested_folder_one/test_bottom_folder.py::test_bottom_function_f",
dual_level_nested_folder_bottom_path,
): {
"test": get_absolute_test_id(
"z_nested_folder_one/test_bottom_folder.py::test_bottom_function_f",
"nested_folder_one/test_bottom_folder.py::test_bottom_function_f",
dual_level_nested_folder_bottom_path,
),
"outcome": "failure",
Expand Down Expand Up @@ -479,7 +479,7 @@
dual_level_nested_folder_bottom_path = (
TEST_DATA_PATH
/ "dual_level_nested_folder"
/ "z_nested_folder_one"
/ "nested_folder_one"
/ "test_bottom_folder.py"
)
unittest_folder_add_path = TEST_DATA_PATH / "unittest_folder" / "test_add.py"
Expand Down
21 changes: 14 additions & 7 deletions pythonFiles/tests/pytestadapter/test_discovery.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,18 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import os
import pathlib
import shutil
import sys
from typing import Any, Dict, List, Optional

import pytest

script_dir = pathlib.Path(__file__).parent.parent
sys.path.append(os.fspath(script_dir))

from tests.tree_comparison_helper import is_same_tree

from . import expected_discovery_test_output
from .helpers import TEST_DATA_PATH, runner, runner_with_cwd

Expand Down Expand Up @@ -195,7 +202,7 @@ def test_pytest_collect(file, expected_const):
assert all(item in actual_item.keys() for item in ("status", "cwd", "error"))
assert actual_item.get("status") == "success"
assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH)
assert actual_item.get("tests") == expected_const
assert is_same_tree(actual_item.get("tests"), expected_const)


def test_pytest_root_dir():
Expand All @@ -219,9 +226,9 @@ def test_pytest_root_dir():
assert all(item in actual_item.keys() for item in ("status", "cwd", "error"))
assert actual_item.get("status") == "success"
assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH / "root")
assert (
actual_item.get("tests")
== expected_discovery_test_output.root_with_config_expected_output
assert is_same_tree(
actual_item.get("tests"),
expected_discovery_test_output.root_with_config_expected_output,
)


Expand All @@ -245,7 +252,7 @@ def test_pytest_config_file():
assert all(item in actual_item.keys() for item in ("status", "cwd", "error"))
assert actual_item.get("status") == "success"
assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH / "root")
assert (
actual_item.get("tests")
== expected_discovery_test_output.root_with_config_expected_output
assert is_same_tree(
actual_item.get("tests"),
expected_discovery_test_output.root_with_config_expected_output,
)
4 changes: 2 additions & 2 deletions pythonFiles/tests/pytestadapter/test_execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -193,8 +193,8 @@ def test_bad_id_error_execution():
[
"dual_level_nested_folder/test_top_folder.py::test_top_function_t",
"dual_level_nested_folder/test_top_folder.py::test_top_function_f",
"dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py::test_bottom_function_t",
"dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py::test_bottom_function_f",
"dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_t",
"dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_f",
],
expected_execution_test_output.dual_level_nested_folder_execution_expected_output,
),
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

import pathlib

TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data"


def is_same_tree(tree1, tree2) -> bool:
"""Helper function to test if two test trees are the same.
Expand All @@ -17,8 +13,9 @@ def is_same_tree(tree1, tree2) -> bool:

# Compare child test nodes if they exist, otherwise compare test items.
if "children" in tree1 and "children" in tree2:
children1 = tree1["children"]
children2 = tree2["children"]
# sort children by path before comparing since order doesn't matter of children
children1 = sorted(tree1["children"], key=lambda x: x["path"])
children2 = sorted(tree2["children"], key=lambda x: x["path"])

# Compare test nodes.
if len(children1) != len(children2):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@

import os
from unittestadapter.pvsc_utils import TestNodeTypeEnum
from .helpers import TEST_DATA_PATH
import pathlib

TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data"


skip_unittest_folder_discovery_output = {
"path": os.fspath(TEST_DATA_PATH / "unittest_skip"),
"name": "unittest_skip",
Expand Down
9 changes: 8 additions & 1 deletion pythonFiles/tests/unittestadapter/test_discovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,21 @@

import os
import pathlib
import sys
from typing import List

import pytest
from unittestadapter.discovery import discover_tests
from unittestadapter.pvsc_utils import TestNodeTypeEnum, parse_unittest_args

script_dir = pathlib.Path(__file__).parent.parent
sys.path.append(os.fspath(script_dir))


from . import expected_discovery_test_output
from .helpers import TEST_DATA_PATH, is_same_tree
from tests.tree_comparison_helper import is_same_tree

TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data"


@pytest.mark.parametrize(
Expand Down
9 changes: 8 additions & 1 deletion pythonFiles/tests/unittestadapter/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import os
import pathlib
import sys
import unittest

import pytest
Expand All @@ -15,7 +16,13 @@
get_test_case,
)

from .helpers import TEST_DATA_PATH, is_same_tree
script_dir = pathlib.Path(__file__).parent.parent
sys.path.append(os.fspath(script_dir))

from tests.tree_comparison_helper import is_same_tree


TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data"


@pytest.mark.parametrize(
Expand Down

0 comments on commit c4c05a6

Please sign in to comment.