SERVER-45680: Burn_in_tests should pick up changed files in mongo-ent…

…erprise-modules files

buildscripts/burn_in_tags.py

@@ -20,7 +20,8 @@
 import buildscripts.util.read_config as read_config
 from buildscripts.ciconfig import evergreen
 from buildscripts.burn_in_tests import create_generate_tasks_config, create_tests_by_task, \
-    GenerateConfig, RepeatConfig
+    GenerateConfig, RepeatConfig, DEFAULT_REPO_LOCATIONS
+
 # pylint: enable=wrong-import-position
 
 CONFIG_DIRECTORY = "generated_burn_in_tags_config"
@@ -122,7 +123,7 @@ def _generate_evg_build_variant(shrub_config, build_variant, run_build_variant,
 
 
 # pylint: disable=too-many-arguments
-def _generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data, build_variant_map, repo,
+def _generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data, build_variant_map, repos,
                         evg_conf):
     """
     Generate burn in tests tasks for a given shrub config and group of buildvariants.
@@ -131,11 +132,11 @@ def _generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data, build
     :param shrub_config: Shrub config object that the build variants will be built upon.
     :param expansions_file_data: Config data file to use.
     :param build_variant_map: Map of base buildvariants to their generated buildvariant.
-    :param repo: Git repository.
+    :param repos: Git repositories.
     """
     for build_variant, run_build_variant in build_variant_map.items():
         config_options = _get_config_options(expansions_file_data, build_variant, run_build_variant)
-        tests_by_task = create_tests_by_task(build_variant, repo, evg_conf)
+        tests_by_task = create_tests_by_task(build_variant, repos, evg_conf)
         if tests_by_task:
             _generate_evg_build_variant(shrub_config, build_variant, run_build_variant,
                                         expansions_file_data["build_variant"], evg_conf)
@@ -162,7 +163,7 @@ def _write_to_file(shrub_config):
         file_handle.write(shrub_config.to_json())
 
 
-def main(evergreen_api, repo):
+def main(evergreen_api, repos):
     """Execute Main program."""
 
     parser = argparse.ArgumentParser(description=main.__doc__)
@@ -174,10 +175,11 @@ def main(evergreen_api, repo):
     shrub_config = Configuration()
     evg_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE)
     build_variant_map = _create_evg_build_variant_map(expansions_file_data, evg_conf)
-    _generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data, build_variant_map, repo,
+    _generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data, build_variant_map, repos,
                         evg_conf)
     _write_to_file(shrub_config)
 
 
 if __name__ == '__main__':
-    main(RetryingEvergreenApi.get_api(config_file=EVG_CONFIG_FILE), Repo("."))
+    REPOSITORIES = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)]
+    main(RetryingEvergreenApi.get_api(config_file=EVG_CONFIG_FILE), REPOSITORIES)

buildscripts/burn_in_tests.py

@@ -12,7 +12,7 @@
 
 from math import ceil
 from collections import defaultdict
-from typing import Optional, Set, Tuple, List, Dict
+from typing import Optional, Set, Tuple, List, Dict, Iterable
 
 import click
 import requests
@@ -58,6 +58,7 @@
 AVG_TEST_TIME_MULTIPLIER = 3
 CONFIG_FILE = ".evergreen.yml"
 DEFAULT_PROJECT = "mongodb-mongo-master"
+DEFAULT_REPO_LOCATIONS = [".", "./src/mongo/db/modules/enterprise"]
 REPEAT_SUITES = 2
 EVERGREEN_FILE = "etc/evergreen.yml"
 MAX_TASKS_TO_CREATE = 1000
@@ -234,21 +235,26 @@ def is_file_a_test_file(file_path: str) -> bool:
     return True
 
 
-def find_changed_tests(repo: Repo) -> Set[str]:
+def find_changed_tests(repos: Iterable[Repo]) -> Set[str]:
     """
     Find the changed tests.
 
     Use git to find which files have changed in this patch.
-    TODO: This should be expanded to search for enterprise modules.
     The returned file paths are in normalized form (see os.path.normpath(path)).
 
     :returns: Set of changed tests.
     """
-    changed_files = find_changed_files(repo)
-    LOGGER.debug("Found changed files", files=changed_files)
-    changed_tests = {os.path.normpath(path) for path in changed_files if is_file_a_test_file(path)}
-    LOGGER.debug("Found changed tests", files=changed_tests)
-    return changed_tests
+    all_changed_tests = set()
+    for repo in repos:
+        changed_files = find_changed_files(repo)
+        LOGGER.debug("Found changed files", files=changed_files)
+        changed_tests = {
+            os.path.normpath(path)
+            for path in changed_files if is_file_a_test_file(path)
+        }
+        LOGGER.debug("Found changed tests", files=changed_tests)
+        all_changed_tests.update(changed_tests)
+    return all_changed_tests
 
 
 def find_excludes(selector_file: str) -> Tuple[List, List, List]:
@@ -704,16 +710,17 @@ def create_task_list_for_tests(
     return create_task_list(evg_conf, build_variant, tests_by_executor, exclude_tasks)
 
 
-def create_tests_by_task(build_variant: str, repo: Repo, evg_conf: EvergreenProjectConfig) -> Dict:
+def create_tests_by_task(build_variant: str, repos: Iterable[Repo],
+                         evg_conf: EvergreenProjectConfig) -> Dict:
     """
     Create a list of tests by task.
 
     :param build_variant: Build variant to collect tasks from.
-    :param repo: Git repo being tracked.
+    :param repos: Git repositories being tracked.
     :param evg_conf: Evergreen configuration.
     :return: Tests by task.
     """
-    changed_tests = find_changed_tests(repo)
+    changed_tests = find_changed_tests(repos)
     exclude_suites, exclude_tasks, exclude_tests = find_excludes(SELECTOR_FILE)
     changed_tests = filter_tests(changed_tests, exclude_tests)
 
@@ -812,8 +819,8 @@ def _get_evg_api(evg_api_config: str, local_mode: bool) -> Optional[EvergreenApi
 
 
 def burn_in(repeat_config: RepeatConfig, generate_config: GenerateConfig, resmoke_args: str,
-            generate_tasks_file: str, no_exec: bool, evg_conf: EvergreenProjectConfig, repo: Repo,
-            evg_api: EvergreenApi):
+            generate_tasks_file: str, no_exec: bool, evg_conf: EvergreenProjectConfig,
+            repos: Iterable[Repo], evg_api: EvergreenApi):
     """
     Run burn_in_tests with the given configuration.
 
@@ -823,13 +830,13 @@ def burn_in(repeat_config: RepeatConfig, generate_config: GenerateConfig, resmok
     :param generate_tasks_file: File to write generated config to.
     :param no_exec: Do not execute tests, just discover tests to run.
     :param evg_conf: Evergreen configuration.
-    :param repo: Git repository.
+    :param repos: Git repositories to check.
     :param evg_api: Evergreen API client.
     """
     # Populate the config values in order to use the helpers from resmokelib.suitesconfig.
     resmoke_cmd = _set_resmoke_cmd(repeat_config, list(resmoke_args))
 
-    tests_by_task = create_tests_by_task(generate_config.build_variant, repo, evg_conf)
+    tests_by_task = create_tests_by_task(generate_config.build_variant, repos, evg_conf)
     LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
 
     if generate_tasks_file:
@@ -948,13 +955,15 @@ def main(build_variant, run_build_variant, distro, project, generate_tasks_file,
                                      project=project,
                                      task_id=task_id,
                                      use_multiversion=use_multiversion)  # yapf: disable
-    generate_config.validate(evg_conf, local_mode)
+    if generate_tasks_file:
+        generate_config.validate(evg_conf, local_mode)
 
     evg_api = _get_evg_api(evg_api_config, local_mode)
-    repo = Repo(".")
+
+    repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)]
 
     burn_in(repeat_config, generate_config, resmoke_args, generate_tasks_file, no_exec, evg_conf,
-            repo, evg_api)
+            repos, evg_api)
 
 
 if __name__ == "__main__":

buildscripts/patch_builds/change_data.py

@@ -1,5 +1,6 @@
 """Tools for detecting changes in a commit."""
 from typing import Any, Set
+import os
 
 from git import Repo, DiffIndex
 import structlog
@@ -63,4 +64,9 @@ def find_changed_files(repo: Repo) -> Set[str]:
     untracked_files = set(repo.untracked_files)
     LOGGER.info("untracked files", files=untracked_files, diff="untracked diff")
 
-    return work_tree_files.union(index_files).union(untracked_files)
+    paths = work_tree_files.union(index_files).union(untracked_files)
+
+    return [
+        os.path.relpath(f"{repo.working_dir}/{os.path.normpath(path)}", os.getcwd())
+        for path in paths
+    ]

buildscripts/tests/test_burn_in_tags.py

@@ -106,7 +106,7 @@ def test_generate_evg_tasks_no_tests_changed(self, create_tests_by_task_mock):
         evergreen_api = MagicMock()
         repo = MagicMock()
         under_test._generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data,
-                                       buildvariant_map, repo, evg_conf_mock)
+                                       buildvariant_map, [repo], evg_conf_mock)
 
         self.assertEqual(shrub_config.to_map(), {})
 
@@ -135,7 +135,7 @@ def test_generate_evg_tasks_one_test_changed(self, create_tests_by_task_mock):
             MagicMock(test_file="dir/test2.js", avg_duration_pass=10)
         ]
         under_test._generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data,
-                                       buildvariant_map, repo, evg_conf_mock)
+                                       buildvariant_map, [repo], evg_conf_mock)
 
         generated_config = shrub_config.to_map()
         self.assertEqual(len(generated_config["buildvariants"]), 2)

buildscripts/tests/test_burn_in_tests.py

@@ -78,6 +78,7 @@ def mock_git_diff(change_list):
 def mock_changed_git_files(add_files):
     repo = MagicMock()
     repo.index.diff.return_value = mock_git_diff([mock_a_file(f) for f in add_files])
+    repo.working_dir = "."
     return repo
 
 
@@ -120,17 +121,17 @@ def test_no_tests_run_if_none_changed(self, write_json_mock):
     @patch(ns("_write_json_file"))
     def test_tests_generated_if_a_file_changed(self, write_json_mock):
         """
-        Given a git repository with no changes,
+        Given a git repository with changes,
         When burn_in_tests is run,
-        Then no tests are discovered to run.
+        Then tests are discovered to run.
         """
         # Note: this test is using actual tests and suites. So changes to those suites could
         # introduce failures and require this test to be updated.
         # You can see the test file it is using below. This test is used in the 'auth' and
         # 'auth_audit' test suites. It needs to be in at least one of those for the test to pass.
         _config.NAMED_SUITES = None
         variant = "enterprise-rhel-62-64-bit"
-        repo = mock_changed_git_files(["jstests/auth/auth1.js"])
+        repos = [mock_changed_git_files(["jstests/auth/auth1.js"])]
         repeat_config = under_test.RepeatConfig()
         gen_config = under_test.GenerateConfig(
             variant,
@@ -139,7 +140,7 @@ def test_tests_generated_if_a_file_changed(self, write_json_mock):
         )  # yapf: disable
         evg_config = get_evergreen_config("etc/evergreen.yml")
 
-        under_test.burn_in(repeat_config, gen_config, "", "testfile.json", False, evg_config, repo,
+        under_test.burn_in(repeat_config, gen_config, "", "testfile.json", False, evg_config, repos,
                            None)
 
         write_json_mock.assert_called_once()
@@ -1114,7 +1115,7 @@ def test_non_js_files_filtered(self, is_file_mock, changed_files_mock):
         changed_files_mock.return_value = set(file_list)
         is_file_mock.return_value = True
 
-        found_tests = under_test.find_changed_tests(repo_mock)
+        found_tests = under_test.find_changed_tests([repo_mock])
 
         self.assertIn(file_list[0], found_tests)
         self.assertIn(file_list[2], found_tests)
@@ -1132,7 +1133,7 @@ def test_missing_files_filtered(self, is_file_mock, changed_files_mock):
         changed_files_mock.return_value = set(file_list)
         is_file_mock.return_value = False
 
-        found_tests = under_test.find_changed_tests(repo_mock)
+        found_tests = under_test.find_changed_tests([repo_mock])
 
         self.assertEqual(0, len(found_tests))
 
@@ -1148,7 +1149,7 @@ def test_non_jstests_files_filtered(self, is_file_mock, changed_files_mock):
         changed_files_mock.return_value = set(file_list)
         is_file_mock.return_value = True
 
-        found_tests = under_test.find_changed_tests(repo_mock)
+        found_tests = under_test.find_changed_tests([repo_mock])
 
         self.assertIn(file_list[0], found_tests)
         self.assertIn(file_list[2], found_tests)