From e035463e88a3a9124cf3fa7e559066975591a6cc Mon Sep 17 00:00:00 2001 From: Luis Padron Date: Wed, 19 Jun 2024 16:19:58 -0400 Subject: [PATCH] Remove emit symbol graph API Removes the now old `emit_symbol_grah` feature in favor of the new `swift_symbol_graph_extract` rule which can be used to collec the symbol graph of Swift targets. Depends on: https://github.com/bazelbuild/rules_swift/pull/772 Rebase --- MODULE.bazel | 6 + doc/api.md | 37 +++- swift/BUILD | 2 + swift/internal/BUILD | 41 ++++ swift/internal/actions.bzl | 5 + swift/internal/compiling.bzl | 94 +++------ swift/internal/derived_files.bzl | 25 ++- swift/internal/feature_names.bzl | 3 - swift/internal/linking.bzl | 7 +- swift/internal/output_groups.bzl | 5 - swift/internal/providers.bzl | 33 ++- swift/internal/swift_binary_test_rules.bzl | 104 +++++++++- swift/internal/swift_common.bzl | 2 + swift/internal/swift_extract_symbol_graph.bzl | 125 ++++++++++++ swift/internal/swift_symbol_graph_aspect.bzl | 188 ++++++++++++++++++ swift/internal/swift_toolchain.bzl | 62 +++++- swift/internal/symbol_graph_extracting.bzl | 144 ++++++++++++++ swift/internal/xcode_swift_toolchain.bzl | 18 +- swift/repositories.bzl | 18 ++ swift/swift.bzl | 12 ++ test/BUILD | 3 + test/fixtures/symbol_graphs/BUILD | 52 +++++ .../symbol_graphs/ImportingModule.swift | 4 + test/fixtures/symbol_graphs/SomeModule.swift | 19 ++ test/rules/directory_test.bzl | 103 ++++++++++ test/split_derived_files_tests.bzl | 48 ----- test/symbol_graphs_tests.bzl | 76 +++++++ .../BUILD.overlay | 17 ++ .../BUILD.overlay | 10 + tools/test_discoverer/BUILD | 17 ++ tools/test_discoverer/DiscoveredTests.swift | 55 +++++ tools/test_discoverer/SymbolCollector.swift | 186 +++++++++++++++++ .../test_discoverer/SymbolKitExtensions.swift | 54 +++++ tools/test_discoverer/TestDiscoverer.swift | 94 +++++++++ tools/test_discoverer/TestPrinter.swift | 164 +++++++++++++++ 35 files changed, 1670 insertions(+), 163 deletions(-) create mode 100644 swift/internal/swift_extract_symbol_graph.bzl create mode 100644 swift/internal/swift_symbol_graph_aspect.bzl create mode 100644 swift/internal/symbol_graph_extracting.bzl create mode 100644 test/fixtures/symbol_graphs/BUILD create mode 100644 test/fixtures/symbol_graphs/ImportingModule.swift create mode 100644 test/fixtures/symbol_graphs/SomeModule.swift create mode 100644 test/rules/directory_test.bzl create mode 100644 test/symbol_graphs_tests.bzl create mode 100644 third_party/com_github_apple_swift_argument_parser/BUILD.overlay create mode 100644 third_party/com_github_apple_swift_docc_symbolkit/BUILD.overlay create mode 100644 tools/test_discoverer/BUILD create mode 100644 tools/test_discoverer/DiscoveredTests.swift create mode 100644 tools/test_discoverer/SymbolCollector.swift create mode 100644 tools/test_discoverer/SymbolKitExtensions.swift create mode 100644 tools/test_discoverer/TestDiscoverer.swift create mode 100644 tools/test_discoverer/TestPrinter.swift diff --git a/MODULE.bazel b/MODULE.bazel index 83c368a71..a9f408f73 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -14,12 +14,18 @@ bazel_dep(name = "platforms", version = "0.0.9") bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf") bazel_dep(name = "rules_proto", version = "5.3.0-21.7") bazel_dep(name = "nlohmann_json", version = "3.6.1", repo_name = "com_github_nlohmann_json") +bazel_dep( + name = "swift_argument_parser", + version = "1.3.0", + repo_name = "com_github_apple_swift_argument_parser", +) non_module_deps = use_extension("//swift:extensions.bzl", "non_module_deps") use_repo( non_module_deps, "build_bazel_rules_swift_index_import", "build_bazel_rules_swift_local_config", + "com_github_apple_swift_docc_symbolkit", "com_github_apple_swift_log", "com_github_apple_swift_nio", "com_github_apple_swift_nio_extras", diff --git a/doc/api.md b/doc/api.md index be11307ed..7a36b2bd0 100644 --- a/doc/api.md +++ b/doc/api.md @@ -140,10 +140,6 @@ A tuple containing three elements: the index store data generated by the compiler if the `"swift.index_while_building"` feature is enabled, otherwise this will be `None`. - * `symbol_graph`: A `File` representing the directory that - contains the symbol graph data generated by the compiler if the - `"swift.emit_symbol_graph"` feature is enabled, otherwise this - will be `None`. * `const_values_files`: A list of `File`s that contains JSON representations of constant values extracted from the source files, if requested via a direct dependency. @@ -471,7 +467,7 @@ A provider whose type/layout is an implementation detail and should not 
 swift_common.create_swift_module(swiftdoc, swiftmodule, ast_files, defines, indexstore, plugins,
                                  swiftsourceinfo, swiftinterface, private_swiftinterface,
-                                 const_protocols_to_gather, symbol_graph)
+                                 const_protocols_to_gather)
Creates a value representing a Swift module use as a Swift dependency. @@ -491,12 +487,11 @@ Creates a value representing a Swift module use as a Swift dependency. | swiftinterface | The `.swiftinterface` file emitted by the compiler for this module. May be `None` if no module interface file was emitted. | `None` | | private_swiftinterface | The `.private.swiftinterface` file emitted by the compiler for this module. May be `None` if no private module interface file was emitted. | `None` | | const_protocols_to_gather | A list of protocol names from which constant values should be extracted from source code that takes this module as a *direct* dependency. | `[]` | -| symbol_graph | A `File` representing the directory that contains the symbol graph data generated by the compiler if the `"swift.emit_symbol_graph"` feature is enabled, otherwise this will be `None`. | `None` | **RETURNS** A `struct` containing the `ast_files`, `defines`, `indexstore, - `swiftdoc`, `swiftmodule`, `swiftinterface`, and `symbol_graph` fields + `swiftdoc`, `swiftmodule`, and `swiftinterface` fields provided as arguments. @@ -538,6 +533,34 @@ This mapping is intended to be fairly predictable, but not reversible. The module name derived from the label. + + +## swift_common.extract_symbol_graph + +
+swift_common.extract_symbol_graph(actions, compilation_contexts, feature_configuration,
+                                  include_dev_srch_paths, minimum_access_level, module_name,
+                                  output_dir, swift_infos, swift_toolchain)
+
+ +Extracts the symbol graph from a Swift module. + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| actions | The object used to register actions. | none | +| compilation_contexts | A list of `CcCompilationContext`s that represent C/Objective-C requirements of the target being compiled, such as Swift-compatible preprocessor defines, header search paths, and so forth. These are typically retrieved from the `CcInfo` providers of a target's dependencies. | none | +| feature_configuration | The Swift feature configuration. | none | +| include_dev_srch_paths | A `bool` that indicates whether the developer framework search paths will be added to the compilation command. | none | +| minimum_access_level | The minimum access level of the declarations that should be extracted into the symbol graphs. The default value is `None`, which means the Swift compiler's default behavior should be used (at the time of this writing, the default behavior is "public"). | `None` | +| module_name | The name of the module whose symbol graph should be extracted. | none | +| output_dir | A directory-type `File` into which `.symbols.json` files representing the module's symbol graph will be extracted. If extraction is successful, this directory will contain a file named `${MODULE_NAME}.symbols.json`. Optionally, if the module contains extensions to types in other modules, then there will also be files named `${MODULE_NAME}@${EXTENDED_MODULE}.symbols.json`. | none | +| swift_infos | A list of `SwiftInfo` providers from dependencies of the target being compiled. This should include both propagated and non-propagated (implementation-only) dependencies. | none | +| swift_toolchain | The `SwiftToolchainInfo` provider of the toolchain. | none | + + ## swift_common.is_enabled diff --git a/swift/BUILD b/swift/BUILD index a93abb3fc..150420b08 100644 --- a/swift/BUILD +++ b/swift/BUILD @@ -54,6 +54,7 @@ bzl_library( "//swift/internal:providers", "//swift/internal:swift_binary_test_rules", "//swift/internal:swift_common", + "//swift/internal:swift_extract_symbol_graph", "//swift/internal:swift_feature_allowlist", "//swift/internal:swift_import", "//swift/internal:swift_interop_hint", @@ -61,6 +62,7 @@ bzl_library( "//swift/internal:swift_library_group", "//swift/internal:swift_module_alias", "//swift/internal:swift_package_configuration", + "//swift/internal:swift_symbol_graph_aspect", "//swift/internal:swift_usage_aspect", ], ) diff --git a/swift/internal/BUILD b/swift/internal/BUILD index 4c2f6f0fe..cba799bdb 100644 --- a/swift/internal/BUILD +++ b/swift/internal/BUILD @@ -192,6 +192,45 @@ bzl_library( ":linking", ":providers", ":swift_clang_module_aspect", + ":symbol_graph_extracting", + ], +) + +bzl_library( + name = "swift_symbol_graph_aspect", + srcs = ["swift_symbol_graph_aspect.bzl"], + visibility = ["//swift:__subpackages__"], + deps = [ + ":attrs", + ":derived_files", + ":features", + ":providers", + ":symbol_graph_extracting", + ":utils", + "@bazel_skylib//lib:dicts", + ], +) + +bzl_library( + name = "swift_extract_symbol_graph", + srcs = ["swift_extract_symbol_graph.bzl"], + visibility = ["//swift:__subpackages__"], + deps = [ + ":derived_files", + ":providers", + ":swift_symbol_graph_aspect", + ], +) + +bzl_library( + name = "symbol_graph_extracting", + srcs = ["symbol_graph_extracting.bzl"], + visibility = ["//swift:__subpackages__"], + deps = [ + ":actions", + ":providers", + ":toolchain_config", + ":utils", ], ) @@ -336,6 +375,7 @@ bzl_library( ":feature_names", ":features", ":providers", + ":symbol_graph_extracting", ":toolchain_config", ":utils", "@bazel_skylib//lib:collections", @@ -391,6 +431,7 @@ bzl_library( ":output_groups", ":providers", ":swift_common", + ":swift_symbol_graph_aspect", ":utils", "@bazel_skylib//lib:dicts", ], diff --git a/swift/internal/actions.bzl b/swift/internal/actions.bzl index e9098b565..831eb1bc9 100644 --- a/swift/internal/actions.bzl +++ b/swift/internal/actions.bzl @@ -45,6 +45,11 @@ swift_action_names = struct( # headers, emitting a `.pcm` file. PRECOMPILE_C_MODULE = "SwiftPrecompileCModule", + # Extracts a JSON-formatted symbol graph from a module, which can be used as + # an input to documentation generating tools like `docc` or analyzed with + # other tooling. + SYMBOL_GRAPH_EXTRACT = "SwiftSymbolGraphExtract", + # Produces files that are usually fallout of the compilation such as # .swiftmodule, -Swift.h and more. DERIVE_FILES = "SwiftDeriveFiles", diff --git a/swift/internal/compiling.bzl b/swift/internal/compiling.bzl index fee1d4a32..32fbd8a88 100644 --- a/swift/internal/compiling.bzl +++ b/swift/internal/compiling.bzl @@ -53,7 +53,6 @@ load( "SWIFT_FEATURE_EMIT_SWIFTDOC", "SWIFT_FEATURE_EMIT_SWIFTINTERFACE", "SWIFT_FEATURE_EMIT_SWIFTSOURCEINFO", - "SWIFT_FEATURE_EMIT_SYMBOL_GRAPH", "SWIFT_FEATURE_ENABLE_BATCH_MODE", "SWIFT_FEATURE_ENABLE_LIBRARY_EVOLUTION", "SWIFT_FEATURE_ENABLE_SKIP_FUNCTION_BODIES", @@ -176,8 +175,8 @@ def compile_action_configs( swift_action_names.COMPILE, swift_action_names.COMPILE_MODULE_INTERFACE, swift_action_names.DERIVE_FILES, - swift_action_names.PRECOMPILE_C_MODULE, swift_action_names.DUMP_AST, + swift_action_names.PRECOMPILE_C_MODULE, ], configurators = [ swift_toolchain_config.add_arg("-disallow-use-new-driver"), @@ -193,8 +192,8 @@ def compile_action_configs( swift_action_names.COMPILE, swift_action_names.COMPILE_MODULE_INTERFACE, swift_action_names.DERIVE_FILES, - swift_action_names.PRECOMPILE_C_MODULE, swift_action_names.DUMP_AST, + swift_action_names.PRECOMPILE_C_MODULE, ], configurators = [ swift_toolchain_config.add_arg( @@ -212,8 +211,8 @@ def compile_action_configs( swift_action_names.COMPILE, swift_action_names.COMPILE_MODULE_INTERFACE, swift_action_names.DERIVE_FILES, - swift_action_names.PRECOMPILE_C_MODULE, swift_action_names.DUMP_AST, + swift_action_names.PRECOMPILE_C_MODULE, ], configurators = [ swift_toolchain_config.add_arg( @@ -232,8 +231,8 @@ def compile_action_configs( swift_action_names.COMPILE, swift_action_names.COMPILE_MODULE_INTERFACE, swift_action_names.DERIVE_FILES, - swift_action_names.PRECOMPILE_C_MODULE, swift_action_names.DUMP_AST, + swift_action_names.PRECOMPILE_C_MODULE, ], configurators = [ swift_toolchain_config.add_arg( @@ -607,6 +606,7 @@ def compile_action_configs( swift_toolchain_config.action_config( actions = [ swift_action_names.PRECOMPILE_C_MODULE, + swift_action_names.SYMBOL_GRAPH_EXTRACT, ], configurators = [_pcm_developer_framework_paths_configurator], ), @@ -777,8 +777,9 @@ def compile_action_configs( swift_action_names.COMPILE, swift_action_names.COMPILE_MODULE_INTERFACE, swift_action_names.DERIVE_FILES, - swift_action_names.PRECOMPILE_C_MODULE, swift_action_names.DUMP_AST, + swift_action_names.PRECOMPILE_C_MODULE, + swift_action_names.SYMBOL_GRAPH_EXTRACT, ], configurators = [ swift_toolchain_config.add_arg("-Xcc", "-Xclang"), @@ -847,26 +848,6 @@ def compile_action_configs( SWIFT_FEATURE_USE_PCH_OUTPUT_DIR, ], ), - swift_toolchain_config.action_config( - actions = [ - swift_action_names.COMPILE, - ], - configurators = [_emit_symbol_graph_configurator], - features = [ - SWIFT_FEATURE_EMIT_SYMBOL_GRAPH, - ], - not_features = [SWIFT_FEATURE_SPLIT_DERIVED_FILES_GENERATION], - ), - swift_toolchain_config.action_config( - actions = [ - swift_action_names.DERIVE_FILES, - ], - configurators = [_emit_symbol_graph_configurator], - features = [ - SWIFT_FEATURE_EMIT_SYMBOL_GRAPH, - SWIFT_FEATURE_SPLIT_DERIVED_FILES_GENERATION, - ], - ), # When using C modules, disable the implicit search for module map files # because all of them, including system dependencies, will be provided @@ -876,8 +857,9 @@ def compile_action_configs( swift_action_names.COMPILE, swift_action_names.COMPILE_MODULE_INTERFACE, swift_action_names.DERIVE_FILES, - swift_action_names.PRECOMPILE_C_MODULE, swift_action_names.DUMP_AST, + swift_action_names.PRECOMPILE_C_MODULE, + swift_action_names.SYMBOL_GRAPH_EXTRACT, ], configurators = [ swift_toolchain_config.add_arg( @@ -893,7 +875,7 @@ def compile_action_configs( swift_action_names.COMPILE, swift_action_names.COMPILE_MODULE_INTERFACE, swift_action_names.PRECOMPILE_C_MODULE, - # swift_action_names.SYMBOL_GRAPH_EXTRACT, # TODO: Enable once supported + swift_action_names.SYMBOL_GRAPH_EXTRACT, ], configurators = [ swift_toolchain_config.add_arg( @@ -963,6 +945,7 @@ def compile_action_configs( swift_action_names.COMPILE_MODULE_INTERFACE, swift_action_names.DERIVE_FILES, swift_action_names.DUMP_AST, + swift_action_names.SYMBOL_GRAPH_EXTRACT, ], configurators = [_dependencies_swiftmodules_configurator], not_features = [ @@ -1043,6 +1026,7 @@ def compile_action_configs( swift_action_names.COMPILE_MODULE_INTERFACE, swift_action_names.DERIVE_FILES, swift_action_names.DUMP_AST, + swift_action_names.SYMBOL_GRAPH_EXTRACT, ], configurators = [ lambda prereqs, args: _framework_search_paths_configurator( @@ -1072,8 +1056,9 @@ def compile_action_configs( swift_action_names.COMPILE, swift_action_names.COMPILE_MODULE_INTERFACE, swift_action_names.DERIVE_FILES, - swift_action_names.PRECOMPILE_C_MODULE, swift_action_names.DUMP_AST, + swift_action_names.PRECOMPILE_C_MODULE, + swift_action_names.SYMBOL_GRAPH_EXTRACT, ], configurators = [ _clang_search_paths_configurator, @@ -1088,8 +1073,9 @@ def compile_action_configs( swift_action_names.COMPILE, swift_action_names.COMPILE_MODULE_INTERFACE, swift_action_names.DERIVE_FILES, - swift_action_names.PRECOMPILE_C_MODULE, swift_action_names.DUMP_AST, + swift_action_names.PRECOMPILE_C_MODULE, + swift_action_names.SYMBOL_GRAPH_EXTRACT, ], configurators = [_dependencies_clang_modules_configurator], features = [SWIFT_FEATURE_USE_C_MODULES], @@ -1099,8 +1085,9 @@ def compile_action_configs( swift_action_names.COMPILE, swift_action_names.COMPILE_MODULE_INTERFACE, swift_action_names.DERIVE_FILES, - swift_action_names.PRECOMPILE_C_MODULE, swift_action_names.DUMP_AST, + swift_action_names.PRECOMPILE_C_MODULE, + swift_action_names.SYMBOL_GRAPH_EXTRACT, ], configurators = [_dependencies_clang_modulemaps_configurator], not_features = [SWIFT_FEATURE_USE_C_MODULES], @@ -1198,8 +1185,9 @@ def compile_action_configs( actions = [ swift_action_names.COMPILE, swift_action_names.DERIVE_FILES, - swift_action_names.PRECOMPILE_C_MODULE, swift_action_names.DUMP_AST, + swift_action_names.PRECOMPILE_C_MODULE, + swift_action_names.SYMBOL_GRAPH_EXTRACT, ], configurators = [_module_name_configurator], ), @@ -1208,8 +1196,8 @@ def compile_action_configs( actions = [ swift_action_names.COMPILE, swift_action_names.DERIVE_FILES, - swift_action_names.PRECOMPILE_C_MODULE, swift_action_names.DUMP_AST, + swift_action_names.PRECOMPILE_C_MODULE, ], configurators = [_package_name_configurator], ), @@ -1294,8 +1282,8 @@ def compile_action_configs( swift_action_names.COMPILE, swift_action_names.COMPILE_MODULE_INTERFACE, swift_action_names.DERIVE_FILES, - swift_action_names.PRECOMPILE_C_MODULE, swift_action_names.DUMP_AST, + swift_action_names.PRECOMPILE_C_MODULE, ], configurators = [ lambda _, args: args.add_all( @@ -1329,8 +1317,8 @@ def compile_action_configs( swift_action_names.COMPILE, swift_action_names.COMPILE_MODULE_INTERFACE, swift_action_names.DERIVE_FILES, - swift_action_names.PRECOMPILE_C_MODULE, swift_action_names.DUMP_AST, + swift_action_names.PRECOMPILE_C_MODULE, ], configurators = [_source_files_configurator], ), @@ -1990,21 +1978,6 @@ def _pch_output_dir_configurator(prerequisites, args): paths.join(prerequisites.bin_dir.path, "_pch_output_dir"), ) -def _emit_symbol_graph_configurator(prerequisites, args): - """Adds flags for `-emit-symbol-graph` configuration to the command line. - - This is a directory to persist symbol graph files that can be used by - tools such as DocC or jazzy to generate documentation. - """ - args.add( - "-Xfrontend", - "-emit-symbol-graph", - ) - args.add( - "-emit-symbol-graph-dir", - prerequisites.symbol_graph_directory.path, - ) - def _global_index_store_configurator(prerequisites, args): """Adds flags for index-store generation to the command line.""" out_dir = prerequisites.indexstore_directory.dirname.split("/")[0] @@ -2429,10 +2402,6 @@ def compile( the index store data generated by the compiler if the `"swift.index_while_building"` feature is enabled, otherwise this will be `None`. - * `symbol_graph`: A `File` representing the directory that - contains the symbol graph data generated by the compiler if the - `"swift.emit_symbol_graph"` feature is enabled, otherwise this - will be `None`. * `const_values_files`: A list of `File`s that contains JSON representations of constant values extracted from the source files, if requested via a direct dependency. @@ -2501,7 +2470,6 @@ def compile( # for that action). This guarantees some predictability. compile_outputs.swiftmodule_file, compile_outputs.generated_header_file, - compile_outputs.symbol_graph_directory, ]) + other_outputs if include_swiftdoc: all_derived_outputs.append(compile_outputs.swiftdoc_file) @@ -2519,7 +2487,6 @@ def compile( compile_outputs.generated_header_file, compile_outputs.indexstore_directory, compile_outputs.macro_expansion_directory, - compile_outputs.symbol_graph_directory, ]) + compile_outputs.object_files + compile_outputs.const_values_files + other_outputs if include_swiftdoc: all_compile_outputs.append(compile_outputs.swiftdoc_file) @@ -2786,7 +2753,6 @@ to use swift_common.compile(include_dev_srch_paths = ...) instead.\ swiftmodule = compile_outputs.swiftmodule_file, swiftsourceinfo = compile_outputs.swiftsourceinfo_file, const_protocols_to_gather = compile_outputs.const_values_files, - symbol_graph = compile_outputs.symbol_graph_directory, ), ) @@ -2799,7 +2765,6 @@ to use swift_common.compile(include_dev_srch_paths = ...) instead.\ ast_files = compile_outputs.ast_files, indexstore = compile_outputs.indexstore_directory, macro_expansion_directory = compile_outputs.macro_expansion_directory, - symbol_graph = compile_outputs.symbol_graph_directory, const_values_files = compile_outputs.const_values_files, ) @@ -3290,18 +3255,6 @@ def _declare_compile_outputs( else: indexstore_directory = None - emit_symbol_graph = is_feature_enabled( - feature_configuration = feature_configuration, - feature_name = SWIFT_FEATURE_EMIT_SYMBOL_GRAPH, - ) - if (emit_symbol_graph): - symbol_graph_directory = derived_files.symbol_graph_directory( - actions = actions, - target_name = target_name, - ) - else: - symbol_graph_directory = None - if is_feature_enabled( feature_configuration = feature_configuration, feature_name = SWIFT_FEATURE__SUPPORTS_MACROS, @@ -3323,7 +3276,6 @@ def _declare_compile_outputs( indexstore_directory = indexstore_directory, macro_expansion_directory = macro_expansion_directory, private_swiftinterface_file = private_swiftinterface_file, - symbol_graph_directory = symbol_graph_directory, object_files = object_files, output_file_map = output_file_map, derived_files_output_file_map = derived_files_output_file_map, diff --git a/swift/internal/derived_files.bzl b/swift/internal/derived_files.bzl index 60a530c5b..40269871a 100644 --- a/swift/internal/derived_files.bzl +++ b/swift/internal/derived_files.bzl @@ -68,18 +68,6 @@ def _indexstore_directory(actions, target_name): """ return actions.declare_directory("{}.indexstore".format(target_name)) -def _symbol_graph_directory(actions, target_name): - """Declares a directory in which the compiler's symbol graph will be written. - - Args: - actions: The context's actions object. - target_name: The name of the target being built. - - Returns: - The declared `File`. - """ - return actions.declare_directory("{}.symbolgraph".format(target_name)) - def _intermediate_bc_file(actions, target_name, src): """Declares a file for an intermediate llvm bc file during compilation. @@ -373,6 +361,18 @@ def _swiftsourceinfo(actions, add_target_name_to_output_path, target_name, modul "{}.swiftsourceinfo".format(module_name), ) +def _symbol_graph_directory(actions, target_name): + """Declares a directory for symbol graphs extracted from a Swift module. + + Args: + actions: The context's actions object. + target_name: The name of the target being built. + + Returns: + The declared `File`. + """ + return actions.declare_directory("{}.symbolgraphs".format(target_name)) + def _vfsoverlay(actions, target_name): """Declares a file for the VFS overlay for a compilation action. @@ -470,7 +470,6 @@ derived_files = struct( swiftinterface = _swiftinterface, swiftmodule = _swiftmodule, swiftsourceinfo = _swiftsourceinfo, - symbol_graph_directory = _symbol_graph_directory, vfsoverlay = _vfsoverlay, whole_module_object_file = _whole_module_object_file, swift_const_values_file = _swift_const_values_file, diff --git a/swift/internal/feature_names.bzl b/swift/internal/feature_names.bzl index d9a6a9e27..6019d7c23 100644 --- a/swift/internal/feature_names.bzl +++ b/swift/internal/feature_names.bzl @@ -107,9 +107,6 @@ SWIFT_FEATURE_CODEVIEW_DEBUG_INFO = "swift.codeview_debug_info" # https://docs.google.com/document/d/1cH2sTpgSnJZCkZtJl1aY-rzy4uGPcrI-6RrUpdATO2Q/ SWIFT_FEATURE_INDEX_WHILE_BUILDING = "swift.index_while_building" -# If enabled, the compilation action for a target will produce a symbol graph. -SWIFT_FEATURE_EMIT_SYMBOL_GRAPH = "swift.emit_symbol_graph" - # If enabled the compilation action will not produce indexes for system modules. SWIFT_FEATURE_DISABLE_SYSTEM_INDEX = "swift.disable_system_index" diff --git a/swift/internal/linking.bzl b/swift/internal/linking.bzl index fe1b9ca02..f03a57539 100644 --- a/swift/internal/linking.bzl +++ b/swift/internal/linking.bzl @@ -50,11 +50,14 @@ _OBJC_PROVIDER_LINKING = hasattr(apple_common.new_objc_provider(), "linkopt") def binary_rule_attrs( *, + additional_deps_aspects = [], additional_deps_providers = [], stamp_default): """Returns attributes common to both `swift_binary` and `swift_test`. Args: + additional_deps_aspects: A list of additional aspects that should be + applied to the `deps` attribute of the rule. additional_deps_providers: A list of lists representing additional providers that should be allowed by the `deps` attribute of the rule. @@ -65,7 +68,9 @@ def binary_rule_attrs( """ return dicts.add( swift_compilation_attrs( - additional_deps_aspects = [swift_clang_module_aspect], + additional_deps_aspects = [ + swift_clang_module_aspect, + ] + additional_deps_aspects, additional_deps_providers = additional_deps_providers, requires_srcs = False, ), diff --git a/swift/internal/output_groups.bzl b/swift/internal/output_groups.bzl index 241197577..26ef79a15 100644 --- a/swift/internal/output_groups.bzl +++ b/swift/internal/output_groups.bzl @@ -35,7 +35,6 @@ def supplemental_compilation_output_groups(*supplemental_outputs): const_values_files = [] indexstore_files = [] macro_expansions_files = [] - symbol_graph_files = [] for outputs in supplemental_outputs: if outputs.ast_files: @@ -46,8 +45,6 @@ def supplemental_compilation_output_groups(*supplemental_outputs): indexstore_files.append(outputs.indexstore) if outputs.macro_expansion_directory: macro_expansions_files.append(outputs.macro_expansion_directory) - if outputs.symbol_graph: - symbol_graph_files.append(outputs.symbol_graph) output_groups = {} if ast_files: @@ -58,6 +55,4 @@ def supplemental_compilation_output_groups(*supplemental_outputs): output_groups["swift_index_store"] = depset(indexstore_files) if macro_expansions_files: output_groups["macro_expansions"] = depset(macro_expansions_files) - if symbol_graph_files: - output_groups["swift_symbol_graph"] = depset(symbol_graph_files) return output_groups diff --git a/swift/internal/providers.bzl b/swift/internal/providers.bzl index dc3b7b613..b07c7e412 100644 --- a/swift/internal/providers.bzl +++ b/swift/internal/providers.bzl @@ -169,6 +169,29 @@ of the `swift_proto_library` target. }, ) +SwiftSymbolGraphInfo = provider( + doc = "Propagates extracted symbol graph files from Swift modules.", + fields = { + "direct_symbol_graphs": """\ +`List` of `struct`s representing the symbol graphs extracted from the target +that propagated this provider. This list will be empty if propagated by a +non-Swift target (although its `transitive_symbol_graphs` may be non-empty if it +has Swift dependencies). + +Each `struct` has the following fields: + +* `module_name`: A string denoting the name of the Swift module. +* `symbol_graph_dir`: A directory-type `File` containing one or more + `.symbols.json` files representing the symbol graph(s) for the module. +""", + "transitive_symbol_graphs": """\ +`Depset` of `struct`s representing the symbol graphs extracted from the target +that propagated this provider and all of its Swift dependencies. Each `struct` +has the same fields as documented in `direct_symbol_graphs`. +""", + }, +) + SwiftToolchainInfo = provider( doc = """ Propagates information about a Swift toolchain to compilation and linking rules @@ -439,8 +462,7 @@ def create_swift_module( swiftsourceinfo = None, swiftinterface = None, private_swiftinterface = None, - const_protocols_to_gather = [], - symbol_graph = None): + const_protocols_to_gather = []): """Creates a value representing a Swift module use as a Swift dependency. Args: @@ -467,14 +489,10 @@ def create_swift_module( const_protocols_to_gather: A list of protocol names from which constant values should be extracted from source code that takes this module as a *direct* dependency. - symbol_graph: A `File` representing the directory that contains the - symbol graph data generated by the compiler if the - `"swift.emit_symbol_graph"` feature is enabled, otherwise this will - be `None`. Returns: A `struct` containing the `ast_files`, `defines`, `indexstore, - `swiftdoc`, `swiftmodule`, `swiftinterface`, and `symbol_graph` fields + `swiftdoc`, `swiftmodule`, and `swiftinterface` fields provided as arguments. """ return struct( @@ -488,7 +506,6 @@ def create_swift_module( swiftmodule = swiftmodule, swiftsourceinfo = swiftsourceinfo, const_protocols_to_gather = tuple(const_protocols_to_gather), - symbol_graph = symbol_graph, ) def create_swift_info( diff --git a/swift/internal/swift_binary_test_rules.bzl b/swift/internal/swift_binary_test_rules.bzl index 6011a74cd..6b0459440 100644 --- a/swift/internal/swift_binary_test_rules.bzl +++ b/swift/internal/swift_binary_test_rules.bzl @@ -33,9 +33,11 @@ load( ":providers.bzl", "SwiftCompilerPluginInfo", "SwiftInfo", + "SwiftSymbolGraphInfo", "SwiftToolchainInfo", ) load(":swift_common.bzl", "swift_common") +load(":swift_symbol_graph_aspect.bzl", "test_discovery_symbol_graph_aspect") load( ":utils.bzl", "expand_locations", @@ -66,6 +68,7 @@ def _swift_linking_rule_impl( ctx, binary_path, feature_configuration, + srcs, swift_toolchain, additional_linking_contexts = [], extra_link_deps = [], @@ -78,6 +81,7 @@ def _swift_linking_rule_impl( binary_path: The path to output the linked binary to. feature_configuration: A feature configuration obtained from `swift_common.configure_features`. + srcs: The Swift sources to be compiled into the binary. swift_toolchain: The `SwiftToolchainInfo` provider of the toolchain being used to build the target. additional_linking_contexts: Additional linking contexts that provide @@ -103,7 +107,6 @@ def _swift_linking_rule_impl( cc_feature_configuration = swift_common.cc_feature_configuration( feature_configuration = feature_configuration, ) - srcs = ctx.files.srcs user_link_flags = list(linkopts) # If the rule has sources, compile those first and collect the outputs to @@ -266,6 +269,7 @@ def _swift_binary_impl(ctx): ctx, binary_path = derived_files.path(ctx, add_target_name_to_output_path, ctx.label.name), feature_configuration = feature_configuration, + srcs = ctx.files.srcs, swift_toolchain = swift_toolchain, ) @@ -280,6 +284,79 @@ def _swift_binary_impl(ctx): ), ] +def _generate_test_discovery_srcs(*, actions, deps, name, test_discoverer): + """Generate Swift sources to run discovered XCTest-style tests. + + Args: + actions: The context's actions object. + deps: The list of direct dependencies of the test target. + name: The name of the target being built, which will be used to derive + the basename of the directory containing the generated files. + test_discoverer: The executable `File` representing the test discoverer + tool that will be spawned to generate the test runner sources. + + Returns: + A list of `File`s representing generated `.swift` source files that + should be compiled as part of the test target. + """ + inputs = [] + outputs = [] + args = actions.args() + + # For each direct dependency/module that we have a symbol graph for (i.e., + # every testonly dependency), declare a `.swift` source file where the + # discovery tool will generate an extension that lists the test entries for + # the classes/methods found in that module. + for dep in deps: + if SwiftSymbolGraphInfo not in dep: + continue + + symbol_graph_info = dep[SwiftSymbolGraphInfo] + + for symbol_graph in symbol_graph_info.direct_symbol_graphs: + output_file = actions.declare_file( + "{target}_test_discovery_srcs/{module}.entries.swift".format( + module = symbol_graph.module_name, + target = name, + ), + ) + outputs.append(output_file) + args.add( + "--module-output", + "{module}={path}".format( + module = symbol_graph.module_name, + path = output_file.path, + ), + ) + + for symbol_graph in ( + symbol_graph_info.transitive_symbol_graphs.to_list() + ): + inputs.append(symbol_graph.symbol_graph_dir) + + # Also declare a single `main.swift` file where the discovery tool will + # generate the main runner. + main_file = actions.declare_file( + "{target}_test_discovery_srcs/main.swift".format(target = name), + ) + outputs.append(main_file) + args.add("--main-output", main_file) + + # The discovery tool expects symbol graph directories as its inputs (it + # iterates over their contents), so we must not expand directories here. + args.add_all(inputs, expand_directories = False, uniquify = True) + + actions.run( + arguments = [args], + executable = test_discoverer, + inputs = inputs, + mnemonic = "SwiftTestDiscovery", + outputs = outputs, + progress_message = "Discovering tests for %{label}", + ) + + return outputs + def _swift_test_impl(ctx): swift_toolchain = ctx.attr._toolchain[SwiftToolchainInfo] feature_configuration = configure_features_for_binary( @@ -319,6 +396,22 @@ def _swift_test_impl(ctx): extra_swift_infos.append(plugin_info.swift_info) additional_linking_contexts.append(plugin_info.cc_info.linking_context) + srcs = ctx.files.srcs + + # If no sources were provided and we're not using `.xctest` bundling, assume + # that we need to discover tests using symbol graphs. + # TODO(b/220945250): This supports SPM-style tests where each test target + # (a separate module) maps to its own `swift_library`. We'll need to modify + # this approach if we want to support test discovery for simple `swift_test` + # targets that just write XCTest-style tests in the `srcs` directly. + if not srcs and not is_bundled: + srcs = _generate_test_discovery_srcs( + actions = ctx.actions, + deps = ctx.attr.deps, + name = ctx.label.name, + test_discoverer = ctx.executable._test_discoverer, + ) + _, linking_outputs, providers = _swift_linking_rule_impl( ctx, additional_linking_contexts = additional_linking_contexts, @@ -327,6 +420,7 @@ def _swift_test_impl(ctx): extra_link_deps = extra_link_deps, feature_configuration = feature_configuration, linkopts = linkopts, + srcs = srcs, swift_toolchain = swift_toolchain, ) @@ -403,6 +497,7 @@ please use one of the platform-specific application rules in swift_test = rule( attrs = dicts.add( binary_rule_attrs( + additional_deps_aspects = [test_discovery_symbol_graph_aspect], additional_deps_providers = [[SwiftCompilerPluginInfo]], stamp_default = 0, ), @@ -423,6 +518,13 @@ swift_test = rule( "@build_bazel_rules_swift//swift/internal:swizzle_absolute_xcttestsourcelocation", ), ), + "_test_discoverer": attr.label( + cfg = "exec", + default = Label( + "@build_bazel_rules_swift//tools/test_discoverer", + ), + executable = True, + ), "_xctest_runner_template": attr.label( allow_single_file = True, default = Label( diff --git a/swift/internal/swift_common.bzl b/swift/internal/swift_common.bzl index 2b2de2801..42fe54864 100644 --- a/swift/internal/swift_common.bzl +++ b/swift/internal/swift_common.bzl @@ -52,6 +52,7 @@ load( "create_swift_module", ) load(":swift_clang_module_aspect.bzl", "create_swift_interop_info") +load(":symbol_graph_extracting.bzl", "extract_symbol_graph") # The exported `swift_common` module, which defines the public API for directly # invoking actions that compile Swift code from other rules. @@ -69,6 +70,7 @@ swift_common = struct( create_swift_interop_info = create_swift_interop_info, create_swift_module = create_swift_module, derive_module_name = derive_module_name, + extract_symbol_graph = extract_symbol_graph, is_enabled = is_feature_enabled, library_rule_attrs = swift_library_rule_attrs, precompile_clang_module = precompile_clang_module, diff --git a/swift/internal/swift_extract_symbol_graph.bzl b/swift/internal/swift_extract_symbol_graph.bzl new file mode 100644 index 000000000..84d5ee306 --- /dev/null +++ b/swift/internal/swift_extract_symbol_graph.bzl @@ -0,0 +1,125 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Implementation of the `swift_extract_module_graph` rule.""" + +load(":derived_files.bzl", "derived_files") +load(":providers.bzl", "SwiftInfo", "SwiftSymbolGraphInfo") +load(":swift_symbol_graph_aspect.bzl", "swift_symbol_graph_aspect") + +def _swift_extract_symbol_graph_impl(ctx): + actions = ctx.actions + + output_dir = derived_files.symbol_graph_directory( + actions = actions, + target_name = ctx.label.name, + ) + + args = actions.args() + args.add(output_dir.path) + + seen_modules = {} + inputs = [] + + for target in ctx.attr.targets: + if SwiftSymbolGraphInfo in target: + symbol_graph_info = target[SwiftSymbolGraphInfo] + for symbol_graph in symbol_graph_info.direct_symbol_graphs: + if symbol_graph.module_name in seen_modules: + fail("Module '{}' was provided by multiple targets.".format( + symbol_graph.module_name, + )) + + seen_modules[symbol_graph.module_name] = True + + # Expand the directory's files into the argument list so that + # they are individually copied into output directory (resulting + # in a flat layout). + args.add_all( + [symbol_graph.symbol_graph_dir], + expand_directories = True, + ) + inputs.append(symbol_graph.symbol_graph_dir) + + actions.run_shell( + arguments = [args], + command = """\ +set -e +output_dir="$1" +mkdir -p "${output_dir}" +shift +for symbol_file in "$@"; do + cp "${symbol_file}" "${output_dir}/$(basename ${symbol_file})" +done +""", + inputs = inputs, + mnemonic = "SwiftCollectSymbolGraphs", + outputs = [output_dir], + ) + + return [DefaultInfo(files = depset([output_dir]))] + +swift_extract_symbol_graph = rule( + attrs = { + "minimum_access_level": attr.string( + default = "public", + doc = """\ +The minimum access level of the declarations that should be emitted in the +symbol graphs. + +This value must be either `fileprivate`, `internal`, `private`, or `public`. The +default value is `public`. +""", + values = [ + "fileprivate", + "internal", + "private", + "public", + ], + ), + "targets": attr.label_list( + allow_empty = False, + aspects = [swift_symbol_graph_aspect], + doc = """\ +One or more Swift targets from which to extract symbol graphs. +""", + mandatory = True, + providers = [[SwiftInfo]], + ), + }, + doc = """\ +Extracts symbol graphs from one or more Swift targets. + +The output of this rule is a single directory named +`${TARGET_NAME}.symbolgraphs` that contains the symbol graph JSON files for the +Swift modules directly compiled by all the targets listed in the `targets` +attribute (but not their transitive dependencies). Therefore, for each module +the directory will contain: + +* One or more files named `${MODULE_NAME}.symbols.json` containing the symbol + graphs for non-`extension` declarations in each module. +* Optionally, one or more files named + `${MODULE_NAME}@${EXTENDED_MODULE}.symbols.json` containing the symbol + graphs for declarations in each module that extend types from other modules. + +This rule can be used as a simple interface to extract symbol graphs for later +processing by other Bazel rules (for example, a `genrule` that operates on the +resulting JSON files). For more complex workflows, we recommend writing a custom +rule that applies `swift_symbol_graph_aspect` to the targets of interest and +registers other Starlark actions that read the symbol graphs based on the +`SwiftSymbolGraphInfo` providers attached to those targets. The implementation +of this rule can serve as a guide for implementing such a rule. +""", + implementation = _swift_extract_symbol_graph_impl, +) diff --git a/swift/internal/swift_symbol_graph_aspect.bzl b/swift/internal/swift_symbol_graph_aspect.bzl new file mode 100644 index 000000000..c3380eebc --- /dev/null +++ b/swift/internal/swift_symbol_graph_aspect.bzl @@ -0,0 +1,188 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Implementation of the `swift_symbol_graph_aspect` aspect.""" + +load("@bazel_skylib//lib:dicts.bzl", "dicts") +load(":attrs.bzl", "swift_toolchain_attrs") +load(":derived_files.bzl", "derived_files") +load(":features.bzl", "configure_features") +load( + ":providers.bzl", + "SwiftInfo", + "SwiftSymbolGraphInfo", + "SwiftToolchainInfo", +) +load(":symbol_graph_extracting.bzl", "extract_symbol_graph") +load(":utils.bzl", "include_developer_search_paths") + +def _swift_symbol_graph_aspect_impl(target, aspect_ctx): + symbol_graphs = [] + + if SwiftInfo in target: + swift_toolchain = aspect_ctx.attr._toolchain[SwiftToolchainInfo] + feature_configuration = configure_features( + ctx = aspect_ctx, + swift_toolchain = swift_toolchain, + requested_features = aspect_ctx.features, + unsupported_features = aspect_ctx.disabled_features, + ) + + swift_info = target[SwiftInfo] + if CcInfo in target: + compilation_context = target[CcInfo].compilation_context + else: + compilation_context = cc_common.create_compilation_context() + + minimum_access_level = aspect_ctx.attr.minimum_access_level + + for module in swift_info.direct_modules: + output_dir = derived_files.symbol_graph_directory( + actions = aspect_ctx.actions, + target_name = target.label.name, + ) + extract_symbol_graph( + actions = aspect_ctx.actions, + compilation_contexts = [compilation_context], + feature_configuration = feature_configuration, + include_dev_srch_paths = include_developer_search_paths( + aspect_ctx.rule.attr, + ), + minimum_access_level = minimum_access_level, + module_name = module.name, + output_dir = output_dir, + swift_infos = [swift_info], + swift_toolchain = swift_toolchain, + ) + symbol_graphs.append( + struct( + module_name = module.name, + symbol_graph_dir = output_dir, + ), + ) + + # TODO(b/204480390): We intentionally don't propagate symbol graphs from + # private deps at this time, since the main use case for them is + # documentation. Are there use cases where we should consider this? + transitive_symbol_graphs = [] + for dep in getattr(aspect_ctx.rule.attr, "deps", []): + if SwiftSymbolGraphInfo in dep: + symbol_graph_info = dep[SwiftSymbolGraphInfo] + transitive_symbol_graphs.append( + symbol_graph_info.transitive_symbol_graphs, + ) + + return [ + SwiftSymbolGraphInfo( + direct_symbol_graphs = symbol_graphs, + transitive_symbol_graphs = depset( + symbol_graphs, + transitive = transitive_symbol_graphs, + ), + ), + ] + +def _testonly_symbol_graph_aspect_impl(target, aspect_ctx): + if not getattr(aspect_ctx.rule.attr, "testonly", False): + # It's safe to return early (and not propagate transitive info) because + # a non-`testonly` target can't depend on a `testonly` target, so there + # is no possibility of losing anything we'd want to keep. + return [ + SwiftSymbolGraphInfo( + direct_symbol_graphs = [], + transitive_symbol_graphs = depset(), + ), + ] + + return _swift_symbol_graph_aspect_impl(target, aspect_ctx) + +def _make_swift_symbol_graph_aspect( + *, + default_minimum_access_level, + doc = "", + testonly_targets): + """Creates an aspect that extracts Swift symbol graphs from dependencies. + + Args: + default_minimum_access_level: The default minimum access level of the + declarations that should be emitted in the symbol graphs. A rule + that applies this aspect can let users override this value if it + also provides an attribute named `minimum_access_level`. + doc: The documentation string for the aspect. + testonly_targets: If True, symbol graphs will only be extracted from + targets that have the `testonly` attribute set. + + Returns: + An `aspect` that can be applied to a rule's dependencies. + """ + if testonly_targets: + aspect_impl = _testonly_symbol_graph_aspect_impl + else: + aspect_impl = _swift_symbol_graph_aspect_impl + + return aspect( + attr_aspects = ["deps"], + attrs = dicts.add( + swift_toolchain_attrs(), + { + "minimum_access_level": attr.string( + default = default_minimum_access_level, + doc = """\ +The minimum access level of the declarations that should be emitted in the +symbol graphs. + +This value must be either `fileprivate`, `internal`, `private`, or `public`. The +default value is {default_value}. +""".format( + default_value = default_minimum_access_level, + ), + values = [ + "fileprivate", + "internal", + "private", + "public", + ], + ), + }, + ), + doc = doc, + fragments = ["cpp"], + implementation = aspect_impl, + provides = [SwiftSymbolGraphInfo], + ) + +# This aspect is exported as public API by `swift_common`. +swift_symbol_graph_aspect = _make_swift_symbol_graph_aspect( + default_minimum_access_level = "public", + doc = """\ +Extracts symbol graphs from Swift modules in the build graph. + +This aspect propagates a `SwiftSymbolGraphInfo` provider on any target to which +it is applied. This provider will contain the transitive module graph +information for the target's dependencies, and if the target propagates Swift +modules via its `SwiftInfo` provider, it will also extract and propagate their +symbol graphs by invoking the `swift-symbolgraph-extract` tool. + +For an example of how to apply this to a custom rule, refer to the +implementation of `swift_extract_symbol_graph`. + """, + testonly_targets = False, +) + +# This aspect is only meant to be used by `swift_test` and should not be +# exported by `swift_common`. +test_discovery_symbol_graph_aspect = _make_swift_symbol_graph_aspect( + default_minimum_access_level = "internal", + testonly_targets = True, +) diff --git a/swift/internal/swift_toolchain.bzl b/swift/internal/swift_toolchain.bzl index ec3451f6a..f71637503 100644 --- a/swift/internal/swift_toolchain.bzl +++ b/swift/internal/swift_toolchain.bzl @@ -53,6 +53,8 @@ load( "SwiftToolchainInfo", ) load(":toolchain_config.bzl", "swift_toolchain_config") +load(":symbol_graph_extracting.bzl", "symbol_graph_action_configs") +load(":target_triples.bzl", "target_triples") load( ":utils.bzl", "collect_implicit_deps_providers", @@ -103,10 +105,22 @@ def _all_tool_configs( env = env, ) + swift_symbolgraph_extract_config = _swift_driver_tool_config( + driver_mode = "swift-symbolgraph-extract", + swift_executable = swift_executable, + tools = additional_tools, + toolchain_root = toolchain_root, + tool_executable_suffix = tool_executable_suffix, + use_param_file = True, + worker_mode = "wrap", + env = env, + ) + configs = { swift_action_names.COMPILE: compile_tool_config, swift_action_names.DERIVE_FILES: compile_tool_config, swift_action_names.DUMP_AST: compile_tool_config, + swift_action_names.SYMBOL_GRAPH_EXTRACT: swift_symbolgraph_extract_config, } if use_autolink_extract: @@ -132,12 +146,13 @@ def _all_tool_configs( ) return configs -def _all_action_configs(os, arch, sdkroot, xctest_version, additional_swiftc_copts): +def _all_action_configs(os, arch, target_triple, sdkroot, xctest_version, additional_swiftc_copts): """Returns the action configurations for the Swift toolchain. Args: os: The OS that we are compiling for. arch: The architecture we are compiling for. + target_triple: The triple of the platform being targeted. sdkroot: The path to the SDK that we should use to build against. xctest_version: The version of XCTest to use. additional_swiftc_copts: Additional Swift compiler flags obtained from @@ -146,7 +161,23 @@ def _all_action_configs(os, arch, sdkroot, xctest_version, additional_swiftc_cop Returns: A list of action configurations for the toolchain. """ - return ( + + # Basic compilation flags (target triple). + action_configs = [ + swift_toolchain_config.action_config( + actions = [ + swift_action_names.SYMBOL_GRAPH_EXTRACT, + ], + configurators = [ + swift_toolchain_config.add_arg( + "-target", + target_triples.str(target_triple), + ), + ], + ), + ] + + action_configs.extend(( compile_action_configs( os = os, arch = arch, @@ -155,8 +186,11 @@ def _all_action_configs(os, arch, sdkroot, xctest_version, additional_swiftc_cop additional_swiftc_copts = additional_swiftc_copts, ) + modulewrap_action_configs() + - autolink_extract_action_configs() - ) + autolink_extract_action_configs() + + symbol_graph_action_configs() + )) + + return action_configs def _swift_windows_linkopts_cc_info( arch, @@ -271,9 +305,28 @@ def _entry_point_linkopts_provider(*, entry_point_name): linkopts = ["-Wl,--defsym,main={}".format(entry_point_name)], ) +def _parse_target_system_name(*, arch, os, target_system_name): + """Returns the target system name set by the CC toolchain or attempts to create one based on the OS and arch.""" + + if target_system_name != "local": + return target_system_name + + if os == "linux": + return "%s-unknown-linux-gnu" % arch + else: + return "%s-unknown-%s" % (arch, os) + def _swift_toolchain_impl(ctx): toolchain_root = ctx.attr.root cc_toolchain = find_cpp_toolchain(ctx) + target_system_name = _parse_target_system_name( + arch = ctx.attr.arch, + os = ctx.attr.os, + target_system_name = cc_toolchain.target_gnu_system_name, + ) + target_triple = target_triples.normalize_for_swift( + target_triples.parse(target_system_name), + ) if "clang" not in cc_toolchain.compiler: fail("Swift requires the configured CC toolchain to be LLVM (clang). " + @@ -344,6 +397,7 @@ def _swift_toolchain_impl(ctx): all_action_configs = _all_action_configs( os = ctx.attr.os, arch = ctx.attr.arch, + target_triple = target_triple, sdkroot = ctx.attr.sdkroot, xctest_version = ctx.attr.xctest_version, additional_swiftc_copts = swiftcopts, diff --git a/swift/internal/symbol_graph_extracting.bzl b/swift/internal/symbol_graph_extracting.bzl new file mode 100644 index 000000000..a5f02d80f --- /dev/null +++ b/swift/internal/symbol_graph_extracting.bzl @@ -0,0 +1,144 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Functions relating to symbol graph extraction.""" + +load(":actions.bzl", "run_toolchain_action", "swift_action_names") +load(":providers.bzl", "create_swift_info") +load(":toolchain_config.bzl", "swift_toolchain_config") +load(":utils.bzl", "merge_compilation_contexts") + +def symbol_graph_action_configs(): + """Returns the list of action configs needed to extract symbol graphs. + + If a toolchain supports symbol graph extraction, it should add these to its + list of action configs so that those actions will be correctly configured. + (Other required configuration is provided by `compile_action_configs`.) + + Returns: + The list of action configs needed to extract symbol graphs. + """ + return [ + swift_toolchain_config.action_config( + actions = [swift_action_names.SYMBOL_GRAPH_EXTRACT], + configurators = [ + _symbol_graph_minimum_access_level_configurator, + ], + ), + swift_toolchain_config.action_config( + actions = [swift_action_names.SYMBOL_GRAPH_EXTRACT], + configurators = [ + _symbol_graph_output_configurator, + ], + ), + ] + +def _symbol_graph_minimum_access_level_configurator(prerequisites, args): + """Configures the minimum access level of the symbol graph extraction.""" + if prerequisites.minimum_access_level: + args.add("-minimum-access-level", prerequisites.minimum_access_level) + +def _symbol_graph_output_configurator(prerequisites, args): + """Configures the outputs of the symbol graph extract action.""" + args.add("-output-dir", prerequisites.output_dir.path) + +def extract_symbol_graph( + *, + actions, + compilation_contexts, + feature_configuration, + include_dev_srch_paths, + minimum_access_level = None, + module_name, + output_dir, + swift_infos, + swift_toolchain): + """Extracts the symbol graph from a Swift module. + + Args: + actions: The object used to register actions. + compilation_contexts: A list of `CcCompilationContext`s that represent + C/Objective-C requirements of the target being compiled, such as + Swift-compatible preprocessor defines, header search paths, and so + forth. These are typically retrieved from the `CcInfo` providers of + a target's dependencies. + feature_configuration: The Swift feature configuration. + include_dev_srch_paths: A `bool` that indicates whether the developer + framework search paths will be added to the compilation command. + minimum_access_level: The minimum access level of the declarations that + should be extracted into the symbol graphs. The default value is + `None`, which means the Swift compiler's default behavior should be + used (at the time of this writing, the default behavior is + "public"). + module_name: The name of the module whose symbol graph should be + extracted. + output_dir: A directory-type `File` into which `.symbols.json` files + representing the module's symbol graph will be extracted. If + extraction is successful, this directory will contain a file named + `${MODULE_NAME}.symbols.json`. Optionally, if the module contains + extensions to types in other modules, then there will also be files + named `${MODULE_NAME}@${EXTENDED_MODULE}.symbols.json`. + swift_infos: A list of `SwiftInfo` providers from dependencies of the + target being compiled. This should include both propagated and + non-propagated (implementation-only) dependencies. + swift_toolchain: The `SwiftToolchainInfo` provider of the toolchain. + """ + merged_compilation_context = merge_compilation_contexts( + transitive_compilation_contexts = compilation_contexts + [ + cc_info.compilation_context + for cc_info in swift_toolchain.implicit_deps_providers.cc_infos + ], + ) + merged_swift_info = create_swift_info( + swift_infos = ( + swift_infos + swift_toolchain.implicit_deps_providers.swift_infos + ), + ) + + # Flattening this `depset` is necessary because we need to extract the + # module maps or precompiled modules out of structured values and do so + # conditionally. + transitive_modules = merged_swift_info.transitive_modules.to_list() + + transitive_swiftmodules = [] + for module in transitive_modules: + swift_module = module.swift + if swift_module: + transitive_swiftmodules.append(swift_module.swiftmodule) + + prerequisites = struct( + bin_dir = feature_configuration._bin_dir, + cc_compilation_context = merged_compilation_context, + developer_dirs = swift_toolchain.developer_dirs, + genfiles_dir = feature_configuration._genfiles_dir, + include_dev_srch_paths = include_dev_srch_paths, + is_swift = True, + minimum_access_level = minimum_access_level, + module_name = module_name, + output_dir = output_dir, + transitive_modules = transitive_modules, + transitive_swiftmodules = transitive_swiftmodules, + ) + + run_toolchain_action( + actions = actions, + action_name = swift_action_names.SYMBOL_GRAPH_EXTRACT, + feature_configuration = feature_configuration, + outputs = [output_dir], + prerequisites = prerequisites, + progress_message = ( + "Extracting symbol graph for {}".format(module_name) + ), + swift_toolchain = swift_toolchain, + ) diff --git a/swift/internal/xcode_swift_toolchain.bzl b/swift/internal/xcode_swift_toolchain.bzl index 1069297b6..baf9ba250 100644 --- a/swift/internal/xcode_swift_toolchain.bzl +++ b/swift/internal/xcode_swift_toolchain.bzl @@ -63,6 +63,7 @@ load( "SwiftPackageConfigurationInfo", "SwiftToolchainInfo", ) +load(":symbol_graph_extracting.bzl", "symbol_graph_action_configs") load(":target_triples.bzl", "target_triples") load(":toolchain_config.bzl", "swift_toolchain_config") load( @@ -325,8 +326,9 @@ def _all_action_configs( swift_action_names.COMPILE, swift_action_names.COMPILE_MODULE_INTERFACE, swift_action_names.DERIVE_FILES, - swift_action_names.PRECOMPILE_C_MODULE, swift_action_names.DUMP_AST, + swift_action_names.PRECOMPILE_C_MODULE, + swift_action_names.SYMBOL_GRAPH_EXTRACT, ], configurators = [ swift_toolchain_config.add_arg( @@ -407,8 +409,9 @@ def _all_action_configs( actions = [ swift_action_names.COMPILE, swift_action_names.DERIVE_FILES, - swift_action_names.PRECOMPILE_C_MODULE, swift_action_names.DUMP_AST, + swift_action_names.PRECOMPILE_C_MODULE, + swift_action_names.SYMBOL_GRAPH_EXTRACT, ], configurators = [ partial.make( @@ -424,6 +427,7 @@ def _all_action_configs( additional_swiftc_copts = additional_swiftc_copts, generated_header_rewriter = generated_header_rewriter, )) + action_configs.extend(symbol_graph_action_configs()) return action_configs def _all_tool_configs( @@ -497,6 +501,16 @@ def _all_tool_configs( worker_mode = "wrap", ) ), + swift_action_names.SYMBOL_GRAPH_EXTRACT: ( + swift_toolchain_config.driver_tool_config( + driver_mode = "swift-symbolgraph-extract", + env = env, + execution_requirements = execution_requirements, + swift_executable = swift_executable, + use_param_file = True, + worker_mode = "wrap", + ) + ), } return tool_configs diff --git a/swift/repositories.bzl b/swift/repositories.bzl index 692def5b6..266429dd5 100644 --- a/swift/repositories.bzl +++ b/swift/repositories.bzl @@ -89,6 +89,15 @@ def swift_rules_dependencies(include_bzlmod_ready_dependencies = True): url = "https://github.com/bazel-contrib/bazel_features/releases/download/v1.3.0/bazel_features-v1.3.0.tar.gz", ) + _maybe( + http_archive, + name = "com_github_apple_swift_argument_parser", + urls = ["https://github.com/apple/swift-argument-parser/archive/refs/tags/1.3.0.tar.gz"], + sha256 = "e5010ff37b542807346927ba68b7f06365a53cf49d36a6df13cef50d86018204", + strip_prefix = "swift-argument-parser-1.3.0", + build_file = "@build_bazel_rules_swift//third_party:com_github_apple_swift_argument_parser/BUILD.overlay", + ) + _maybe( http_archive, name = "com_github_apple_swift_protobuf", @@ -107,6 +116,15 @@ def swift_rules_dependencies(include_bzlmod_ready_dependencies = True): build_file = "@build_bazel_rules_swift//third_party:com_github_grpc_grpc_swift/BUILD.overlay", ) + _maybe( + http_archive, + name = "com_github_apple_swift_docc_symbolkit", + urls = ["https://github.com/apple/swift-docc-symbolkit/archive/refs/tags/swift-5.10-RELEASE.tar.gz"], + sha256 = "de1d4b6940468ddb53b89df7aa1a81323b9712775b0e33e8254fa0f6f7469a97", + strip_prefix = "swift-docc-symbolkit-swift-5.10-RELEASE", + build_file = "@build_bazel_rules_swift//third_party:com_github_apple_swift_docc_symbolkit/BUILD.overlay", + ) + _maybe( http_archive, name = "com_github_apple_swift_nio", diff --git a/swift/swift.bzl b/swift/swift.bzl index bd326a2db..55e9cc13b 100644 --- a/swift/swift.bzl +++ b/swift/swift.bzl @@ -37,6 +37,7 @@ load( _SwiftInfo = "SwiftInfo", _SwiftProtoCompilerInfo = "SwiftProtoCompilerInfo", _SwiftProtoInfo = "SwiftProtoInfo", + _SwiftSymbolGraphInfo = "SwiftSymbolGraphInfo", _SwiftToolchainInfo = "SwiftToolchainInfo", _SwiftUsageInfo = "SwiftUsageInfo", ) @@ -53,6 +54,10 @@ load( "@build_bazel_rules_swift//swift/internal:swift_common.bzl", _swift_common = "swift_common", ) +load( + "@build_bazel_rules_swift//swift/internal:swift_extract_symbol_graph.bzl", + _swift_extract_symbol_graph = "swift_extract_symbol_graph", +) load( "@build_bazel_rules_swift//swift/internal:swift_feature_allowlist.bzl", _swift_feature_allowlist = "swift_feature_allowlist", @@ -81,6 +86,10 @@ load( "@build_bazel_rules_swift//swift/internal:swift_package_configuration.bzl", _swift_package_configuration = "swift_package_configuration", ) +load( + "@build_bazel_rules_swift//swift/internal:swift_symbol_graph_aspect.bzl", + _swift_symbol_graph_aspect = "swift_symbol_graph_aspect", +) load( "@build_bazel_rules_swift//swift/internal:swift_usage_aspect.bzl", _swift_usage_aspect = "swift_usage_aspect", @@ -90,6 +99,7 @@ load( SwiftInfo = _SwiftInfo SwiftProtoCompilerInfo = _SwiftProtoCompilerInfo SwiftProtoInfo = _SwiftProtoInfo +SwiftSymbolGraphInfo = _SwiftSymbolGraphInfo SwiftToolchainInfo = _SwiftToolchainInfo SwiftUsageInfo = _SwiftUsageInfo @@ -97,6 +107,7 @@ SwiftUsageInfo = _SwiftUsageInfo swift_common = _swift_common # Re-export rules. +swift_extract_symbol_graph = _swift_extract_symbol_graph swift_binary = _swift_binary swift_compiler_plugin = _swift_compiler_plugin universal_swift_compiler_plugin = _universal_swift_compiler_plugin @@ -111,4 +122,5 @@ swift_test = _swift_test # Re-export public aspects. swift_clang_module_aspect = _swift_clang_module_aspect +swift_symbol_graph_aspect = _swift_symbol_graph_aspect swift_usage_aspect = _swift_usage_aspect diff --git a/test/BUILD b/test/BUILD index 33355b492..9f5b59a8c 100644 --- a/test/BUILD +++ b/test/BUILD @@ -19,6 +19,7 @@ load(":private_deps_tests.bzl", "private_deps_test_suite") load(":private_swiftinterface_tests.bzl", "private_swiftinterface_test_suite") load(":split_derived_files_tests.bzl", "split_derived_files_test_suite") load(":swift_binary_linking_tests.bzl", "swift_binary_linking_test_suite") +load(":symbol_graphs_tests.bzl", "symbol_graphs_test_suite") load(":swift_through_non_swift_tests.bzl", "swift_through_non_swift_test_suite") load(":utils_tests.bzl", "utils_test_suite") load(":xctest_runner_tests.bzl", "xctest_runner_test_suite") @@ -63,6 +64,8 @@ pch_output_dir_test_suite(name = "pch_output_dir_settings") private_swiftinterface_test_suite(name = "private_swiftinterface") +symbol_graphs_test_suite(name = "symbol_graphs") + xctest_runner_test_suite(name = "xctest_runner") utils_test_suite(name = "utils") diff --git a/test/fixtures/symbol_graphs/BUILD b/test/fixtures/symbol_graphs/BUILD new file mode 100644 index 000000000..dfc1b4aeb --- /dev/null +++ b/test/fixtures/symbol_graphs/BUILD @@ -0,0 +1,52 @@ +load( + "//swift:swift.bzl", + "swift_extract_symbol_graph", + "swift_library", +) +load( + "//test/fixtures:common.bzl", + "FIXTURE_TAGS", +) + +package( + default_testonly = True, + default_visibility = ["//test:__subpackages__"], +) + +licenses(["notice"]) + +swift_library( + name = "some_module", + srcs = ["SomeModule.swift"], + module_name = "SomeModule", + tags = FIXTURE_TAGS, +) + +swift_library( + name = "importing_module", + srcs = ["ImportingModule.swift"], + module_name = "ImportingModule", + tags = FIXTURE_TAGS, + deps = [":some_module"], +) + +swift_extract_symbol_graph( + name = "some_module_symbol_graph", + tags = FIXTURE_TAGS, + targets = [":some_module"], +) + +swift_extract_symbol_graph( + name = "importing_module_symbol_graph", + tags = FIXTURE_TAGS, + targets = [":importing_module"], +) + +swift_extract_symbol_graph( + name = "all_symbol_graphs", + tags = FIXTURE_TAGS, + targets = [ + ":importing_module", + ":some_module", + ], +) diff --git a/test/fixtures/symbol_graphs/ImportingModule.swift b/test/fixtures/symbol_graphs/ImportingModule.swift new file mode 100644 index 000000000..eea8f240c --- /dev/null +++ b/test/fixtures/symbol_graphs/ImportingModule.swift @@ -0,0 +1,4 @@ +import SomeModule + +/// Here's another documented symbol in another module. +public class AnotherClass {} diff --git a/test/fixtures/symbol_graphs/SomeModule.swift b/test/fixtures/symbol_graphs/SomeModule.swift new file mode 100644 index 000000000..89cf0471d --- /dev/null +++ b/test/fixtures/symbol_graphs/SomeModule.swift @@ -0,0 +1,19 @@ +/// This class is documented. +public class SomeClass { + /// This method is documented. + /// + /// - Parameter count: This parameter is documented. + /// - Returns: This return value is documented. + public func someMethod(someParameter count: Int) -> String { + return String(repeating: "someString", count: count) + } +} + +extension String { + /// This method is documented, and it's on an extension. + /// + /// - Returns: This return value is documented. + public func someExtensionMethod() -> String { + return self + } +} diff --git a/test/rules/directory_test.bzl b/test/rules/directory_test.bzl new file mode 100644 index 000000000..d614bce80 --- /dev/null +++ b/test/rules/directory_test.bzl @@ -0,0 +1,103 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Rules for testing the contents of directory artifact outputs. + +Since the contents of these directories are not known at analysis time, we need +to spawn a shell script that lists their contents. +""" + +def _directory_test_impl(ctx): + target_under_test = ctx.attr.target_under_test + path_suffixes = ctx.attr.expected_directories.keys() + + # Map the path suffixes to files output by the target. If multiple outputs + # match, fail the build. + path_suffix_to_output = {} + for path_suffix in path_suffixes: + for output in target_under_test[DefaultInfo].files.to_list(): + if output.short_path.endswith(path_suffix): + if path_suffix in path_suffix_to_output: + fail(("Target {} had multiple outputs whose paths end in " + + "'{}'; use additional path segments to distinguish " + + "them.").format(target_under_test.label, path_suffix)) + path_suffix_to_output[path_suffix] = output + + # If a path suffix did not match any of the outputs, fail. + for path_suffix in path_suffixes: + if path_suffix not in path_suffix_to_output: + fail(("Target {} did not output a directory whose path ends in " + + "'{}'.").format(target_under_test.label, path_suffix)) + + # Generate a script that verifies the existence of each expected file. + generated_script = [ + "#!/usr/bin/env bash", + "function check_file() {", + " if [[ -f \"$1/$2\" ]]; then", + " return 0", + " else", + " echo \"ERROR: Expected file '$2' did not exist in output " + + "directory '$1'\"", + " return 1", + " fi", + "}", + "failed=0", + ] + for path_suffix, files in ctx.attr.expected_directories.items(): + dir_path = path_suffix_to_output[path_suffix].short_path + for file in files: + generated_script.append( + "check_file \"{dir}\" \"{file}\" || failed=1".format( + dir = dir_path, + file = file, + ), + ) + generated_script.append("echo") + + generated_script.append("exit ${failed}") + + output_script = ctx.actions.declare_file( + "{}_test_script".format(ctx.label.name), + ) + ctx.actions.write( + output = output_script, + content = "\n".join(generated_script), + is_executable = True, + ) + + return [ + DefaultInfo( + executable = output_script, + runfiles = ctx.runfiles(files = path_suffix_to_output.values()), + ), + ] + +directory_test = rule( + attrs = { + "expected_directories": attr.string_list_dict( + mandatory = True, + doc = """\ +A dictionary where each key is the path suffix of a directory (tree artifact) +output by the target under test, and the corresponding value is a list of files +expected to exist in that directory (expressed as paths relative to the key). +""", + ), + "target_under_test": attr.label( + mandatory = True, + doc = "The target whose outputs are to be verified.", + ), + }, + implementation = _directory_test_impl, + test = True, +) diff --git a/test/split_derived_files_tests.bzl b/test/split_derived_files_tests.bzl index 8808a6d4a..f95805403 100644 --- a/test/split_derived_files_tests.bzl +++ b/test/split_derived_files_tests.bzl @@ -115,21 +115,6 @@ split_swiftmodule_copts_test = make_action_command_line_test_rule( ], }, ) -split_swiftmodule_symbol_graph_test = make_action_command_line_test_rule( - config_settings = { - "//command_line_option:features": [ - "swift.emit_symbol_graph", - "swift.split_derived_files_generation", - ], - }, -) -default_no_split_swiftmodule_symbol_graph_test = make_action_command_line_test_rule( - config_settings = { - "//command_line_option:features": [ - "swift.emit_symbol_graph", - ], - }, -) def split_derived_files_test_suite(name): """Test suite for split derived files options. @@ -292,39 +277,6 @@ def split_derived_files_test_suite(name): target_under_test = "@build_bazel_rules_swift//test/fixtures/debug_settings:simple", ) - split_swiftmodule_symbol_graph_test( - name = "{}_symbol_graph_in_derive_action".format(name), - expected_argv = [ - "-emit-symbol-graph", - "-emit-symbol-graph-dir", - ], - mnemonic = "SwiftDeriveFiles", - tags = [name], - target_under_test = "@build_bazel_rules_swift//test/fixtures/debug_settings:simple", - ) - - split_swiftmodule_symbol_graph_test( - name = "{}_no_symbol_graph_in_compile_action".format(name), - not_expected_argv = [ - "-emit-symbol-graph", - "-emit-symbol-graph-dir", - ], - mnemonic = "SwiftCompile", - tags = [name], - target_under_test = "@build_bazel_rules_swift//test/fixtures/debug_settings:simple", - ) - - default_no_split_swiftmodule_symbol_graph_test( - name = "{}_default_no_split_symbol_graph_in_compile_action".format(name), - expected_argv = [ - "-emit-symbol-graph", - "-emit-symbol-graph-dir", - ], - mnemonic = "SwiftCompile", - tags = [name], - target_under_test = "@build_bazel_rules_swift//test/fixtures/debug_settings:simple", - ) - split_swiftmodule_test( name = "{}_swiftmodule_only".format(name), expected_argv = [ diff --git a/test/symbol_graphs_tests.bzl b/test/symbol_graphs_tests.bzl new file mode 100644 index 000000000..98f6764f2 --- /dev/null +++ b/test/symbol_graphs_tests.bzl @@ -0,0 +1,76 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for extracting symbol graphs.""" + +load( + "@build_bazel_rules_swift//test/rules:directory_test.bzl", + "directory_test", +) + +def symbol_graphs_test_suite(name): + """Test suite for extracting symbol graphs. + + Args: + name: The base name to be used in targets created by this macro. + """ + + # Verify that the `swift_extract_symbol_graph` rule produces a directory + # output containing the correct files when the requested target is a leaf + # module. + directory_test( + name = "{}_extract_rule_outputs_only_requested_target_files_if_it_is_leaf".format(name), + expected_directories = { + "test/fixtures/symbol_graphs/some_module_symbol_graph.symbolgraphs": [ + "SomeModule.symbols.json", + "SomeModule@Swift.symbols.json", + ], + }, + tags = [name], + target_under_test = "@build_bazel_rules_swift//test/fixtures/symbol_graphs:some_module_symbol_graph", + ) + + # Verify that the `swift_extract_symbol_graph` rule produces a directory + # output containing only the graph for the requested target and not its + # dependencies. + directory_test( + name = "{}_extract_rule_outputs_only_requested_target_files_if_it_has_deps".format(name), + expected_directories = { + "test/fixtures/symbol_graphs/importing_module_symbol_graph.symbolgraphs": [ + "ImportingModule.symbols.json", + ], + }, + tags = [name], + target_under_test = "@build_bazel_rules_swift//test/fixtures/symbol_graphs:importing_module_symbol_graph", + ) + + # Verify that the `swift_extract_symbol_graph` rule produces a directory + # output containing the correct files when multiple targets are requested. + directory_test( + name = "{}_extract_rule_outputs_all_requested_target_files".format(name), + expected_directories = { + "test/fixtures/symbol_graphs/all_symbol_graphs.symbolgraphs": [ + "ImportingModule.symbols.json", + "SomeModule.symbols.json", + "SomeModule@Swift.symbols.json", + ], + }, + tags = [name], + target_under_test = "@build_bazel_rules_swift//test/fixtures/symbol_graphs:all_symbol_graphs", + ) + + native.test_suite( + name = name, + tags = [name], + ) diff --git a/third_party/com_github_apple_swift_argument_parser/BUILD.overlay b/third_party/com_github_apple_swift_argument_parser/BUILD.overlay new file mode 100644 index 000000000..7db3aee8d --- /dev/null +++ b/third_party/com_github_apple_swift_argument_parser/BUILD.overlay @@ -0,0 +1,17 @@ +load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") + +swift_library( + name = "ArgumentParserToolInfo", + srcs = glob(["Sources/ArgumentParserToolInfo/**/*.swift"]), + module_name = "ArgumentParserToolInfo", +) + +swift_library( + name = "ArgumentParser", + srcs = glob(["Sources/ArgumentParser/**/*.swift"]), + module_name = "ArgumentParser", + visibility = ["//visibility:public"], + deps = [ + ":ArgumentParserToolInfo", + ], +) diff --git a/third_party/com_github_apple_swift_docc_symbolkit/BUILD.overlay b/third_party/com_github_apple_swift_docc_symbolkit/BUILD.overlay new file mode 100644 index 000000000..573a8ec7a --- /dev/null +++ b/third_party/com_github_apple_swift_docc_symbolkit/BUILD.overlay @@ -0,0 +1,10 @@ +load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") + +swift_library( + name = "SymbolKit", + srcs = glob([ + "Sources/SymbolKit/**/*.swift", + ]), + module_name = "SymbolKit", + visibility = ["//visibility:public"], +) diff --git a/tools/test_discoverer/BUILD b/tools/test_discoverer/BUILD new file mode 100644 index 000000000..565cf73ed --- /dev/null +++ b/tools/test_discoverer/BUILD @@ -0,0 +1,17 @@ +load("//swift:swift.bzl", "swift_binary") + +swift_binary( + name = "test_discoverer", + srcs = [ + "DiscoveredTests.swift", + "SymbolCollector.swift", + "SymbolKitExtensions.swift", + "TestDiscoverer.swift", + "TestPrinter.swift", + ], + visibility = ["//visibility:public"], + deps = [ + "@com_github_apple_swift_argument_parser//:ArgumentParser", + "@com_github_apple_swift_docc_symbolkit//:SymbolKit", + ], +) diff --git a/tools/test_discoverer/DiscoveredTests.swift b/tools/test_discoverer/DiscoveredTests.swift new file mode 100644 index 000000000..fdeb25c78 --- /dev/null +++ b/tools/test_discoverer/DiscoveredTests.swift @@ -0,0 +1,55 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Structured information about test classes and methods discovered by scanning symbol graphs. +struct DiscoveredTests { + /// The modules containing test classes/methods that were discovered in the symbol graph, keyed by + /// the module name. + var modules: [String: Module] = [:] +} + +extension DiscoveredTests { + /// Information about a module discovered in the symbol graphs that contains tests. + struct Module { + /// The name of the module. + var name: String + + /// The `XCTestCase`-inheriting classes (or extensions to `XCTestCase`-inheriting classes) in + /// the module, keyed by the class name. + var classes: [String: Class] = [:] + } +} + +extension DiscoveredTests { + /// Information about a class or class extension discovered in the symbol graphs that inherits + /// (directly or indirectly) from `XCTestCase`. + struct Class { + /// The name of the `XCTestCase`-inheriting class. + var name: String + + /// The methods that were discovered in the class to represent tests. + var methods: [Method] = [] + } +} + +extension DiscoveredTests { + /// Information about a discovered test method in an `XCTestCase` subclass. + struct Method { + /// The name of the discovered test method. + var name: String + + /// Indicates whether the test method was declared `async` or not. + var isAsync: Bool + } +} diff --git a/tools/test_discoverer/SymbolCollector.swift b/tools/test_discoverer/SymbolCollector.swift new file mode 100644 index 000000000..484006964 --- /dev/null +++ b/tools/test_discoverer/SymbolCollector.swift @@ -0,0 +1,186 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import SymbolKit + +/// The precise identifier of the `XCTest.XCTestCase` class. +/// +/// This is the mangled name of `XCTest.XCTestCase` with the leading "$s" replaced by "s:" (a common +/// notation used in Clang/Swift UIDs used for indexing). +private let xcTestCasePreciseIdentifier = "s:6XCTest0A4CaseC" + +/// Collects information from one or more symbol graphs in order to determine which classes and +/// methods correspond to `XCTest`-style test cases. +final class SymbolCollector { + /// `inheritsFrom` relationships collected from symbol graphs, keyed by their source identifier + /// (i.e., the subclass in the relationship). + private var inheritanceRelationships: [String: SymbolGraph.Relationship] = [:] + + /// `memberOf` relationships collected from symbol graphs, keyed by their source identifier (i.e., + /// the nested or contained declaration). + private var memberRelationships: [String: SymbolGraph.Relationship] = [:] + + /// A mapping from class identifiers to Boolean values indicating whether or not the class is + /// known to be or not be a test class (that is, inherit directly or indirectly from + /// `XCTestCase`). + /// + /// If the value for a class identifier is true, the class is known to inherit from `XCTestCase`. + /// If the value is false, the class is known to not inherit from `XCTestCase`. If the class + /// identifier is not present in the map, then its state is not yet known. + private var testCaseClassCache: [String: Bool] = [:] + + /// A mapping from discovered class identifiers to their symbol graph data. + private var possibleTestClasses: [String: SymbolGraph.Symbol] = [:] + + /// A collection of methods that match heuristics to be considered as test methods -- their names + /// begin with "test", they take no arguments, and they return `Void`. + private var possibleTestMethods: [SymbolGraph.Symbol] = [] + + /// A mapping from class (or class extension) identifiers to the module name where they were + /// declared. + private var modulesForClassIdentifiers: [String: String] = [:] + + /// Collects information from the given symbol graph that is needed to discover test classes and + /// test methods in the module. + func consume(_ symbolGraph: SymbolGraph) { + // First, collect all the inheritance and member relationships from the graph. We cannot filter + // them at this time, since they only contain the identifiers and might reference symbols in + // modules whose graphs haven't been processed yet. + for relationship in symbolGraph.relationships { + switch relationship.kind { + case .inheritsFrom: + inheritanceRelationships[relationship.source] = relationship + case .memberOf: + memberRelationships[relationship.source] = relationship + default: + break + } + } + + // Next, collect classes and methods that might be tests. We can do limited filtering here, as + // described below. + symbolLoop: for (preciseIdentifier, symbol) in symbolGraph.symbols { + switch symbol.kind.identifier { + case .class: + // Keep track of all classes for now; their inheritance relationships will be resolved + // on-demand once we have all the symbol graphs loaded. + possibleTestClasses[preciseIdentifier] = symbol + modulesForClassIdentifiers[preciseIdentifier] = symbolGraph.module.name + + case .method: + // Swift Package Manager uses the index store to discover tests; index-while-building writes + // a unit-test property for any method that satisfies this method: + // https://github.com/apple/swift/blob/da3856c45b7149730d6e5fdf528ac82b43daccac/lib/Index/IndexSymbol.cpp#L40-L82 + // We duplicate that logic here. + + guard symbol.swiftGenerics == nil else { + // Generic methods cannot be tests. + continue symbolLoop + } + + guard symbol.functionSignature?.isTestLike == true else { + // Functions with parameters or which return something other than `Void` cannot be tests. + continue symbolLoop + } + + let lastComponent = symbol.pathComponents.last! + guard lastComponent.hasPrefix("test") else { + // Test methods must be named `test*`. + continue symbolLoop + } + + // If we got this far, record the symbol as a possible test method. We still need to make + // sure later that it is a member of a class that inherits from `XCTestCase`. + possibleTestMethods.append(symbol) + + default: + break + } + } + } + + /// Returns a `DiscoveredTests` value containing structured information about the tests discovered + /// in the symbol graph. + func discoveredTests() -> DiscoveredTests { + var discoveredTests = DiscoveredTests() + + for method in possibleTestMethods { + if let classSymbol = testClassSymbol(for: method), + let moduleName = modulesForClassIdentifiers[classSymbol.identifier.precise] + { + let className = classSymbol.pathComponents.last! + + let lastMethodComponent = method.pathComponents.last! + let methodName = + lastMethodComponent.hasSuffix("()") + ? String(lastMethodComponent.dropLast(2)) + : lastMethodComponent + + discoveredTests.modules[moduleName, default: DiscoveredTests.Module(name: moduleName)] + .classes[className, default: DiscoveredTests.Class(name: className)] + .methods.append( + DiscoveredTests.Method(name: methodName, isAsync: method.isAsyncDeclaration)) + } + } + + return discoveredTests + } +} + +extension SymbolCollector { + /// Returns the symbol graph symbol information for the class (or class extension) that contains + /// the given method if and only if the class or class extension is a test class. + /// + /// If the containing class is unknown or it is not a test class, this method returns nil. + private func testClassSymbol(for method: SymbolGraph.Symbol) -> SymbolGraph.Symbol? { + guard let memberRelationship = memberRelationships[method.identifier.precise] else { + return nil + } + + let classIdentifier = memberRelationship.target + guard isTestClass(classIdentifier) else { + return nil + } + + return possibleTestClasses[classIdentifier] + } + + /// Returns a value indicating whether or not the class with the given identifier extends + /// `XCTestCase` (or if the identifier is a class extension, whether it extends a subclass of + /// `XCTestCase`). + private func isTestClass(_ preciseIdentifier: String) -> Bool { + if let known = testCaseClassCache[preciseIdentifier] { + return known + } + + guard let inheritanceRelationship = inheritanceRelationships[preciseIdentifier] else { + // If there are no inheritance relationships with the identifier as the source, then the class + // is either a root class or we didn't process the symbol graph for the module that declares + // it. In either case, we can't go any further so we mark the class as not-a-test. + testCaseClassCache[preciseIdentifier] = false + return false + } + + if inheritanceRelationship.target == xcTestCasePreciseIdentifier { + // If the inheritance relationship has the precise identifier for `XCTest.XCTestCase` as its + // target, then we know definitively that the class is a direct subclass of `XCTestCase`. + testCaseClassCache[preciseIdentifier] = true + return true + } + + // If the inheritance relationship had some other class as its target (the superclass), then + // (inductively) the source (subclass) is a test class if the superclass is. + return isTestClass(inheritanceRelationship.target) + } +} diff --git a/tools/test_discoverer/SymbolKitExtensions.swift b/tools/test_discoverer/SymbolKitExtensions.swift new file mode 100644 index 000000000..866cc003d --- /dev/null +++ b/tools/test_discoverer/SymbolKitExtensions.swift @@ -0,0 +1,54 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import SymbolKit + +extension SymbolGraph.Symbol { + /// Returns true if the given symbol represents an `async` declaration, or false otherwise. + var isAsyncDeclaration: Bool { + guard let mixin = declarationFragments else { return false } + + return mixin.declarationFragments.contains { fragment in + fragment.kind == .keyword && fragment.spelling == "async" + } + } + + /// Returns the symbol's `DeclarationFragments` mixin, or `nil` if it does not exist. + var declarationFragments: DeclarationFragments? { + mixins[DeclarationFragments.mixinKey] as? DeclarationFragments + } + + /// Returns the symbol's `FunctionSignature` mixin, or `nil` if it does not exist. + var functionSignature: FunctionSignature? { + mixins[FunctionSignature.mixinKey] as? FunctionSignature + } + + /// Returns the symbol's `Swift.Generics` mixin, or `nil` if it does not exist. + var swiftGenerics: Swift.Generics? { + mixins[Swift.Generics.mixinKey] as? Swift.Generics + } +} + +extension SymbolGraph.Symbol.FunctionSignature { + /// Returns true if the given function signature satisfies the requirements to be a test function; + /// that is, it has no parameters and returns `Void`. + var isTestLike: Bool { + // TODO(b/220940013): Do we need to support the `Void` spelling here too, if someone writes + // `Void` specifically instead of omitting the return type? + parameters.isEmpty + && returns.count == 1 + && returns[0].kind == .text + && returns[0].spelling == "()" + } +} diff --git a/tools/test_discoverer/TestDiscoverer.swift b/tools/test_discoverer/TestDiscoverer.swift new file mode 100644 index 000000000..7263753f5 --- /dev/null +++ b/tools/test_discoverer/TestDiscoverer.swift @@ -0,0 +1,94 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import ArgumentParser +import Foundation +import SymbolKit + +@main +struct TestDiscoverer: ParsableCommand { + /// A parsed module name and output path pair passed to the test discovery tool using the + /// `--module-output =` flag. + struct ModuleOutput: ExpressibleByArgument { + /// The name of the module. + var moduleName: String + + /// The file URL to a `.swift` source file that should be created or overwritten with the + /// discovered test entries. + var outputURL: URL + + init?(argument: String) { + let components = argument.split(separator: "=", maxSplits: 1) + guard components.count == 2 else { return nil } + + self.moduleName = String(components[0]) + self.outputURL = URL(fileURLWithPath: String(components[1])) + } + } + + @Argument(help: "Paths to directories containing symbol graph JSON files.") + var symbolGraphDirectories: [String] = [] + + @Option(help: "The path to the '.swift' file where the main test runner should be generated.") + var mainOutput: String + + @Option( + help: .init( + """ + The name of a module containing tests and the path to the '.swift' file where the test entries + for that module should be generated, in the form '='. Must be + specified at least once. + """, + valueName: "module-name-output-path-mapping")) + var moduleOutput: [ModuleOutput] = [] + + func validate() throws { + guard !moduleOutput.isEmpty else { + throw ValidationError("At least one '--module-output' must be provided.") + } + guard !symbolGraphDirectories.isEmpty else { + throw ValidationError("At least one symbol graph directory must be provided.") + } + } + + mutating func run() throws { + let collector = SymbolCollector() + + for directoryPath in symbolGraphDirectories { + // Each symbol graph directory might contain multiple files, all of which need to be parsed; + // there are files for symbols declared in the module itself and for symbols that represent + // extensions to types declared in other modules. + for url in try FileManager.default.contentsOfDirectory( + at: URL(fileURLWithPath: directoryPath), + includingPropertiesForKeys: nil) + { + let jsonData = try Data(contentsOf: url) + let decoder = JSONDecoder() + let symbolGraph = try decoder.decode(SymbolGraph.self, from: jsonData) + collector.consume(symbolGraph) + } + } + + // For each module, print the list of test entries that were discovered in a source file that + // extends that module. + let testPrinter = TestPrinter(discoveredTests: collector.discoveredTests()) + for output in moduleOutput { + testPrinter.printTestEntries(forModule: output.moduleName, toFileAt: output.outputURL) + } + + // Print the runner source file, which implements the `@main` type that executes the tests. + let mainFileURL = URL(fileURLWithPath: mainOutput) + testPrinter.printTestRunner(toFileAt: mainFileURL) + } +} diff --git a/tools/test_discoverer/TestPrinter.swift b/tools/test_discoverer/TestPrinter.swift new file mode 100644 index 000000000..e9bbb5148 --- /dev/null +++ b/tools/test_discoverer/TestPrinter.swift @@ -0,0 +1,164 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation + +private let availabilityAttribute = """ + @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow \ + inclusion of deprecated tests (which test deprecated functionality) without warnings.") + """ + +/// Creates a text file with the given contents at a file URL. +private func createTextFile(at url: URL, contents: String) { + FileManager.default.createFile(atPath: url.path, contents: contents.data(using: .utf8)!) +} + +/// Returns a Swift expression used to populate the function references in the `XCTestCaseEntry` +/// array for the given test method. +/// +/// The returned string considers whether the test is declared `async`, wrapping it with a helper +/// function if necessary. +private func generatedTestEntry(for method: DiscoveredTests.Method) -> String { + if method.isAsync { + return "asyncTest(\(method.name))" + } else { + return method.name + } +} + +/// Returns the Swift identifier that represents the generated array of test entries for the given +/// test class. +private func allTestsIdentifier(for testClass: DiscoveredTests.Class) -> String { + return "__allTests__\(testClass.name)" +} + +/// Returns the Swift identifier that represents the generated function that returns the combined +/// test entries for all the test classes in the given module. +private func allTestsIdentifier(for module: DiscoveredTests.Module) -> String { + return "\(module.name)__allTests" +} + +/// Prints discovered test entries and a test runner as Swift source code to be compiled in order to +/// run the tests. +struct TestPrinter { + /// The discovered tests whose entries and runner should be printed as Swift source code. + let discoveredTests: DiscoveredTests + + init(discoveredTests: DiscoveredTests) { + self.discoveredTests = discoveredTests + } + + /// Writes the accessor for the test entries discovered in the given module to a Swift source + /// file. + func printTestEntries(forModule moduleName: String, toFileAt url: URL) { + guard let discoveredModule = discoveredTests.modules[moduleName] else { + // No tests were discovered in a module passed to the tool, but Bazel still declared the file + // and expects us to generate something, so print an "empty" file for it to compile. + createTextFile( + at: url, + contents: """ + // No tests were discovered in module \(moduleName). + + """) + return + } + + var contents = """ + import XCTest + @testable import \(moduleName) + + """ + + let sortedClassNames = discoveredModule.classes.keys.sorted() + for className in sortedClassNames { + let testClass = discoveredModule.classes[className]! + + contents += """ + + fileprivate extension \(className) { + \(availabilityAttribute) + static let \(allTestsIdentifier(for: testClass)) = [ + + """ + + for testMethod in testClass.methods.sorted(by: { $0.name < $1.name }) { + contents += """ + ("\(testMethod.name)", \(generatedTestEntry(for: testMethod))), + + """ + } + + contents += """ + ] + } + + """ + } + + contents += """ + + \(availabilityAttribute) + func \(allTestsIdentifier(for: discoveredModule))() -> [XCTestCaseEntry] { + return [ + + """ + + for className in sortedClassNames { + let testClass = discoveredModule.classes[className]! + contents += """ + testCase(\(className).\(allTestsIdentifier(for: testClass))), + + """ + } + + contents += """ + ] + } + + """ + + createTextFile(at: url, contents: contents) + } + + /// Prints the main test runner to a Swift source file. + func printTestRunner(toFileAt url: URL) { + var contents = """ + import XCTest + + @main + \(availabilityAttribute) + struct Runner { + static func main() { + var tests = [XCTestCaseEntry]() + + """ + + for moduleName in discoveredTests.modules.keys.sorted() { + let module = discoveredTests.modules[moduleName]! + contents += """ + tests += \(allTestsIdentifier(for: module))() + + """ + } + + contents += """ + XCTMain(tests) + } + } + + """ + + createTextFile(at: url, contents: contents) + } +}