diff --git a/analyzer/src/funTest/kotlin/PackageManagerFunTest.kt b/analyzer/src/funTest/kotlin/PackageManagerFunTest.kt index 65bf665b13011..589486c29cdcc 100644 --- a/analyzer/src/funTest/kotlin/PackageManagerFunTest.kt +++ b/analyzer/src/funTest/kotlin/PackageManagerFunTest.kt @@ -39,6 +39,7 @@ import org.ossreviewtoolkit.model.config.PathExcludeReason class PackageManagerFunTest : WordSpec({ val definitionFiles = listOf( "bazel/MODULE.bazel", + "bitbake/recipe.bb", "bower/bower.json", "bundler/Gemfile", "cargo/Cargo.toml", diff --git a/plugins/package-managers/bitbake/README.md b/plugins/package-managers/bitbake/README.md new file mode 100644 index 0000000000000..1ff145be06cba --- /dev/null +++ b/plugins/package-managers/bitbake/README.md @@ -0,0 +1,13 @@ +# About + +This is a package manager plugin for the [OSS Review Toolkit][ORT] to analyze [Yocto] projects managed by [BitBake]. +It supersedes the combination of the [meta-doubleopen] and [do-convert] projects by relying on upstream [SBOM] generation in [SPDX] format, and converting the generated files to an ORT analyzer result file via ORT's [SPDX document file analyzer]. + +[ORT]: https://github.com/oss-review-toolkit/ort +[BitBake]: https://docs.yoctoproject.org/bitbake.html +[Yocto]: https://www.yoctoproject.org/ +[meta-doubleopen]: https://github.com/doubleopen-project/meta-doubleopen +[do-convert]: https://github.com/doubleopen-project/do-convert +[SBOM]: https://docs.yoctoproject.org/dev/dev-manual/sbom.html +[SPDX]: https://spdx.dev/ +[SPDX document file analyzer]: https://oss-review-toolkit.org/ort/docs/tools/analyzer diff --git a/plugins/package-managers/bitbake/build.gradle.kts b/plugins/package-managers/bitbake/build.gradle.kts new file mode 100644 index 0000000000000..57901f4466982 --- /dev/null +++ b/plugins/package-managers/bitbake/build.gradle.kts @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +plugins { + // Apply precompiled plugins. + id("ort-library-conventions") +} + +dependencies { + api(project(":analyzer")) + api(project(":model")) + + implementation(project(":utils:common-utils")) + implementation(project(":plugins:package-managers:spdx-package-manager")) + + funTestImplementation(project(":downloader")) + funTestImplementation(project(":plugins:version-control-systems:git-version-control-system")) + funTestImplementation(testFixtures(project(":analyzer"))) +} diff --git a/plugins/package-managers/bitbake/src/funTest/kotlin/BitBakeToolFunTest.kt b/plugins/package-managers/bitbake/src/funTest/kotlin/BitBakeToolFunTest.kt new file mode 100644 index 0000000000000..71f1e47a83021 --- /dev/null +++ b/plugins/package-managers/bitbake/src/funTest/kotlin/BitBakeToolFunTest.kt @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.packagemanagers.bitbake + +import io.kotest.core.spec.style.WordSpec +import io.kotest.engine.spec.tempdir +import io.kotest.matchers.collections.beEmpty +import io.kotest.matchers.collections.shouldHaveSize +import io.kotest.matchers.result.shouldBeSuccess +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldMatch + +import org.ossreviewtoolkit.analyzer.Analyzer +import org.ossreviewtoolkit.analyzer.create +import org.ossreviewtoolkit.model.Identifier +import org.ossreviewtoolkit.model.VcsInfo +import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.config.AnalyzerConfiguration +import org.ossreviewtoolkit.plugins.versioncontrolsystems.git.Git +import org.ossreviewtoolkit.utils.test.ExpensiveTag +import org.ossreviewtoolkit.utils.test.shouldNotBeNull + +class BitBakeToolFunTest : WordSpec({ + "BitBake" should { + "get the version correctly" { + val bitBake = create("BitBake") as BitBake + + val version = bitBake.getBitBakeVersion(tempdir()) + + version shouldMatch "\\d+\\.\\d+\\.\\d+" + } + } + + "Analyzing recipes from Poky" should { + val projectDir = tempdir() + val pokyVcsInfo = VcsInfo(VcsType.GIT, "https://git.yoctoproject.org/poky", "kirkstone-4.0.17") + + Git().run { + val workingTree = initWorkingTree(projectDir, pokyVcsInfo) + updateWorkingTree(workingTree, pokyVcsInfo.revision) + } shouldBeSuccess pokyVcsInfo.revision + + "create an SPDX file for the 'quilt-native' package" { + val recipeFileName = "quilt-native_0.67.bb" + val result = Analyzer(AnalyzerConfiguration()).run { + val fileInfo = findManagedFiles(projectDir) + val singleFileInfo = fileInfo.copy( + managedFiles = fileInfo.managedFiles.map { (packageManager, definitionsFiles) -> + packageManager to definitionsFiles.filter { it.name == recipeFileName } + }.toMap() + ) + analyze(singleFileInfo) + } + + result.analyzer?.result shouldNotBeNull { + projects shouldHaveSize 1 + + with(projects.single()) { + id shouldBe Identifier("BitBake:OpenEmbedded ():quilt-native:0.67") + declaredLicenses shouldBe setOf("GPL-2.0-only") + homepageUrl shouldBe "http://savannah.nongnu.org/projects/quilt/" + scopes should beEmpty() + } + } + } + + "create a SPDX files for the 'xmlto' package".config(tags = setOf(ExpensiveTag)) { + val recipeFileName = "xmlto_0.0.28.bb" + val result = Analyzer(AnalyzerConfiguration()).run { + val fileInfo = findManagedFiles(projectDir) + val singleFileInfo = fileInfo.copy( + managedFiles = fileInfo.managedFiles.map { (packageManager, definitionsFiles) -> + packageManager to definitionsFiles.filter { it.name == recipeFileName } + }.toMap() + ) + analyze(singleFileInfo) + } + + result.analyzer?.result shouldNotBeNull { + projects shouldHaveSize 90 + } + } + } +}) diff --git a/plugins/package-managers/bitbake/src/main/kotlin/BitBake.kt b/plugins/package-managers/bitbake/src/main/kotlin/BitBake.kt new file mode 100644 index 0000000000000..b3b65b5a8073c --- /dev/null +++ b/plugins/package-managers/bitbake/src/main/kotlin/BitBake.kt @@ -0,0 +1,166 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.packagemanagers.bitbake + +import java.io.File + +import kotlin.time.measureTime + +import org.apache.logging.log4j.kotlin.logger + +import org.ossreviewtoolkit.analyzer.AbstractPackageManagerFactory +import org.ossreviewtoolkit.analyzer.PackageManager +import org.ossreviewtoolkit.analyzer.PackageManagerResult +import org.ossreviewtoolkit.model.ProjectAnalyzerResult +import org.ossreviewtoolkit.model.config.AnalyzerConfiguration +import org.ossreviewtoolkit.model.config.RepositoryConfiguration +import org.ossreviewtoolkit.plugins.packagemanagers.spdx.SpdxDocumentFile +import org.ossreviewtoolkit.utils.common.ProcessCapture +import org.ossreviewtoolkit.utils.common.getCommonParentFile +import org.ossreviewtoolkit.utils.common.safeDeleteRecursively +import org.ossreviewtoolkit.utils.common.withoutPrefix +import org.ossreviewtoolkit.utils.ort.createOrtTempDir +import org.ossreviewtoolkit.utils.ort.createOrtTempFile + +/** + * A package manager that uses OpenEmbedded's "bitbake" tool to create SPDX SBOMs [1][2] e.g. for Yocto distributions, + * and post-processes these into ORT analyzer results. + * + * [1]: https://docs.yoctoproject.org/dev/dev-manual/sbom.html + * [2]: https://dev.to/angrymane/create-spdx-with-yocto-2od9 + */ +class BitBake( + name: String, + analysisRoot: File, + analyzerConfig: AnalyzerConfiguration, + repoConfig: RepositoryConfiguration +) : PackageManager(name, analysisRoot, analyzerConfig, repoConfig) { + class Factory : AbstractPackageManagerFactory("BitBake") { + override val globsForDefinitionFiles = listOf("*.bb") + + override fun create( + analysisRoot: File, + analyzerConfig: AnalyzerConfiguration, + repoConfig: RepositoryConfiguration + ) = BitBake(type, analysisRoot, analyzerConfig, repoConfig) + } + + private val scriptFile by lazy { extractResourceToTempFile(BITBAKE_SCRIPT_NAME).apply { setExecutable(true) } } + private val spdxConfFile by lazy { extractResourceToTempFile(SPDX_CONF_NAME) } + + private val spdxManager by lazy { SpdxDocumentFile(name, analysisRoot, analyzerConfig, repoConfig) } + + override fun resolveDependencies(definitionFiles: List, labels: Map): PackageManagerResult { + val commonDefinitionDir = getCommonParentFile(definitionFiles) + val workingDir = requireNotNull(commonDefinitionDir.searchUpwardsForFile(INIT_SCRIPT_NAME)) { + "No '$INIT_SCRIPT_NAME' script file found for directory '$commonDefinitionDir'." + } + + logger.info { "Determined the working directory to be '$workingDir'." } + + val localVersion = getBitBakeVersion(workingDir) + val globalVersion = createOrtTempDir().let { dir -> + getBitBakeVersion(dir).also { dir.safeDeleteRecursively(force = true) } + } + + if (localVersion != globalVersion) { + logger.warn { "Local $managerName version $localVersion differs from global version $globalVersion." } + } + + val deployDirs = mutableSetOf() + + definitionFiles.forEach { definitionFile -> + val target = definitionFile.nameWithoutExtension.substringBeforeLast('_') + + val deployDir = getDeployDir(workingDir, target) + deployDirs += deployDir + + val spdxFile = deployDir.findSpdxFiles().find { it.name == "recipe-$target.spdx.json" } + if (spdxFile != null) { + logger.info { "Not creating SPDX files for target '$target' as it already exists at '$spdxFile'." } + } else { + logger.info { "Creating SPDX files for target '$target'..." } + + // This implicitly triggers the build and can take a very long time. + val duration = measureTime { createSpdx(workingDir, target) } + + logger.info { "Creating SPDX files for target '$target' took $duration." } + } + } + + if (!scriptFile.delete()) logger.warn { "Unable to delete the temporary '$scriptFile' file." } + if (!spdxConfFile.delete()) logger.warn { "Unable to delete the temporary '$spdxConfFile' file." } + + val commonDeployDir = deployDirs.singleOrNull() ?: getCommonParentFile(deployDirs) + val spdxFiles = commonDeployDir.findSpdxFiles().toList() + + logger.info { "Found ${spdxFiles.size} SPDX file(s) in '$commonDeployDir'." } + + return spdxManager.resolveDependencies(spdxFiles, labels) + } + + override fun resolveDependencies(definitionFile: File, labels: Map): List = + throw NotImplementedError("This function is not supported for $managerName.") + + private fun getDeployDir(workingDir: File, target: String): File { + val bitbakeEnv = runBitBake(workingDir, "-e", target) + return bitbakeEnv.stdout.lineSequence().mapNotNull { it.withoutPrefix("DEPLOY_DIR=") }.first() + .let { File(it.removeSurrounding("\"")) } + } + + private fun createSpdx(workingDir: File, target: String) = + runBitBake(workingDir, "-r", spdxConfFile.absolutePath, "-c", "create_spdx", target) + + private fun File.findSpdxFiles() = resolve("spdx").walk().filter { it.isFile && it.name.endsWith(".spdx.json") } + + private fun runBitBake(workingDir: File, vararg args: String): ProcessCapture = + ProcessCapture(scriptFile.absolutePath, workingDir.absolutePath, *args, workingDir = workingDir) + .requireSuccess() + + internal fun getBitBakeVersion(workingDir: File): String = + runBitBake(workingDir, "--version").stdout.lineSequence().first { + it.startsWith("BitBake Build Tool") + }.substringAfterLast(' ') + + private fun extractResourceToTempFile(resourceName: String): File { + val prefix = resourceName.substringBefore('.') + val suffix = resourceName.substringAfter(prefix) + val scriptFile = createOrtTempFile(prefix, suffix) + val script = checkNotNull(javaClass.getResource("/$resourceName")).readText() + + return scriptFile.apply { writeText(script) } + } +} + +private const val INIT_SCRIPT_NAME = "oe-init-build-env" +private const val BITBAKE_SCRIPT_NAME = "bitbake.sh" +private const val SPDX_CONF_NAME = "spdx.conf" + +private fun File.searchUpwardsForFile(searchFileName: String): File? { + if (!isDirectory) return null + + var currentDir: File? = absoluteFile + + while (currentDir != null && !currentDir.resolve(searchFileName).isFile) { + currentDir = currentDir.parentFile + } + + return currentDir +} diff --git a/plugins/package-managers/bitbake/src/main/resources/META-INF/services/org.ossreviewtoolkit.analyzer.PackageManagerFactory b/plugins/package-managers/bitbake/src/main/resources/META-INF/services/org.ossreviewtoolkit.analyzer.PackageManagerFactory new file mode 100644 index 0000000000000..464be750b02b3 --- /dev/null +++ b/plugins/package-managers/bitbake/src/main/resources/META-INF/services/org.ossreviewtoolkit.analyzer.PackageManagerFactory @@ -0,0 +1 @@ +org.ossreviewtoolkit.plugins.packagemanagers.bitbake.BitBake$Factory diff --git a/plugins/package-managers/bitbake/src/main/resources/bitbake.sh b/plugins/package-managers/bitbake/src/main/resources/bitbake.sh new file mode 100755 index 0000000000000..e2743cdd39c6f --- /dev/null +++ b/plugins/package-managers/bitbake/src/main/resources/bitbake.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Copyright (C) 2024 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +BUILD_DIR=$1; shift + +if [ -z "$BBPATH" ] && [ -f oe-init-build-env ]; then + # Initialize the build environment. + . oe-init-build-env "$BUILD_DIR" +fi + +bitbake "$@" diff --git a/plugins/package-managers/bitbake/src/main/resources/spdx.conf b/plugins/package-managers/bitbake/src/main/resources/spdx.conf new file mode 100644 index 0000000000000..078f60e7d38ba --- /dev/null +++ b/plugins/package-managers/bitbake/src/main/resources/spdx.conf @@ -0,0 +1,52 @@ +# Copyright (C) 2024 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +# Disable sanity checks to allow BitBake to run as root which is required in some Docker scenarios. +INHERIT:remove = "sanity" + +# Work around wget2, as shipping with Fedora 40, to not support `--passive-ftp`, see +# https://github.com/openembedded/bitbake/commit/f10e630fd7561746d835a4378e8777e78f56e44a. +FETCHCMD_wget = "/usr/bin/env wget -t 2 -T 30" + +# See https://docs.yoctoproject.org/ref-manual/variables.html#term-S for a general list of supported variables. +INHERIT += "create-spdx" + +# This option allows to add to SPDX output compressed archives of the files in the generated target packages. +# Such archives are available in tmp/deploy/spdx/MACHINE/packages/packagename.tar.zst under the Build Directory. +SPDX_ARCHIVE_PACKAGED = "1" + +# This option allows to add to SPDX output compressed archives of the sources for packages installed on the target. It +# currently only works when SPDX_INCLUDE_SOURCES is set. +# Such source archives are available in tmp/deploy/spdx/MACHINE/recipes/recipe-packagename.tar.zst under the Build +# Directory. +SPDX_ARCHIVE_SOURCES = "1" + +# This option allows to add a description of the source files used to build the host tools and the target packages, to +# the spdx.json files in tmp/deploy/spdx/MACHINE/recipes/ under the Build Directory. As a consequence, the spdx.json +# files under the by-namespace and packages subdirectories in tmp/deploy/spdx/MACHINE are also modified to include +# references to such source file descriptions. +SPDX_INCLUDE_SOURCES = "1" + +# This option could be used in order to change the prefix of spdxDocument and the prefix of documentNamespace. Older +# BitBake versions set this by default to "http://spdx.org/spdxdoc", which is wrong (see +# https://bugzilla.yoctoproject.org/show_bug.cgi?id=15398), so fix it up here. +SPDX_NAMESPACE_PREFIX = "https://spdx.org/spdxdocs" + +# This option makes the SPDX output more human-readable, using indentation and newlines, instead of the default output in +# a single line. The generated SPDX files are approximately 20% bigger, but this option is recommended if you want to +# inspect the SPDX output files with a text editor. +SPDX_PRETTY = "1" diff --git a/website/docs/tools/analyzer.md b/website/docs/tools/analyzer.md index cbacb5e7de738..be96d634157af 100644 --- a/website/docs/tools/analyzer.md +++ b/website/docs/tools/analyzer.md @@ -14,6 +14,7 @@ Currently, the following package managers (grouped by the programming language t * C / C++ * [Bazel](https://bazel.build/) (**experimental**) (limitations: see [open tasks](https://github.com/oss-review-toolkit/ort/issues/264)) + * [BitBake](https://docs.yoctoproject.org/bitbake/) (**experimental**) (limitations: slow performance, resolution of SPDX external document refs) * [Conan](https://conan.io/) * Also see: [SPDX documents](#analyzer-for-spdx-documents) * Dart / Flutter