From 1a10c3d7c6d38b2d5335fcd4c13e5644d9ef62e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paolo=20Chil=C3=A0?= Date: Fri, 27 Sep 2024 14:55:24 +0200 Subject: [PATCH 01/20] Fix ownership of component specs in elastic-agent docker image (#5616) Co-authored-by: Julien Lind --- .../packaging/templates/docker/Dockerfile.elastic-agent.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl index cd5906f5cf7..ccede203a05 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl @@ -163,7 +163,7 @@ COPY --chown={{ .user }}:{{ .user }} --from=home {{ $beatHome }} {{ $beatHome }} # create fleet.yml when running as non-root. RUN chmod 0777 {{ $beatHome }} && \ usermod -d {{ $beatHome}} {{ .user }} && \ - find {{ $beatHome }}/data/elastic-agent-{{ commit_short }}/components -name "*.yml*" -type f -exec chown root:root {} \; && \ + find {{ $beatHome }}/data/elastic-agent-{{ commit_short }}/components -name "*.yml*" -type f -exec chown {{ .user }}:{{ .user }} {} \; && \ true RUN mkdir /licenses From 2fecda23d5b834d66c26f9a16b35ca1898dadcfb Mon Sep 17 00:00:00 2001 From: Mauri de Souza Meneguzzo Date: Mon, 30 Sep 2024 12:32:39 -0300 Subject: [PATCH 02/20] add kubernetes v1.31 to test matrix (#5587) * add kubernetes v1.31 to test matrix * add 1.31 to buildkite pipeline --- .buildkite/integration.pipeline.yml | 2 +- .buildkite/pipeline.yml | 2 +- docs/test-framework-dev-guide.md | 2 +- pkg/testing/kubernetes/supported.go | 5 +++++ 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml index a5586585915..cccd2ca1908 100644 --- a/.buildkite/integration.pipeline.yml +++ b/.buildkite/integration.pipeline.yml @@ -106,7 +106,7 @@ steps: - label: "Kubernetes Integration tests" key: "k8s-integration-tests" env: - K8S_VERSION: "v1.30.2" + K8S_VERSION: "v1.31.0" KIND_VERSION: "v0.20.0" command: ".buildkite/scripts/steps/k8s-extended-tests.sh" artifact_paths: diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8027b39a961..174096fb28a 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -191,10 +191,10 @@ steps: matrix: setup: k8s_version: + - "1.31.0" - "1.30.0" - "1.29.4" - "1.28.9" - - "1.27.13" retry: manual: allowed: true diff --git a/docs/test-framework-dev-guide.md b/docs/test-framework-dev-guide.md index 67faeae30a5..ff837a09cb4 100644 --- a/docs/test-framework-dev-guide.md +++ b/docs/test-framework-dev-guide.md @@ -77,7 +77,7 @@ between, and it can be very specific or not very specific. - `TEST_PLATFORMS="linux/amd64/ubuntu/20.04 mage integration:test` to execute tests only on Ubuntu 20.04 ARM64. - `TEST_PLATFORMS="windows/amd64/2022 mage integration:test` to execute tests only on Windows Server 2022. - `TEST_PLATFORMS="linux/amd64 windows/amd64/2022 mage integration:test` to execute tests on Linux AMD64 and Windows Server 2022. -- `TEST_PLATFORMS="kubernetes/arm64/1.30.2/wolfi" mage integration:kubernetes` to execute kubernetes tests on Kubernetes version 1.30.2 with wolfi docker variant. +- `TEST_PLATFORMS="kubernetes/arm64/1.31.0/wolfi" mage integration:kubernetes` to execute kubernetes tests on Kubernetes version 1.31.0 with wolfi docker variant. > **_NOTE:_** This only filters down the tests based on the platform. It will not execute a tests on a platform unless > the test defines as supporting it. diff --git a/pkg/testing/kubernetes/supported.go b/pkg/testing/kubernetes/supported.go index e8d8f96cf1c..26f1bef3e6d 100644 --- a/pkg/testing/kubernetes/supported.go +++ b/pkg/testing/kubernetes/supported.go @@ -18,6 +18,11 @@ var arches = []string{define.AMD64, define.ARM64} // versions defines the list of supported version of Kubernetes. var versions = []define.OS{ + // Kubernetes 1.31 + { + Type: define.Kubernetes, + Version: "1.31.0", + }, // Kubernetes 1.30 { Type: define.Kubernetes, From 1eae51c7f6ae588401bd0c9d1ba6094cddb8f459 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Mon, 30 Sep 2024 12:45:01 -0400 Subject: [PATCH 03/20] Skip TestAPMConfig. (#5625) --- testing/integration/apm_propagation_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testing/integration/apm_propagation_test.go b/testing/integration/apm_propagation_test.go index 6325f941259..000a7641916 100644 --- a/testing/integration/apm_propagation_test.go +++ b/testing/integration/apm_propagation_test.go @@ -55,6 +55,9 @@ func TestAPMConfig(t *testing.T) { Group: Default, Stack: &define.Stack{}, }) + + t.Skip("https://github.com/elastic/elastic-agent/issues/5624; apm-server not working correctly") + f, err := define.NewFixtureFromLocalBuild(t, define.Version()) require.NoError(t, err) From c645a5af3f9597e5aecd6eafc6e1d06d1f0a555c Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Mon, 30 Sep 2024 14:32:37 -0400 Subject: [PATCH 04/20] Add ability to generate buildkite pipelines from integration testing framework (#5391) Adds the ability to generate the integration testing workload as buildkite pipelines. This doesn't actually complete the enablement. This just provides a great starting point for generating the buildkite steps. More work is still to be done to actually perform the work in the steps. --- magefile.go | 96 ++++- pkg/testing/buildkite/buildkite.go | 328 +++++++++++++++ pkg/testing/buildkite/steps.go | 25 ++ pkg/testing/common/batch.go | 21 + pkg/testing/common/build.go | 19 + pkg/testing/{runner => common}/config.go | 2 +- pkg/testing/{runner => common}/config_test.go | 6 +- .../provisioner.go => common/instance.go} | 64 +-- pkg/testing/common/logger.go | 11 + .../{runner => common}/prefix_output.go | 5 +- pkg/testing/common/runner.go | 45 +++ pkg/testing/common/stack.go | 76 ++++ pkg/testing/common/supported.go | 15 + pkg/testing/ess/serverless.go | 6 +- pkg/testing/ess/serverless_provisioner.go | 23 +- pkg/testing/ess/serverless_test.go | 5 +- pkg/testing/ess/statful_provisioner.go | 22 +- pkg/testing/kubernetes/image.go | 8 +- pkg/testing/kubernetes/kind/provisioner.go | 28 +- .../kubernetes.go => kubernetes/runner.go} | 38 +- pkg/testing/{runner => linux}/debian.go | 36 +- pkg/testing/{runner => linux}/linux.go | 9 +- pkg/testing/{runner => linux}/rhel.go | 18 +- pkg/testing/multipass/provisioner.go | 29 +- pkg/testing/ogc/provisioner.go | 23 +- pkg/testing/ogc/supported.go | 10 +- pkg/testing/runner/runner.go | 376 ++++-------------- pkg/testing/runner/runner_test.go | 47 +-- pkg/testing/{runner/ssh.go => ssh/client.go} | 96 +---- pkg/testing/ssh/file.go | 19 + pkg/testing/ssh/interface.go | 49 +++ pkg/testing/ssh/keys.go | 47 +++ pkg/testing/supported/batch.go | 183 +++++++++ .../{runner => supported}/supported.go | 78 ++-- .../{runner => supported}/supported_test.go | 15 +- pkg/testing/{runner => windows}/windows.go | 48 +-- 36 files changed, 1232 insertions(+), 694 deletions(-) create mode 100644 pkg/testing/buildkite/buildkite.go create mode 100644 pkg/testing/buildkite/steps.go create mode 100644 pkg/testing/common/batch.go create mode 100644 pkg/testing/common/build.go rename pkg/testing/{runner => common}/config.go (99%) rename pkg/testing/{runner => common}/config_test.go (98%) rename pkg/testing/{runner/provisioner.go => common/instance.go} (55%) create mode 100644 pkg/testing/common/logger.go rename pkg/testing/{runner => common}/prefix_output.go (90%) create mode 100644 pkg/testing/common/runner.go create mode 100644 pkg/testing/common/stack.go create mode 100644 pkg/testing/common/supported.go rename pkg/testing/{runner/kubernetes.go => kubernetes/runner.go} (66%) rename pkg/testing/{runner => linux}/debian.go (78%) rename pkg/testing/{runner => linux}/linux.go (93%) rename pkg/testing/{runner => linux}/rhel.go (82%) rename pkg/testing/{runner/ssh.go => ssh/client.go} (70%) create mode 100644 pkg/testing/ssh/file.go create mode 100644 pkg/testing/ssh/interface.go create mode 100644 pkg/testing/ssh/keys.go create mode 100644 pkg/testing/supported/batch.go rename pkg/testing/{runner => supported}/supported.go (80%) rename pkg/testing/{runner => supported}/supported_test.go (88%) rename pkg/testing/{runner => windows}/windows.go (83%) diff --git a/magefile.go b/magefile.go index 5408c0a8604..fe5c55489eb 100644 --- a/magefile.go +++ b/magefile.go @@ -40,6 +40,8 @@ import ( "github.com/elastic/elastic-agent/dev-tools/mage/downloads" "github.com/elastic/elastic-agent/dev-tools/mage/manifest" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact/download" + "github.com/elastic/elastic-agent/pkg/testing/buildkite" + tcommon "github.com/elastic/elastic-agent/pkg/testing/common" "github.com/elastic/elastic-agent/pkg/testing/define" "github.com/elastic/elastic-agent/pkg/testing/ess" "github.com/elastic/elastic-agent/pkg/testing/kubernetes/kind" @@ -2134,12 +2136,12 @@ func askForVM() (runner.StateInstance, error) { return instances[id], nil } -func askForStack() (runner.Stack, error) { +func askForStack() (tcommon.Stack, error) { mg.Deps(Integration.Stacks) state, err := readFrameworkState() if err != nil { - return runner.Stack{}, fmt.Errorf("could not read state file: %w", err) + return tcommon.Stack{}, fmt.Errorf("could not read state file: %w", err) } if len(state.Stacks) == 1 { @@ -2150,17 +2152,17 @@ func askForStack() (runner.Stack, error) { id := 0 fmt.Print("Stack number: ") if _, err := fmt.Scanf("%d", &id); err != nil { - return runner.Stack{}, fmt.Errorf("cannot read Stack number: %w", err) + return tcommon.Stack{}, fmt.Errorf("cannot read Stack number: %w", err) } if id >= len(state.Stacks) { - return runner.Stack{}, fmt.Errorf("Invalid Stack number, it must be between 0 and %d", len(state.Stacks)-1) + return tcommon.Stack{}, fmt.Errorf("Invalid Stack number, it must be between 0 and %d", len(state.Stacks)-1) } return state.Stacks[id], nil } -func generateEnvFile(stack runner.Stack) error { +func generateEnvFile(stack tcommon.Stack) error { fileExists := true stat, err := os.Stat("./env.sh") if err != nil { @@ -2471,6 +2473,57 @@ func (Integration) TestOnRemote(ctx context.Context) error { return nil } +func (Integration) Buildkite() error { + goTestFlags := os.Getenv("GOTEST_FLAGS") + batches, err := define.DetermineBatches("testing/integration", goTestFlags, "integration") + if err != nil { + return fmt.Errorf("failed to determine batches: %w", err) + } + agentVersion, agentStackVersion, err := getTestRunnerVersions() + if err != nil { + return fmt.Errorf("failed to get agent versions: %w", err) + } + goVersion, err := mage.DefaultBeatBuildVariableSources.GetGoVersion() + if err != nil { + return fmt.Errorf("failed to get go versions: %w", err) + } + + cfg := tcommon.Config{ + AgentVersion: agentVersion, + StackVersion: agentStackVersion, + GOVersion: goVersion, + Platforms: testPlatforms(), + Packages: testPackages(), + Groups: testGroups(), + Matrix: false, + VerboseMode: mg.Verbose(), + TestFlags: goTestFlags, + } + + steps, err := buildkite.GenerateSteps(cfg, batches...) + if err != nil { + return fmt.Errorf("error generating buildkite steps: %w", err) + } + + // write output to steps.yaml + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("error getting current working directory: %w", err) + } + ymlFilePath := filepath.Join(cwd, "steps.yml") + file, err := os.Create(ymlFilePath) + if err != nil { + return fmt.Errorf("error creating file: %w", err) + } + defer file.Close() + if _, err := file.WriteString(steps); err != nil { + return fmt.Errorf("error writing to file: %w", err) + } + + fmt.Printf(">>> Generated buildkite steps written to: %s\n", ymlFilePath) + return nil +} + func integRunner(ctx context.Context, matrix bool, singleTest string) error { if _, ok := ctx.Deadline(); !ok { // If the context doesn't have a timeout (usually via the mage -t option), give it one. @@ -2541,18 +2594,14 @@ func integRunnerOnce(ctx context.Context, matrix bool, singleTest string) (int, return results.Failures, nil } -func createTestRunner(matrix bool, singleTest string, goTestFlags string, batches ...define.Batch) (*runner.Runner, error) { - goVersion, err := mage.DefaultBeatBuildVariableSources.GetGoVersion() - if err != nil { - return nil, err - } - +func getTestRunnerVersions() (string, string, error) { + var err error agentStackVersion := os.Getenv("AGENT_STACK_VERSION") agentVersion := os.Getenv("AGENT_VERSION") if agentVersion == "" { agentVersion, err = mage.DefaultBeatBuildVariableSources.GetBeatVersion() if err != nil { - return nil, err + return "", "", err } if agentStackVersion == "" { // always use snapshot for stack version @@ -2568,6 +2617,21 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche if agentStackVersion == "" { agentStackVersion = agentVersion } + + return agentVersion, agentStackVersion, nil +} + +func createTestRunner(matrix bool, singleTest string, goTestFlags string, batches ...define.Batch) (*runner.Runner, error) { + goVersion, err := mage.DefaultBeatBuildVariableSources.GetGoVersion() + if err != nil { + return nil, err + } + + agentVersion, agentStackVersion, err := getTestRunnerVersions() + if err != nil { + return nil, err + } + agentBuildDir := os.Getenv("AGENT_BUILD_DIR") if agentBuildDir == "" { agentBuildDir = filepath.Join("build", "distributions") @@ -2606,7 +2670,7 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche Datacenter: datacenter, } - var instanceProvisioner runner.InstanceProvisioner + var instanceProvisioner tcommon.InstanceProvisioner instanceProvisionerMode := os.Getenv("INSTANCE_PROVISIONER") switch instanceProvisionerMode { case "", ogc.Name: @@ -2619,7 +2683,6 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche default: return nil, fmt.Errorf("INSTANCE_PROVISIONER environment variable must be one of 'ogc' or 'multipass', not %s", instanceProvisionerMode) } - fmt.Printf(">>>> Using %s instance provisioner\n", instanceProvisionerMode) email, err := ogcCfg.ClientEmail() if err != nil { @@ -2632,7 +2695,7 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche Region: essRegion, } - var stackProvisioner runner.StackProvisioner + var stackProvisioner tcommon.StackProvisioner stackProvisionerMode := os.Getenv("STACK_PROVISIONER") switch stackProvisionerMode { case "", ess.ProvisionerStateful: @@ -2654,7 +2717,6 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche ess.ProvisionerServerless, stackProvisionerMode) } - fmt.Printf(">>>> Using %s stack provisioner\n", stackProvisionerMode) timestamp := timestampEnabled() @@ -2684,7 +2746,7 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche diagDir := filepath.Join("build", "diagnostics") _ = os.MkdirAll(diagDir, 0755) - cfg := runner.Config{ + cfg := tcommon.Config{ AgentVersion: agentVersion, StackVersion: agentStackVersion, BuildDir: agentBuildDir, diff --git a/pkg/testing/buildkite/buildkite.go b/pkg/testing/buildkite/buildkite.go new file mode 100644 index 00000000000..7b0bac34bc1 --- /dev/null +++ b/pkg/testing/buildkite/buildkite.go @@ -0,0 +1,328 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package buildkite + +import ( + "errors" + "fmt" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/elastic/elastic-agent/pkg/testing/common" + "github.com/elastic/elastic-agent/pkg/testing/define" + "github.com/elastic/elastic-agent/pkg/testing/supported" +) + +const ( + defaultProvider = "gcp" + defaultImageProject = "elastic-images-qa" + defaultAMD64MachineType = "n1-standard-8" + defaultARM64MachineType = "t2a-standard-8" +) + +var ( + bkStackAgent = StepAgent{ + Provider: "gcp", + ImageProject: "elastic-images-qa", + MachineType: "n1-standard-8", // does it need to be this large? + Image: "family/platform-ingest-elastic-agent-ubuntu-2204", // is this the correct image for creating a stack? + } + bkUbuntuAMD64_2004 = StepAgent{ + Provider: defaultProvider, + ImageProject: defaultImageProject, + MachineType: defaultAMD64MachineType, + Image: "family/platform-ingest-elastic-agent-ubuntu-2004", + } + bkUbuntuAMD64_2204 = StepAgent{ + Provider: defaultProvider, + ImageProject: defaultImageProject, + MachineType: defaultAMD64MachineType, + Image: "family/platform-ingest-elastic-agent-ubuntu-2204", + } + bkUbuntuAMD64_2404 = StepAgent{ + Provider: defaultProvider, + ImageProject: defaultImageProject, + MachineType: defaultAMD64MachineType, + Image: "family/platform-ingest-elastic-agent-ubuntu-2404", + } + bkUbuntuARM64_2004 = StepAgent{ + Provider: defaultProvider, + ImageProject: defaultImageProject, + MachineType: defaultARM64MachineType, + Image: "family/platform-ingest-elastic-agent-ubuntu-2004-arm", + } + bkUbuntuARM64_2204 = StepAgent{ + Provider: defaultProvider, + ImageProject: defaultImageProject, + MachineType: defaultARM64MachineType, + Image: "family/platform-ingest-elastic-agent-ubuntu-2204-arm", + } + bkUbuntuARM64_2404 = StepAgent{ + Provider: defaultProvider, + ImageProject: defaultImageProject, + MachineType: defaultARM64MachineType, + Image: "family/platform-ingest-elastic-agent-ubuntu-2404-arm", + } + bkRHELAMD64_8 = StepAgent{ + Provider: defaultProvider, + ImageProject: defaultImageProject, + MachineType: defaultAMD64MachineType, + Image: "family/platform-ingest-elastic-agent-rhel-8", + } + bkRHELARM64_8 = StepAgent{ + Provider: defaultProvider, + ImageProject: defaultImageProject, + MachineType: defaultARM64MachineType, + Image: "family/platform-ingest-elastic-agent-rhel-8-arm", + } + bkWindowsAMD64_2019 = StepAgent{ + Provider: defaultProvider, + ImageProject: defaultImageProject, + MachineType: defaultAMD64MachineType, + Image: "family/platform-ingest-elastic-agent-windows-2019", + } + bkWindowsAMD64_2019_Core = StepAgent{ + Provider: defaultProvider, + ImageProject: defaultImageProject, + MachineType: defaultAMD64MachineType, + Image: "family/platform-ingest-elastic-agent-windows-2019-core", + } + bkWindowsAMD64_2022 = StepAgent{ + Provider: defaultProvider, + ImageProject: defaultImageProject, + MachineType: defaultAMD64MachineType, + Image: "family/platform-ingest-elastic-agent-windows-2022", + } + bkWindowsAMD64_2022_Core = StepAgent{ + Provider: defaultProvider, + ImageProject: defaultImageProject, + MachineType: defaultAMD64MachineType, + Image: "family/platform-ingest-elastic-agent-windows-2022-core", + } +) + +// getAgent returns the agent to use for the provided batch. +func getAgent(os common.SupportedOS) (StepAgent, error) { + switch os.Arch { + case define.AMD64: + switch os.Type { + case define.Linux: + switch os.Distro { + case "", "ubuntu": // default is Ubuntu + switch os.Version { + case "20.04": + return bkUbuntuAMD64_2004, nil + case "22.04": + return bkUbuntuAMD64_2204, nil + case "", "24.04": // default is 24.04 + return bkUbuntuAMD64_2404, nil + default: + return StepAgent{}, fmt.Errorf("unknown ubuntu version: %s", os.Version) + } + case "rhel": + switch os.Version { + case "", "8": // default is 8 + return bkRHELAMD64_8, nil + default: + return StepAgent{}, fmt.Errorf("unknown rhel version: %s", os.Version) + } + } + case define.Kubernetes: + return bkUbuntuAMD64_2404, nil + case define.Windows: + switch os.Version { + case "2019": + return bkWindowsAMD64_2019, nil + case "2019-core": + return bkWindowsAMD64_2019_Core, nil + case "", "2022": // default is 2022 + return bkWindowsAMD64_2022, nil + case "2022-core": + return bkWindowsAMD64_2022_Core, nil + default: + return StepAgent{}, fmt.Errorf("unknown windows version: %s", os.Version) + } + } + case define.ARM64: + switch os.Type { + case define.Linux: + switch os.Distro { + case "", "ubuntu": // default is Ubuntu + switch os.Version { + case "20.04": + return bkUbuntuARM64_2004, nil + case "22.04": + return bkUbuntuARM64_2204, nil + case "", "24.04": // default is 24.04 + return bkUbuntuARM64_2404, nil + default: + return StepAgent{}, fmt.Errorf("unknown ubuntu version: %s", os.Version) + } + case "rhel": + switch os.Version { + case "", "8": // default is 8 + return bkRHELARM64_8, nil + default: + return StepAgent{}, fmt.Errorf("unknown rhel version: %s", os.Version) + } + } + case define.Kubernetes: + return bkUbuntuARM64_2404, nil + case define.Windows: + return StepAgent{}, errors.New("windows ARM support not enabled") + case define.Darwin: + return StepAgent{}, errors.New("darwin ARM support not enabled") + default: + return StepAgent{}, fmt.Errorf("unknown OS type: %s", os.Type) + } + default: + return StepAgent{}, fmt.Errorf("unknown architecture: %s", os.Arch) + } + return StepAgent{}, fmt.Errorf("case missing for %+v", os) +} + +func getCommand(b common.OSBatch) string { + if b.OS.Type == define.Linux { + return "mage integration:testOnRemote" + } + return "TODO" +} + +func shouldSkip(os common.SupportedOS) bool { + if os.Arch == define.AMD64 && os.Type == define.Linux { + // currently only linux/amd64 is being supported + // (but all steps are generated) + return false + } + return true +} + +// GenerateSteps returns a computed set of steps to run the integration tests on buildkite. +func GenerateSteps(cfg common.Config, batches ...define.Batch) (string, error) { + stackSteps := map[string]Step{} + stackTeardown := map[string][]string{} + var steps []Step + + // create the supported batches first + platforms, err := cfg.GetPlatforms() + if err != nil { + return "", err + } + osBatches, err := supported.CreateBatches(batches, platforms, cfg.Groups, cfg.Matrix, cfg.SingleTest) + if err != nil { + return "", err + } + + // create the stack steps first + for _, lb := range osBatches { + if !lb.Skip && lb.Batch.Stack != nil { + if lb.Batch.Stack.Version == "" { + // no version defined on the stack; set it to the defined stack version + lb.Batch.Stack.Version = cfg.StackVersion + } + _, ok := stackSteps[lb.Batch.Stack.Version] + if !ok { + // add a step for creating the stack + stackKey := getStackKey(lb.Batch.Stack) + stackStep := Step{ + Label: fmt.Sprintf("Integration Stack: %s", lb.Batch.Stack.Version), + Key: stackKey, + Command: "false", + Agents: []StepAgent{bkStackAgent}, + } + steps = append(steps, stackStep) + stackSteps[lb.Batch.Stack.Version] = stackStep + stackTeardown[stackKey] = append(stackTeardown[stackKey], stackKey) + } + } + } + + // generate the steps for the tests + for _, lb := range osBatches { + if lb.Skip { + continue + } + agentStep, err := getAgent(lb.OS) + if err != nil { + return "", fmt.Errorf("unable to get machine and image: %w", err) + } + if len(lb.Batch.Tests) > 0 { + var step Step + step.Label = fmt.Sprintf("Integration Test (non-sudo): %s", lb.ID) + step.Key = fmt.Sprintf("integration-non-sudo-%s", lb.ID) + if lb.Batch.Stack != nil { + stackKey := getStackKey(lb.Batch.Stack) + step.DependsOn = append(step.DependsOn, stackKey) + stackTeardown[stackKey] = append(stackTeardown[stackKey], step.Key) + } + step.ArtifactPaths = []string{"build/**"} + step.Agents = []StepAgent{agentStep} + step.Env = map[string]string{ + "AGENT_VERSION": cfg.AgentVersion, + "TEST_DEFINE_PREFIX": step.Key, + "TEST_DEFINE_TESTS": strings.Join(getTestNames(lb.Batch.Tests), ","), + } + step.Command = getCommand(lb) + step.Skip = shouldSkip(lb.OS) + steps = append(steps, step) + } + if len(lb.Batch.SudoTests) > 0 { + var step Step + step.Label = fmt.Sprintf("Integration Test (sudo): %s", lb.ID) + step.Key = fmt.Sprintf("integration-sudo-%s", lb.ID) + if lb.Batch.Stack != nil { + stackKey := getStackKey(lb.Batch.Stack) + step.DependsOn = append(step.DependsOn, stackKey) + stackTeardown[stackKey] = append(stackTeardown[stackKey], step.Key) + } + step.ArtifactPaths = []string{"build/**"} + step.Agents = []StepAgent{agentStep} + step.Env = map[string]string{ + "AGENT_VERSION": cfg.AgentVersion, + "TEST_DEFINE_PREFIX": step.Key, + "TEST_DEFINE_TESTS": strings.Join(getTestNames(lb.Batch.SudoTests), ","), + } + step.Command = getCommand(lb) + step.Skip = shouldSkip(lb.OS) + steps = append(steps, step) + } + } + + // add the teardown steps for the stacks + for _, step := range stackSteps { + steps = append(steps, Step{ + Label: fmt.Sprintf("Teardown: %s", step.Label), + Key: fmt.Sprintf("teardown-%s", step.Key), + DependsOn: stackTeardown[step.Key], + AllowDependencyFailure: true, + Command: "false", + Agents: []StepAgent{bkStackAgent}, + }) + } + + yamlOutput, err := yaml.Marshal(Step{ + Steps: steps, + }) + if err != nil { + return "", fmt.Errorf("unable to marshal yaml: %w", err) + } + return string(yamlOutput), nil +} + +func getTestNames(pt []define.BatchPackageTests) []string { + var tests []string + for _, pkg := range pt { + for _, test := range pkg.Tests { + tests = append(tests, fmt.Sprintf("%s:%s", pkg.Name, test.Name)) + } + } + return tests +} + +func getStackKey(s *define.Stack) string { + version := strings.Replace(s.Version, ".", "-", -1) + return fmt.Sprintf("integration-stack-%s", version) +} diff --git a/pkg/testing/buildkite/steps.go b/pkg/testing/buildkite/steps.go new file mode 100644 index 00000000000..0181eea6ba2 --- /dev/null +++ b/pkg/testing/buildkite/steps.go @@ -0,0 +1,25 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package buildkite + +type StepAgent struct { + Provider string `json:"provider,omitempty" yaml:"provider,omitempty"` + ImageProject string `json:"imageProject,omitempty" yaml:"imageProject,omitempty"` + MachineType string `json:"machineType,omitempty" yaml:"machineType,omitempty"` + Image string `json:"image,omitempty" yaml:"image,omitempty"` +} + +type Step struct { + Key string `json:"key,omitempty" yaml:"key,omitempty"` + Label string `json:"label,omitempty" yaml:"label,omitempty"` + Command string `json:"command,omitempty" yaml:"command,omitempty"` + Env map[string]string `json:"env,omitempty" yaml:"env,omitempty"` + ArtifactPaths []string `json:"artifact_paths,omitempty" yaml:"artifact_paths,omitempty"` + Agents []StepAgent `json:"agents,omitempty" yaml:"agents,omitempty"` + DependsOn []string `json:"depends_on,omitempty" yaml:"depends_on,omitempty"` + AllowDependencyFailure bool `json:"allow_dependency_failure,omitempty" yaml:"allow_dependency_failure,omitempty"` + Steps []Step `json:"steps,omitempty" yaml:"steps,omitempty"` + Skip bool `json:"skip,omitempty" yaml:"skip,omitempty"` +} diff --git a/pkg/testing/common/batch.go b/pkg/testing/common/batch.go new file mode 100644 index 00000000000..ade24e98826 --- /dev/null +++ b/pkg/testing/common/batch.go @@ -0,0 +1,21 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package common + +import ( + "github.com/elastic/elastic-agent/pkg/testing/define" +) + +// OSBatch defines the mapping between a SupportedOS and a define.Batch. +type OSBatch struct { + // ID is the unique ID for the batch. + ID string + // LayoutOS provides all the OS information to create an instance. + OS SupportedOS + // Batch defines the batch of tests to run on this layout. + Batch define.Batch + // Skip defines if this batch will be skipped because no supported layout exists yet. + Skip bool +} diff --git a/pkg/testing/common/build.go b/pkg/testing/common/build.go new file mode 100644 index 00000000000..044584f8eb0 --- /dev/null +++ b/pkg/testing/common/build.go @@ -0,0 +1,19 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package common + +// Build describes a build and its paths. +type Build struct { + // Version of the Elastic Agent build. + Version string + // Type of OS this build is for. + Type string + // Arch is architecture this build is for. + Arch string + // Path is the path to the build. + Path string + // SHA512 is the path to the SHA512 file. + SHA512Path string +} diff --git a/pkg/testing/runner/config.go b/pkg/testing/common/config.go similarity index 99% rename from pkg/testing/runner/config.go rename to pkg/testing/common/config.go index 8840c57c96d..cfed83cca61 100644 --- a/pkg/testing/runner/config.go +++ b/pkg/testing/common/config.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package runner +package common import ( "errors" diff --git a/pkg/testing/runner/config_test.go b/pkg/testing/common/config_test.go similarity index 98% rename from pkg/testing/runner/config_test.go rename to pkg/testing/common/config_test.go index df3f663a37d..c98d9493285 100644 --- a/pkg/testing/runner/config_test.go +++ b/pkg/testing/common/config_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package runner +package common import ( "errors" @@ -78,12 +78,12 @@ func TestConfig_GetPlatforms(t *testing.T) { { Type: define.Linux, Arch: define.AMD64, - Distro: Ubuntu, + Distro: "ubuntu", }, { Type: define.Linux, Arch: define.ARM64, - Distro: Ubuntu, + Distro: "ubuntu", Version: "22.04", }, { diff --git a/pkg/testing/runner/provisioner.go b/pkg/testing/common/instance.go similarity index 55% rename from pkg/testing/runner/provisioner.go rename to pkg/testing/common/instance.go index 44d4e7c5d50..83d2f7ed95c 100644 --- a/pkg/testing/runner/provisioner.go +++ b/pkg/testing/common/instance.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package runner +package common import ( "context" @@ -65,65 +65,3 @@ type InstanceProvisioner interface { // Clean cleans up all provisioned resources. Clean(ctx context.Context, cfg Config, instances []Instance) error } - -// Stack is a created stack. -type Stack struct { - // ID is the identifier of the instance. - // - // This must be the same ID used for requesting a stack. - ID string `yaml:"id"` - - // Provisioner is the stack provisioner. See STACK_PROVISIONER environment - // variable for the supported provisioners. - Provisioner string `yaml:"provisioner"` - - // Version is the version of the stack. - Version string `yaml:"version"` - - // Ready determines if the stack is ready to be used. - Ready bool `yaml:"ready"` - - // Elasticsearch is the URL to communicate with elasticsearch. - Elasticsearch string `yaml:"elasticsearch"` - - // Kibana is the URL to communication with kibana. - Kibana string `yaml:"kibana"` - - // Username is the username. - Username string `yaml:"username"` - - // Password is the password. - Password string `yaml:"password"` - - // Internal holds internal information used by the provisioner. - // Best to not touch the contents of this, and leave it be for - // the provisioner. - Internal map[string]interface{} `yaml:"internal"` -} - -// StackRequest request for a new stack. -type StackRequest struct { - // ID is the unique ID for the stack. - ID string `yaml:"id"` - - // Version is the version of the stack. - Version string `yaml:"version"` -} - -// StackProvisioner performs the provisioning of stacks. -type StackProvisioner interface { - // Name returns the name of the stack provisioner. - Name() string - - // SetLogger sets the logger for it to use. - SetLogger(l Logger) - - // Create creates a stack. - Create(ctx context.Context, request StackRequest) (Stack, error) - - // WaitForReady should block until the stack is ready or the context is cancelled. - WaitForReady(ctx context.Context, stack Stack) (Stack, error) - - // Delete deletes the stack. - Delete(ctx context.Context, stack Stack) error -} diff --git a/pkg/testing/common/logger.go b/pkg/testing/common/logger.go new file mode 100644 index 00000000000..061678b5334 --- /dev/null +++ b/pkg/testing/common/logger.go @@ -0,0 +1,11 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package common + +// Logger is a simple logging interface used by each runner type. +type Logger interface { + // Logf logs the message for this runner. + Logf(format string, args ...any) +} diff --git a/pkg/testing/runner/prefix_output.go b/pkg/testing/common/prefix_output.go similarity index 90% rename from pkg/testing/runner/prefix_output.go rename to pkg/testing/common/prefix_output.go index e3c1a4cbbe8..b3eb9822570 100644 --- a/pkg/testing/runner/prefix_output.go +++ b/pkg/testing/common/prefix_output.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package runner +package common import ( "bytes" @@ -16,7 +16,8 @@ type prefixOutput struct { remainder []byte } -func newPrefixOutput(logger Logger, prefix string) *prefixOutput { +// NewPrefixOutput creates a prefix output `io.Writer`. +func NewPrefixOutput(logger Logger, prefix string) *prefixOutput { return &prefixOutput{ logger: logger, prefix: prefix, diff --git a/pkg/testing/common/runner.go b/pkg/testing/common/runner.go new file mode 100644 index 00000000000..112282d27ec --- /dev/null +++ b/pkg/testing/common/runner.go @@ -0,0 +1,45 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package common + +import ( + "context" + + "github.com/elastic/elastic-agent/pkg/testing/define" + "github.com/elastic/elastic-agent/pkg/testing/ssh" +) + +// OSRunnerPackageResult is the result for each package. +type OSRunnerPackageResult struct { + // Name is the package name. + Name string + // Output is the raw test output. + Output []byte + // XMLOutput is the XML Junit output. + XMLOutput []byte + // JSONOutput is the JSON output. + JSONOutput []byte +} + +// OSRunnerResult is the result of the test run provided by a OSRunner. +type OSRunnerResult struct { + // Packages is the results for each package. + Packages []OSRunnerPackageResult + + // SudoPackages is the results for each package that need to run as sudo. + SudoPackages []OSRunnerPackageResult +} + +// OSRunner provides an interface to run the tests on the OS. +type OSRunner interface { + // Prepare prepares the runner to actual run on the host. + Prepare(ctx context.Context, sshClient ssh.SSHClient, logger Logger, arch string, goVersion string) error + // Copy places the required files on the host. + Copy(ctx context.Context, sshClient ssh.SSHClient, logger Logger, repoArchive string, builds []Build) error + // Run runs the actual tests and provides the result. + Run(ctx context.Context, verbose bool, sshClient ssh.SSHClient, logger Logger, agentVersion string, prefix string, batch define.Batch, env map[string]string) (OSRunnerResult, error) + // Diagnostics gathers any diagnostics from the host. + Diagnostics(ctx context.Context, sshClient ssh.SSHClient, logger Logger, destination string) error +} diff --git a/pkg/testing/common/stack.go b/pkg/testing/common/stack.go new file mode 100644 index 00000000000..3047b340ea0 --- /dev/null +++ b/pkg/testing/common/stack.go @@ -0,0 +1,76 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package common + +import "context" + +// Stack is a created stack. +type Stack struct { + // ID is the identifier of the instance. + // + // This must be the same ID used for requesting a stack. + ID string `yaml:"id"` + + // Provisioner is the stack provisioner. See STACK_PROVISIONER environment + // variable for the supported provisioners. + Provisioner string `yaml:"provisioner"` + + // Version is the version of the stack. + Version string `yaml:"version"` + + // Ready determines if the stack is ready to be used. + Ready bool `yaml:"ready"` + + // Elasticsearch is the URL to communicate with elasticsearch. + Elasticsearch string `yaml:"elasticsearch"` + + // Kibana is the URL to communication with kibana. + Kibana string `yaml:"kibana"` + + // Username is the username. + Username string `yaml:"username"` + + // Password is the password. + Password string `yaml:"password"` + + // Internal holds internal information used by the provisioner. + // Best to not touch the contents of this, and leave it be for + // the provisioner. + Internal map[string]interface{} `yaml:"internal"` +} + +// Same returns true if other is the same stack as this one. +// Two stacks are considered the same if their provisioner and ID are the same. +func (s Stack) Same(other Stack) bool { + return s.Provisioner == other.Provisioner && + s.ID == other.ID +} + +// StackRequest request for a new stack. +type StackRequest struct { + // ID is the unique ID for the stack. + ID string `yaml:"id"` + + // Version is the version of the stack. + Version string `yaml:"version"` +} + +// StackProvisioner performs the provisioning of stacks. +type StackProvisioner interface { + // Name returns the name of the stack provisioner. + Name() string + + // SetLogger sets the logger for it to use. + SetLogger(l Logger) + + // Create creates a stack. + Create(ctx context.Context, request StackRequest) (Stack, error) + + // WaitForReady should block until the stack is ready or the context is cancelled. + WaitForReady(ctx context.Context, stack Stack) (Stack, error) + + // Delete deletes the stack. + Delete(ctx context.Context, stack Stack) error +} diff --git a/pkg/testing/common/supported.go b/pkg/testing/common/supported.go new file mode 100644 index 00000000000..94e17ed56cf --- /dev/null +++ b/pkg/testing/common/supported.go @@ -0,0 +1,15 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package common + +import "github.com/elastic/elastic-agent/pkg/testing/define" + +// SupportedOS maps a OS definition to a OSRunner. +type SupportedOS struct { + define.OS + + // Runner is the runner to use for the OS. + Runner OSRunner +} diff --git a/pkg/testing/ess/serverless.go b/pkg/testing/ess/serverless.go index d95696a1696..3ea4423e053 100644 --- a/pkg/testing/ess/serverless.go +++ b/pkg/testing/ess/serverless.go @@ -14,7 +14,7 @@ import ( "strings" "time" - "github.com/elastic/elastic-agent/pkg/testing/runner" + "github.com/elastic/elastic-agent/pkg/testing/common" ) var serverlessURL = "https://cloud.elastic.co" @@ -25,7 +25,7 @@ type ServerlessClient struct { projectType string api string proj Project - log runner.Logger + log common.Logger } // ServerlessRequest contains the data needed for a new serverless instance @@ -62,7 +62,7 @@ type CredResetResponse struct { } // NewServerlessClient creates a new instance of the serverless client -func NewServerlessClient(region, projectType, api string, logger runner.Logger) *ServerlessClient { +func NewServerlessClient(region, projectType, api string, logger common.Logger) *ServerlessClient { return &ServerlessClient{ region: region, api: api, diff --git a/pkg/testing/ess/serverless_provisioner.go b/pkg/testing/ess/serverless_provisioner.go index 7be19f03e47..f2be04279c8 100644 --- a/pkg/testing/ess/serverless_provisioner.go +++ b/pkg/testing/ess/serverless_provisioner.go @@ -13,7 +13,8 @@ import ( "time" "github.com/elastic/elastic-agent-libs/logp" - "github.com/elastic/elastic-agent/pkg/testing/runner" + + "github.com/elastic/elastic-agent/pkg/testing/common" ) const ProvisionerServerless = "serverless" @@ -21,7 +22,7 @@ const ProvisionerServerless = "serverless" // ServerlessProvisioner contains type ServerlessProvisioner struct { cfg ProvisionerConfig - log runner.Logger + log common.Logger } type defaultLogger struct { @@ -47,7 +48,7 @@ type ServerlessRegions struct { } // NewServerlessProvisioner creates a new StackProvisioner instance for serverless -func NewServerlessProvisioner(ctx context.Context, cfg ProvisionerConfig) (runner.StackProvisioner, error) { +func NewServerlessProvisioner(ctx context.Context, cfg ProvisionerConfig) (common.StackProvisioner, error) { prov := &ServerlessProvisioner{ cfg: cfg, log: &defaultLogger{wrapped: logp.L()}, @@ -64,12 +65,12 @@ func (prov *ServerlessProvisioner) Name() string { } // SetLogger sets the logger for the -func (prov *ServerlessProvisioner) SetLogger(l runner.Logger) { +func (prov *ServerlessProvisioner) SetLogger(l common.Logger) { prov.log = l } // Create creates a stack. -func (prov *ServerlessProvisioner) Create(ctx context.Context, request runner.StackRequest) (runner.Stack, error) { +func (prov *ServerlessProvisioner) Create(ctx context.Context, request common.StackRequest) (common.Stack, error) { // allow up to 4 minutes for requests createCtx, createCancel := context.WithTimeout(ctx, 4*time.Minute) defer createCancel() @@ -80,13 +81,13 @@ func (prov *ServerlessProvisioner) Create(ctx context.Context, request runner.St prov.log.Logf("Creating serverless stack %s [stack_id: %s]", request.Version, request.ID) proj, err := client.DeployStack(createCtx, srvReq) if err != nil { - return runner.Stack{}, fmt.Errorf("error deploying stack for request %s: %w", request.ID, err) + return common.Stack{}, fmt.Errorf("error deploying stack for request %s: %w", request.ID, err) } err = client.WaitForEndpoints(createCtx) if err != nil { - return runner.Stack{}, fmt.Errorf("error waiting for endpoints to become available for serverless stack %s [stack_id: %s, deployment_id: %s]: %w", request.Version, request.ID, proj.ID, err) + return common.Stack{}, fmt.Errorf("error waiting for endpoints to become available for serverless stack %s [stack_id: %s, deployment_id: %s]: %w", request.Version, request.ID, proj.ID, err) } - stack := runner.Stack{ + stack := common.Stack{ ID: request.ID, Provisioner: prov.Name(), Version: request.Version, @@ -105,7 +106,7 @@ func (prov *ServerlessProvisioner) Create(ctx context.Context, request runner.St } // WaitForReady should block until the stack is ready or the context is cancelled. -func (prov *ServerlessProvisioner) WaitForReady(ctx context.Context, stack runner.Stack) (runner.Stack, error) { +func (prov *ServerlessProvisioner) WaitForReady(ctx context.Context, stack common.Stack) (common.Stack, error) { deploymentID, deploymentType, err := prov.getDeploymentInfo(stack) if err != nil { return stack, fmt.Errorf("failed to get deployment info from the stack: %w", err) @@ -162,7 +163,7 @@ func (prov *ServerlessProvisioner) WaitForReady(ctx context.Context, stack runne } // Delete deletes a stack. -func (prov *ServerlessProvisioner) Delete(ctx context.Context, stack runner.Stack) error { +func (prov *ServerlessProvisioner) Delete(ctx context.Context, stack common.Stack) error { deploymentID, deploymentType, err := prov.getDeploymentInfo(stack) if err != nil { return fmt.Errorf("failed to get deployment info from the stack: %w", err) @@ -238,7 +239,7 @@ func (prov *ServerlessProvisioner) CheckCloudRegion(ctx context.Context) error { return nil } -func (prov *ServerlessProvisioner) getDeploymentInfo(stack runner.Stack) (string, string, error) { +func (prov *ServerlessProvisioner) getDeploymentInfo(stack common.Stack) (string, string, error) { if stack.Internal == nil { return "", "", fmt.Errorf("missing internal information") } diff --git a/pkg/testing/ess/serverless_test.go b/pkg/testing/ess/serverless_test.go index d84c766b2f8..05baeb2564c 100644 --- a/pkg/testing/ess/serverless_test.go +++ b/pkg/testing/ess/serverless_test.go @@ -12,7 +12,8 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/elastic-agent-libs/logp" - "github.com/elastic/elastic-agent/pkg/testing/runner" + + "github.com/elastic/elastic-agent/pkg/testing/common" ) func TestProvisionGetRegions(t *testing.T) { @@ -53,7 +54,7 @@ func TestStackProvisioner(t *testing.T) { cfg := ProvisionerConfig{Region: "aws-eu-west-1", APIKey: key} provClient, err := NewServerlessProvisioner(ctx, cfg) require.NoError(t, err) - request := runner.StackRequest{ID: "stack-test-one", Version: "8.9.0"} + request := common.StackRequest{ID: "stack-test-one", Version: "8.9.0"} stack, err := provClient.Create(ctx, request) require.NoError(t, err) diff --git a/pkg/testing/ess/statful_provisioner.go b/pkg/testing/ess/statful_provisioner.go index 44a099bb082..7c6d79d5f9f 100644 --- a/pkg/testing/ess/statful_provisioner.go +++ b/pkg/testing/ess/statful_provisioner.go @@ -12,7 +12,7 @@ import ( "strings" "time" - "github.com/elastic/elastic-agent/pkg/testing/runner" + "github.com/elastic/elastic-agent/pkg/testing/common" ) const ProvisionerStateful = "stateful" @@ -39,13 +39,13 @@ func (c *ProvisionerConfig) Validate() error { } type statefulProvisioner struct { - logger runner.Logger + logger common.Logger cfg ProvisionerConfig client *Client } // NewProvisioner creates the ESS stateful Provisioner -func NewProvisioner(cfg ProvisionerConfig) (runner.StackProvisioner, error) { +func NewProvisioner(cfg ProvisionerConfig) (common.StackProvisioner, error) { err := cfg.Validate() if err != nil { return nil, err @@ -63,12 +63,12 @@ func (p *statefulProvisioner) Name() string { return ProvisionerStateful } -func (p *statefulProvisioner) SetLogger(l runner.Logger) { +func (p *statefulProvisioner) SetLogger(l common.Logger) { p.logger = l } // Create creates a stack. -func (p *statefulProvisioner) Create(ctx context.Context, request runner.StackRequest) (runner.Stack, error) { +func (p *statefulProvisioner) Create(ctx context.Context, request common.StackRequest) (common.Stack, error) { // allow up to 2 minutes for request createCtx, createCancel := context.WithTimeout(ctx, 2*time.Minute) defer createCancel() @@ -88,9 +88,9 @@ func (p *statefulProvisioner) Create(ctx context.Context, request runner.StackRe } resp, err := p.createDeployment(createCtx, request, deploymentTags) if err != nil { - return runner.Stack{}, err + return common.Stack{}, err } - return runner.Stack{ + return common.Stack{ ID: request.ID, Provisioner: p.Name(), Version: request.Version, @@ -106,7 +106,7 @@ func (p *statefulProvisioner) Create(ctx context.Context, request runner.StackRe } // WaitForReady should block until the stack is ready or the context is cancelled. -func (p *statefulProvisioner) WaitForReady(ctx context.Context, stack runner.Stack) (runner.Stack, error) { +func (p *statefulProvisioner) WaitForReady(ctx context.Context, stack common.Stack) (common.Stack, error) { deploymentID, err := p.getDeploymentID(stack) if err != nil { return stack, fmt.Errorf("failed to get deployment ID from the stack: %w", err) @@ -127,7 +127,7 @@ func (p *statefulProvisioner) WaitForReady(ctx context.Context, stack runner.Sta } // Delete deletes a stack. -func (p *statefulProvisioner) Delete(ctx context.Context, stack runner.Stack) error { +func (p *statefulProvisioner) Delete(ctx context.Context, stack common.Stack) error { deploymentID, err := p.getDeploymentID(stack) if err != nil { return err @@ -141,7 +141,7 @@ func (p *statefulProvisioner) Delete(ctx context.Context, stack runner.Stack) er return p.client.ShutdownDeployment(ctx, deploymentID) } -func (p *statefulProvisioner) createDeployment(ctx context.Context, r runner.StackRequest, tags map[string]string) (*CreateDeploymentResponse, error) { +func (p *statefulProvisioner) createDeployment(ctx context.Context, r common.StackRequest, tags map[string]string) (*CreateDeploymentResponse, error) { ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) defer cancel() @@ -173,7 +173,7 @@ func (p *statefulProvisioner) createDeployment(ctx context.Context, r runner.Sta return resp, nil } -func (p *statefulProvisioner) getDeploymentID(stack runner.Stack) (string, error) { +func (p *statefulProvisioner) getDeploymentID(stack common.Stack) (string, error) { if stack.Internal == nil { return "", fmt.Errorf("missing internal information") } diff --git a/pkg/testing/kubernetes/image.go b/pkg/testing/kubernetes/image.go index c1ace5125d3..72c72907b9e 100644 --- a/pkg/testing/kubernetes/image.go +++ b/pkg/testing/kubernetes/image.go @@ -20,6 +20,7 @@ import ( "github.com/docker/docker/client" devtools "github.com/elastic/elastic-agent/dev-tools/mage" + "github.com/elastic/elastic-agent/pkg/testing/common" ) type DockerConfig struct { @@ -45,13 +46,8 @@ type Endpoint struct { Host string `json:"Host"` } -type runnerLogger interface { - // Logf logs the message for this runner. - Logf(format string, args ...any) -} - // AddK8STestsToImage compiles and adds the k8s-inner-tests binary to the given image -func AddK8STestsToImage(ctx context.Context, logger runnerLogger, baseImage string, arch string) (string, error) { +func AddK8STestsToImage(ctx context.Context, logger common.Logger, baseImage string, arch string) (string, error) { // compile k8s test with tag kubernetes_inner buildBase, err := filepath.Abs("build") if err != nil { diff --git a/pkg/testing/kubernetes/kind/provisioner.go b/pkg/testing/kubernetes/kind/provisioner.go index 637bd45f0d4..d572033cb13 100644 --- a/pkg/testing/kubernetes/kind/provisioner.go +++ b/pkg/testing/kubernetes/kind/provisioner.go @@ -14,10 +14,6 @@ import ( "runtime" "strings" - "github.com/elastic/elastic-agent/pkg/testing/define" - "github.com/elastic/elastic-agent/pkg/testing/kubernetes" - "github.com/elastic/elastic-agent/pkg/testing/runner" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/e2e-framework/klient" @@ -25,6 +21,10 @@ import ( "sigs.k8s.io/e2e-framework/klient/k8s/resources" "sigs.k8s.io/e2e-framework/klient/wait" "sigs.k8s.io/e2e-framework/klient/wait/conditions" + + "github.com/elastic/elastic-agent/pkg/testing/common" + "github.com/elastic/elastic-agent/pkg/testing/define" + "github.com/elastic/elastic-agent/pkg/testing/kubernetes" ) const ( @@ -49,23 +49,23 @@ nodes: secure-port: "10257" ` -func NewProvisioner() runner.InstanceProvisioner { +func NewProvisioner() common.InstanceProvisioner { return &provisioner{} } type provisioner struct { - logger runner.Logger + logger common.Logger } func (p *provisioner) Name() string { return Name } -func (p *provisioner) Type() runner.ProvisionerType { - return runner.ProvisionerTypeK8SCluster +func (p *provisioner) Type() common.ProvisionerType { + return common.ProvisionerTypeK8SCluster } -func (p *provisioner) SetLogger(l runner.Logger) { +func (p *provisioner) SetLogger(l common.Logger) { p.logger = l } @@ -80,8 +80,8 @@ func (p *provisioner) Supported(batch define.OS) bool { return true } -func (p *provisioner) Provision(ctx context.Context, cfg runner.Config, batches []runner.OSBatch) ([]runner.Instance, error) { - var instances []runner.Instance +func (p *provisioner) Provision(ctx context.Context, cfg common.Config, batches []common.OSBatch) ([]common.Instance, error) { + var instances []common.Instance for _, batch := range batches { k8sVersion := fmt.Sprintf("v%s", batch.OS.Version) instanceName := fmt.Sprintf("%s-%s", k8sVersion, batch.Batch.Group) @@ -140,7 +140,7 @@ func (p *provisioner) Provision(ctx context.Context, cfg runner.Config, batches return nil, err } - instances = append(instances, runner.Instance{ + instances = append(instances, common.Instance{ ID: batch.ID, Name: instanceName, Provisioner: Name, @@ -208,11 +208,11 @@ func (p *provisioner) WaitForControlPlane(client klient.Client) error { return nil } -func (p *provisioner) Clean(ctx context.Context, cfg runner.Config, instances []runner.Instance) error { +func (p *provisioner) Clean(ctx context.Context, cfg common.Config, instances []common.Instance) error { // doesn't execute in parallel for the same reasons in Provision // multipass just cannot handle it for _, instance := range instances { - func(instance runner.Instance) { + func(instance common.Instance) { err := p.deleteCluster(instance.ID) if err != nil { // prevent a failure from stopping the other instances and clean diff --git a/pkg/testing/runner/kubernetes.go b/pkg/testing/kubernetes/runner.go similarity index 66% rename from pkg/testing/runner/kubernetes.go rename to pkg/testing/kubernetes/runner.go index 67c2e65b6ba..c66fa537738 100644 --- a/pkg/testing/runner/kubernetes.go +++ b/pkg/testing/kubernetes/runner.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package runner +package kubernetes import ( "context" @@ -13,25 +13,28 @@ import ( "strings" "time" + "github.com/elastic/elastic-agent/pkg/testing/ssh" + devtools "github.com/elastic/elastic-agent/dev-tools/mage" + "github.com/elastic/elastic-agent/pkg/testing/common" "github.com/elastic/elastic-agent/pkg/testing/define" ) -// KubernetesRunner is a handler for running tests against a Kubernetes cluster -type KubernetesRunner struct{} +// Runner is a handler for running tests against a Kubernetes cluster +type Runner struct{} // Prepare configures the host for running the test -func (KubernetesRunner) Prepare(ctx context.Context, sshClient SSHClient, logger Logger, arch string, goVersion string) error { +func (Runner) Prepare(ctx context.Context, sshClient ssh.SSHClient, logger common.Logger, arch string, goVersion string) error { return nil } // Copy places the required files on the host -func (KubernetesRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logger, repoArchive string, builds []Build) error { +func (Runner) Copy(ctx context.Context, sshClient ssh.SSHClient, logger common.Logger, repoArchive string, builds []common.Build) error { return nil } // Run the test -func (KubernetesRunner) Run(ctx context.Context, verbose bool, sshClient SSHClient, logger Logger, agentVersion string, prefix string, batch define.Batch, env map[string]string) (OSRunnerResult, error) { +func (Runner) Run(ctx context.Context, verbose bool, sshClient ssh.SSHClient, logger common.Logger, agentVersion string, prefix string, batch define.Batch, env map[string]string) (common.OSRunnerResult, error) { var goTestFlags []string rawTestFlags := os.Getenv("GOTEST_FLAGS") if rawTestFlags != "" { @@ -39,7 +42,7 @@ func (KubernetesRunner) Run(ctx context.Context, verbose bool, sshClient SSHClie } maxDuration := 2 * time.Hour - var result []OSRunnerPackageResult + var result []common.OSRunnerPackageResult for _, pkg := range batch.Tests { packageTestsStrBuilder := strings.Builder{} packageTestsStrBuilder.WriteString("^(") @@ -66,13 +69,13 @@ func (KubernetesRunner) Run(ctx context.Context, verbose bool, sshClient SSHClie buildFolderAbsPath, err := filepath.Abs("build") if err != nil { - return OSRunnerResult{}, err + return common.OSRunnerResult{}, err } podLogsPath := filepath.Join(buildFolderAbsPath, fmt.Sprintf("k8s-logs-%s", testPrefix)) err = os.Mkdir(podLogsPath, 0755) if err != nil && !errors.Is(err, os.ErrExist) { - return OSRunnerResult{}, err + return common.OSRunnerResult{}, err } env["K8S_TESTS_POD_LOGS_BASE"] = podLogsPath @@ -88,33 +91,34 @@ func (KubernetesRunner) Run(ctx context.Context, verbose bool, sshClient SSHClie } err = devtools.GoTest(ctx, params) if err != nil { - return OSRunnerResult{}, err + return common.OSRunnerResult{}, err } - var resultPkg OSRunnerPackageResult + var resultPkg common.OSRunnerPackageResult resultPkg.Name = pkg.Name outputPath := fmt.Sprintf("build/TEST-go-k8s-%s.%s", prefix, filepath.Base(pkg.Name)) resultPkg.Output, err = os.ReadFile(outputPath + ".out") if err != nil { - return OSRunnerResult{}, fmt.Errorf("failed to fetched test output at %s.out", outputPath) + return common.OSRunnerResult{}, fmt.Errorf("failed to fetched test output at %s.out", outputPath) } resultPkg.JSONOutput, err = os.ReadFile(outputPath + ".out.json") if err != nil { - return OSRunnerResult{}, fmt.Errorf("failed to fetched test output at %s.out.json", outputPath) + return common.OSRunnerResult{}, fmt.Errorf("failed to fetched test output at %s.out.json", outputPath) } resultPkg.XMLOutput, err = os.ReadFile(outputPath + ".xml") if err != nil { - return OSRunnerResult{}, fmt.Errorf("failed to fetched test output at %s.xml", outputPath) + return common.OSRunnerResult{}, fmt.Errorf("failed to fetched test output at %s.xml", outputPath) } result = append(result, resultPkg) } - return OSRunnerResult{ + return common.OSRunnerResult{ Packages: result, }, nil } // Diagnostics gathers any diagnostics from the host. -func (KubernetesRunner) Diagnostics(ctx context.Context, sshClient SSHClient, logger Logger, destination string) error { - return linuxDiagnostics(ctx, sshClient, logger, destination) +func (Runner) Diagnostics(ctx context.Context, sshClient ssh.SSHClient, logger common.Logger, destination string) error { + // does nothing for kubernetes + return nil } diff --git a/pkg/testing/runner/debian.go b/pkg/testing/linux/debian.go similarity index 78% rename from pkg/testing/runner/debian.go rename to pkg/testing/linux/debian.go index 7a7cb07332e..b93656c86d3 100644 --- a/pkg/testing/runner/debian.go +++ b/pkg/testing/linux/debian.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package runner +package linux import ( "context" @@ -12,14 +12,16 @@ import ( "strings" "time" + "github.com/elastic/elastic-agent/pkg/testing/common" "github.com/elastic/elastic-agent/pkg/testing/define" + "github.com/elastic/elastic-agent/pkg/testing/ssh" ) // DebianRunner is a handler for running tests on Linux type DebianRunner struct{} // Prepare the test -func (DebianRunner) Prepare(ctx context.Context, sshClient SSHClient, logger Logger, arch string, goVersion string) error { +func (DebianRunner) Prepare(ctx context.Context, sshClient ssh.SSHClient, logger common.Logger, arch string, goVersion string) error { // prepare build-essential and unzip // // apt-get update and install are so terrible that we have to place this in a loop, because in some cases the @@ -86,12 +88,12 @@ func (DebianRunner) Prepare(ctx context.Context, sshClient SSHClient, logger Log } // Copy places the required files on the host. -func (DebianRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logger, repoArchive string, builds []Build) error { +func (DebianRunner) Copy(ctx context.Context, sshClient ssh.SSHClient, logger common.Logger, repoArchive string, builds []common.Build) error { return linuxCopy(ctx, sshClient, logger, repoArchive, builds) } // Run the test -func (DebianRunner) Run(ctx context.Context, verbose bool, sshClient SSHClient, logger Logger, agentVersion string, prefix string, batch define.Batch, env map[string]string) (OSRunnerResult, error) { +func (DebianRunner) Run(ctx context.Context, verbose bool, sshClient ssh.SSHClient, logger common.Logger, agentVersion string, prefix string, batch define.Batch, env map[string]string) (common.OSRunnerResult, error) { var tests []string for _, pkg := range batch.Tests { for _, test := range pkg.Tests { @@ -109,7 +111,7 @@ func (DebianRunner) Run(ctx context.Context, verbose bool, sshClient SSHClient, if verbose { logArg = "-v" } - var result OSRunnerResult + var result common.OSRunnerResult if len(tests) > 0 { vars := fmt.Sprintf(`GOPATH="$HOME/go" PATH="$HOME/go/bin:$PATH" AGENT_VERSION="%s" TEST_DEFINE_PREFIX="%s" TEST_DEFINE_TESTS="%s"`, agentVersion, prefix, strings.Join(tests, ",")) vars = extendVars(vars, env) @@ -117,7 +119,7 @@ func (DebianRunner) Run(ctx context.Context, verbose bool, sshClient SSHClient, script := fmt.Sprintf(`cd agent && %s ~/go/bin/mage %s integration:testOnRemote`, vars, logArg) results, err := runTests(ctx, logger, "non-sudo", prefix, script, sshClient, batch.Tests) if err != nil { - return OSRunnerResult{}, fmt.Errorf("error running non-sudo tests: %w", err) + return common.OSRunnerResult{}, fmt.Errorf("error running non-sudo tests: %w", err) } result.Packages = results } @@ -130,7 +132,7 @@ func (DebianRunner) Run(ctx context.Context, verbose bool, sshClient SSHClient, results, err := runTests(ctx, logger, "sudo", prefix, script, sshClient, batch.SudoTests) if err != nil { - return OSRunnerResult{}, fmt.Errorf("error running sudo tests: %w", err) + return common.OSRunnerResult{}, fmt.Errorf("error running sudo tests: %w", err) } result.SudoPackages = results } @@ -139,11 +141,11 @@ func (DebianRunner) Run(ctx context.Context, verbose bool, sshClient SSHClient, } // Diagnostics gathers any diagnostics from the host. -func (DebianRunner) Diagnostics(ctx context.Context, sshClient SSHClient, logger Logger, destination string) error { +func (DebianRunner) Diagnostics(ctx context.Context, sshClient ssh.SSHClient, logger common.Logger, destination string) error { return linuxDiagnostics(ctx, sshClient, logger, destination) } -func runTests(ctx context.Context, logger Logger, name string, prefix string, script string, sshClient SSHClient, tests []define.BatchPackageTests) ([]OSRunnerPackageResult, error) { +func runTests(ctx context.Context, logger common.Logger, name string, prefix string, script string, sshClient ssh.SSHClient, tests []define.BatchPackageTests) ([]common.OSRunnerPackageResult, error) { execTest := strings.NewReader(script) session, err := sshClient.NewSession() @@ -151,8 +153,8 @@ func runTests(ctx context.Context, logger Logger, name string, prefix string, sc return nil, fmt.Errorf("failed to start session: %w", err) } - session.Stdout = newPrefixOutput(logger, fmt.Sprintf("Test output (%s) (stdout): ", name)) - session.Stderr = newPrefixOutput(logger, fmt.Sprintf("Test output (%s) (stderr): ", name)) + session.Stdout = common.NewPrefixOutput(logger, fmt.Sprintf("Test output (%s) (stdout): ", name)) + session.Stderr = common.NewPrefixOutput(logger, fmt.Sprintf("Test output (%s) (stderr): ", name)) session.Stdin = execTest // allowed to fail because tests might fail @@ -164,7 +166,7 @@ func runTests(ctx context.Context, logger Logger, name string, prefix string, sc // this seems to always return an error _ = session.Close() - var result []OSRunnerPackageResult + var result []common.OSRunnerPackageResult // fetch the contents for each package for _, pkg := range tests { resultPkg, err := getRunnerPackageResult(ctx, sshClient, pkg, prefix) @@ -176,22 +178,22 @@ func runTests(ctx context.Context, logger Logger, name string, prefix string, sc return result, nil } -func getRunnerPackageResult(ctx context.Context, sshClient SSHClient, pkg define.BatchPackageTests, prefix string) (OSRunnerPackageResult, error) { +func getRunnerPackageResult(ctx context.Context, sshClient ssh.SSHClient, pkg define.BatchPackageTests, prefix string) (common.OSRunnerPackageResult, error) { var err error - var resultPkg OSRunnerPackageResult + var resultPkg common.OSRunnerPackageResult resultPkg.Name = pkg.Name outputPath := fmt.Sprintf("$HOME/agent/build/TEST-go-remote-%s.%s", prefix, filepath.Base(pkg.Name)) resultPkg.Output, err = sshClient.GetFileContents(ctx, outputPath+".out") if err != nil { - return OSRunnerPackageResult{}, fmt.Errorf("failed to fetched test output at %s.out", outputPath) + return common.OSRunnerPackageResult{}, fmt.Errorf("failed to fetched test output at %s.out", outputPath) } resultPkg.JSONOutput, err = sshClient.GetFileContents(ctx, outputPath+".out.json") if err != nil { - return OSRunnerPackageResult{}, fmt.Errorf("failed to fetched test output at %s.out.json", outputPath) + return common.OSRunnerPackageResult{}, fmt.Errorf("failed to fetched test output at %s.out.json", outputPath) } resultPkg.XMLOutput, err = sshClient.GetFileContents(ctx, outputPath+".xml") if err != nil { - return OSRunnerPackageResult{}, fmt.Errorf("failed to fetched test output at %s.xml", outputPath) + return common.OSRunnerPackageResult{}, fmt.Errorf("failed to fetched test output at %s.xml", outputPath) } return resultPkg, nil } diff --git a/pkg/testing/runner/linux.go b/pkg/testing/linux/linux.go similarity index 93% rename from pkg/testing/runner/linux.go rename to pkg/testing/linux/linux.go index ffe1f2fdeb4..c38f53dc521 100644 --- a/pkg/testing/runner/linux.go +++ b/pkg/testing/linux/linux.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package runner +package linux import ( "context" @@ -10,9 +10,12 @@ import ( "os" "path/filepath" "strings" + + "github.com/elastic/elastic-agent/pkg/testing/common" + "github.com/elastic/elastic-agent/pkg/testing/ssh" ) -func linuxDiagnostics(ctx context.Context, sshClient SSHClient, logger Logger, destination string) error { +func linuxDiagnostics(ctx context.Context, sshClient ssh.SSHClient, logger common.Logger, destination string) error { // take ownership, as sudo tests will create with root permissions (allow to fail in the case it doesn't exist) diagnosticDir := "$HOME/agent/build/diagnostics" _, _, _ = sshClient.Exec(ctx, "sudo", []string{"chown", "-R", "$USER:$USER", diagnosticDir}, nil) @@ -48,7 +51,7 @@ func linuxDiagnostics(ctx context.Context, sshClient SSHClient, logger Logger, d return nil } -func linuxCopy(ctx context.Context, sshClient SSHClient, logger Logger, repoArchive string, builds []Build) error { +func linuxCopy(ctx context.Context, sshClient ssh.SSHClient, logger common.Logger, repoArchive string, builds []common.Build) error { // copy the archive and extract it on the host logger.Logf("Copying repo") destRepoName := filepath.Base(repoArchive) diff --git a/pkg/testing/runner/rhel.go b/pkg/testing/linux/rhel.go similarity index 82% rename from pkg/testing/runner/rhel.go rename to pkg/testing/linux/rhel.go index da0bf6396ab..e43daf1e13b 100644 --- a/pkg/testing/runner/rhel.go +++ b/pkg/testing/linux/rhel.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package runner +package linux import ( "context" @@ -11,14 +11,16 @@ import ( "strings" "time" + "github.com/elastic/elastic-agent/pkg/testing/common" "github.com/elastic/elastic-agent/pkg/testing/define" + "github.com/elastic/elastic-agent/pkg/testing/ssh" ) // RhelRunner is a handler for running tests on SUSE Linux Enterpriser Server type RhelRunner struct{} // Prepare configures the host for running the test -func (RhelRunner) Prepare(ctx context.Context, sshClient SSHClient, logger Logger, arch string, goVersion string) error { +func (RhelRunner) Prepare(ctx context.Context, sshClient ssh.SSHClient, logger common.Logger, arch string, goVersion string) error { logger.Logf("Install development tools") dnfCtx, dnfCancel := context.WithTimeout(ctx, 20*time.Minute) defer dnfCancel() @@ -54,12 +56,12 @@ func (RhelRunner) Prepare(ctx context.Context, sshClient SSHClient, logger Logge } // Copy places the required files on the host -func (RhelRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logger, repoArchive string, builds []Build) error { +func (RhelRunner) Copy(ctx context.Context, sshClient ssh.SSHClient, logger common.Logger, repoArchive string, builds []common.Build) error { return linuxCopy(ctx, sshClient, logger, repoArchive, builds) } // Run the test -func (RhelRunner) Run(ctx context.Context, verbose bool, sshClient SSHClient, logger Logger, agentVersion string, prefix string, batch define.Batch, env map[string]string) (OSRunnerResult, error) { +func (RhelRunner) Run(ctx context.Context, verbose bool, sshClient ssh.SSHClient, logger common.Logger, agentVersion string, prefix string, batch define.Batch, env map[string]string) (common.OSRunnerResult, error) { var tests []string for _, pkg := range batch.Tests { for _, test := range pkg.Tests { @@ -77,7 +79,7 @@ func (RhelRunner) Run(ctx context.Context, verbose bool, sshClient SSHClient, lo if verbose { logArg = "-v" } - var result OSRunnerResult + var result common.OSRunnerResult if len(tests) > 0 { vars := fmt.Sprintf(`GOPATH="$HOME/go" PATH="$HOME/go/bin:$PATH" AGENT_VERSION="%s" TEST_DEFINE_PREFIX="%s" TEST_DEFINE_TESTS="%s"`, agentVersion, prefix, strings.Join(tests, ",")) vars = extendVars(vars, env) @@ -85,7 +87,7 @@ func (RhelRunner) Run(ctx context.Context, verbose bool, sshClient SSHClient, lo script := fmt.Sprintf(`cd agent && %s ~/go/bin/mage %s integration:testOnRemote`, vars, logArg) results, err := runTests(ctx, logger, "non-sudo", prefix, script, sshClient, batch.Tests) if err != nil { - return OSRunnerResult{}, fmt.Errorf("error running non-sudo tests: %w", err) + return common.OSRunnerResult{}, fmt.Errorf("error running non-sudo tests: %w", err) } result.Packages = results } @@ -98,7 +100,7 @@ func (RhelRunner) Run(ctx context.Context, verbose bool, sshClient SSHClient, lo results, err := runTests(ctx, logger, "sudo", prefix, script, sshClient, batch.SudoTests) if err != nil { - return OSRunnerResult{}, fmt.Errorf("error running sudo tests: %w", err) + return common.OSRunnerResult{}, fmt.Errorf("error running sudo tests: %w", err) } result.SudoPackages = results } @@ -107,6 +109,6 @@ func (RhelRunner) Run(ctx context.Context, verbose bool, sshClient SSHClient, lo } // Diagnostics gathers any diagnostics from the host. -func (RhelRunner) Diagnostics(ctx context.Context, sshClient SSHClient, logger Logger, destination string) error { +func (RhelRunner) Diagnostics(ctx context.Context, sshClient ssh.SSHClient, logger common.Logger, destination string) error { return linuxDiagnostics(ctx, sshClient, logger, destination) } diff --git a/pkg/testing/multipass/provisioner.go b/pkg/testing/multipass/provisioner.go index d00140ace60..7f3ad91db68 100644 --- a/pkg/testing/multipass/provisioner.go +++ b/pkg/testing/multipass/provisioner.go @@ -18,6 +18,7 @@ import ( "gopkg.in/yaml.v2" "github.com/elastic/elastic-agent/pkg/core/process" + "github.com/elastic/elastic-agent/pkg/testing/common" "github.com/elastic/elastic-agent/pkg/testing/define" "github.com/elastic/elastic-agent/pkg/testing/runner" ) @@ -28,11 +29,11 @@ const ( ) type provisioner struct { - logger runner.Logger + logger common.Logger } // NewProvisioner creates the multipass provisioner -func NewProvisioner() runner.InstanceProvisioner { +func NewProvisioner() common.InstanceProvisioner { return &provisioner{} } @@ -40,12 +41,12 @@ func (p *provisioner) Name() string { return Name } -func (p *provisioner) SetLogger(l runner.Logger) { +func (p *provisioner) SetLogger(l common.Logger) { p.logger = l } -func (p *provisioner) Type() runner.ProvisionerType { - return runner.ProvisionerTypeVM +func (p *provisioner) Type() common.ProvisionerType { + return common.ProvisionerTypeVM } // Supported returns true if multipass supports this OS. @@ -68,12 +69,12 @@ func (p *provisioner) Supported(os define.OS) bool { return true } -func (p *provisioner) Provision(ctx context.Context, cfg runner.Config, batches []runner.OSBatch) ([]runner.Instance, error) { +func (p *provisioner) Provision(ctx context.Context, cfg common.Config, batches []common.OSBatch) ([]common.Instance, error) { // this doesn't provision the instances in parallel on purpose // multipass cannot handle it, it either results in instances sharing the same IP address // or some instances stuck in Starting state for _, batch := range batches { - err := func(batch runner.OSBatch) error { + err := func(batch common.OSBatch) error { launchCtx, launchCancel := context.WithTimeout(ctx, 5*time.Minute) defer launchCancel() err := p.launch(launchCtx, cfg, batch) @@ -87,7 +88,7 @@ func (p *provisioner) Provision(ctx context.Context, cfg runner.Config, batches } } - var results []runner.Instance + var results []common.Instance instances, err := p.list(ctx) if err != nil { return nil, err @@ -100,7 +101,7 @@ func (p *provisioner) Provision(ctx context.Context, cfg runner.Config, batches if mi.State != "Running" { return nil, fmt.Errorf("instance %s is not marked as running", batch.ID) } - results = append(results, runner.Instance{ + results = append(results, common.Instance{ ID: batch.ID, Provisioner: Name, Name: batch.ID, @@ -114,11 +115,11 @@ func (p *provisioner) Provision(ctx context.Context, cfg runner.Config, batches } // Clean cleans up all provisioned resources. -func (p *provisioner) Clean(ctx context.Context, _ runner.Config, instances []runner.Instance) error { +func (p *provisioner) Clean(ctx context.Context, _ common.Config, instances []common.Instance) error { // doesn't execute in parallel for the same reasons in Provision // multipass just cannot handle it for _, instance := range instances { - func(instance runner.Instance) { + func(instance common.Instance) { deleteCtx, deleteCancel := context.WithTimeout(ctx, 5*time.Minute) defer deleteCancel() err := p.delete(deleteCtx, instance) @@ -132,7 +133,7 @@ func (p *provisioner) Clean(ctx context.Context, _ runner.Config, instances []ru } // launch creates an instance. -func (p *provisioner) launch(ctx context.Context, cfg runner.Config, batch runner.OSBatch) error { +func (p *provisioner) launch(ctx context.Context, cfg common.Config, batch common.OSBatch) error { // check if instance already exists err := p.ensureInstanceNotExist(ctx, batch) if err != nil { @@ -187,7 +188,7 @@ func (p *provisioner) launch(ctx context.Context, cfg runner.Config, batch runne return nil } -func (p *provisioner) ensureInstanceNotExist(ctx context.Context, batch runner.OSBatch) error { +func (p *provisioner) ensureInstanceNotExist(ctx context.Context, batch common.OSBatch) error { var output bytes.Buffer var stdErr bytes.Buffer proc, err := process.Start("multipass", @@ -258,7 +259,7 @@ func (p *provisioner) ensureInstanceNotExist(ctx context.Context, batch runner.O } // delete deletes an instance. -func (p *provisioner) delete(ctx context.Context, instance runner.Instance) error { +func (p *provisioner) delete(ctx context.Context, instance common.Instance) error { args := []string{ "delete", "-p", diff --git a/pkg/testing/ogc/provisioner.go b/pkg/testing/ogc/provisioner.go index 3417b2974b3..52a3833e586 100644 --- a/pkg/testing/ogc/provisioner.go +++ b/pkg/testing/ogc/provisioner.go @@ -16,6 +16,7 @@ import ( "gopkg.in/yaml.v2" "github.com/elastic/elastic-agent/pkg/core/process" + "github.com/elastic/elastic-agent/pkg/testing/common" "github.com/elastic/elastic-agent/pkg/testing/define" "github.com/elastic/elastic-agent/pkg/testing/runner" ) @@ -27,12 +28,12 @@ const ( ) type provisioner struct { - logger runner.Logger + logger common.Logger cfg Config } // NewProvisioner creates the OGC provisioner -func NewProvisioner(cfg Config) (runner.InstanceProvisioner, error) { +func NewProvisioner(cfg Config) (common.InstanceProvisioner, error) { err := cfg.Validate() if err != nil { return nil, err @@ -46,12 +47,12 @@ func (p *provisioner) Name() string { return Name } -func (p *provisioner) SetLogger(l runner.Logger) { +func (p *provisioner) SetLogger(l common.Logger) { p.logger = l } -func (p *provisioner) Type() runner.ProvisionerType { - return runner.ProvisionerTypeVM +func (p *provisioner) Type() common.ProvisionerType { + return common.ProvisionerTypeVM } // Supported returns true when we support this OS for OGC. @@ -60,7 +61,7 @@ func (p *provisioner) Supported(os define.OS) bool { return ok } -func (p *provisioner) Provision(ctx context.Context, cfg runner.Config, batches []runner.OSBatch) ([]runner.Instance, error) { +func (p *provisioner) Provision(ctx context.Context, cfg common.Config, batches []common.OSBatch) ([]common.Instance, error) { // ensure the latest version pullCtx, pullCancel := context.WithTimeout(ctx, 5*time.Minute) defer pullCancel() @@ -99,7 +100,7 @@ func (p *provisioner) Provision(ctx context.Context, cfg runner.Config, batches } // map the machines to instances - var instances []runner.Instance + var instances []common.Instance for _, b := range batches { machine, ok := findMachine(machines, b.ID) if !ok { @@ -109,7 +110,7 @@ func (p *provisioner) Provision(ctx context.Context, cfg runner.Config, batches fmt.Fprintf(os.Stdout, "%s\n", upOutput) return nil, fmt.Errorf("failed to find machine for batch ID: %s", b.ID) } - instances = append(instances, runner.Instance{ + instances = append(instances, common.Instance{ ID: b.ID, Provisioner: Name, Name: machine.InstanceName, @@ -125,7 +126,7 @@ func (p *provisioner) Provision(ctx context.Context, cfg runner.Config, batches } // Clean cleans up all provisioned resources. -func (p *provisioner) Clean(ctx context.Context, cfg runner.Config, _ []runner.Instance) error { +func (p *provisioner) Clean(ctx context.Context, cfg common.Config, _ []common.Instance) error { return p.ogcDown(ctx) } @@ -151,7 +152,7 @@ func (p *provisioner) ogcPull(ctx context.Context) error { } // ogcImport imports all the required batches into OGC. -func (p *provisioner) ogcImport(ctx context.Context, cfg runner.Config, batches []runner.OSBatch) error { +func (p *provisioner) ogcImport(ctx context.Context, cfg common.Config, batches []common.OSBatch) error { var layouts []Layout for _, ob := range batches { layouts = append(layouts, osBatchToOGC(cfg.StateDir, ob)) @@ -288,7 +289,7 @@ func (p *provisioner) ogcRun(ctx context.Context, args []string, interactive boo return process.Start("docker", opts...) } -func osBatchToOGC(cacheDir string, batch runner.OSBatch) Layout { +func osBatchToOGC(cacheDir string, batch common.OSBatch) Layout { tags := []string{ LayoutIntegrationTag, batch.OS.Type, diff --git a/pkg/testing/ogc/supported.go b/pkg/testing/ogc/supported.go index 3aea5f35fb8..f4768ff270a 100644 --- a/pkg/testing/ogc/supported.go +++ b/pkg/testing/ogc/supported.go @@ -6,7 +6,7 @@ package ogc import ( "github.com/elastic/elastic-agent/pkg/testing/define" - "github.com/elastic/elastic-agent/pkg/testing/runner" + "github.com/elastic/elastic-agent/pkg/testing/supported" ) const ( @@ -24,7 +24,7 @@ var ogcSupported = []LayoutOS{ OS: define.OS{ Type: define.Linux, Arch: define.AMD64, - Distro: runner.Ubuntu, + Distro: supported.Ubuntu, Version: "24.04", }, Provider: Google, @@ -37,7 +37,7 @@ var ogcSupported = []LayoutOS{ OS: define.OS{ Type: define.Linux, Arch: define.AMD64, - Distro: runner.Ubuntu, + Distro: supported.Ubuntu, Version: "22.04", }, Provider: Google, @@ -50,7 +50,7 @@ var ogcSupported = []LayoutOS{ OS: define.OS{ Type: define.Linux, Arch: define.AMD64, - Distro: runner.Ubuntu, + Distro: supported.Ubuntu, Version: "20.04", }, Provider: Google, @@ -105,7 +105,7 @@ var ogcSupported = []LayoutOS{ OS: define.OS{ Type: define.Linux, Arch: define.AMD64, - Distro: runner.Rhel, + Distro: supported.Rhel, Version: "8", }, Provider: Google, diff --git a/pkg/testing/runner/runner.go b/pkg/testing/runner/runner.go index df2087a45ce..973df39f41f 100644 --- a/pkg/testing/runner/runner.go +++ b/pkg/testing/runner/runner.go @@ -7,7 +7,6 @@ package runner import ( "bytes" "context" - "crypto/md5" "errors" "fmt" "io" @@ -17,68 +16,18 @@ import ( "strings" "sync" "time" - "unicode/utf8" - - "gopkg.in/yaml.v2" "golang.org/x/crypto/ssh" "golang.org/x/sync/errgroup" + "gopkg.in/yaml.v2" "github.com/elastic/elastic-agent/pkg/testing" + "github.com/elastic/elastic-agent/pkg/testing/common" "github.com/elastic/elastic-agent/pkg/testing/define" + tssh "github.com/elastic/elastic-agent/pkg/testing/ssh" + "github.com/elastic/elastic-agent/pkg/testing/supported" ) -// OSBatch defines the mapping between a SupportedOS and a define.Batch. -type OSBatch struct { - // ID is the unique ID for the batch. - ID string - // LayoutOS provides all the OS information to create an instance. - OS SupportedOS - // Batch defines the batch of tests to run on this layout. - Batch define.Batch - // Skip defines if this batch will be skipped because no supported layout exists yet. - Skip bool -} - -// OSRunnerPackageResult is the result for each package. -type OSRunnerPackageResult struct { - // Name is the package name. - Name string - // Output is the raw test output. - Output []byte - // XMLOutput is the XML Junit output. - XMLOutput []byte - // JSONOutput is the JSON output. - JSONOutput []byte -} - -// OSRunnerResult is the result of the test run provided by a OSRunner. -type OSRunnerResult struct { - // Packages is the results for each package. - Packages []OSRunnerPackageResult - - // SudoPackages is the results for each package that need to run as sudo. - SudoPackages []OSRunnerPackageResult -} - -// OSRunner provides an interface to run the tests on the OS. -type OSRunner interface { - // Prepare prepares the runner to actual run on the host. - Prepare(ctx context.Context, sshClient SSHClient, logger Logger, arch string, goVersion string) error - // Copy places the required files on the host. - Copy(ctx context.Context, sshClient SSHClient, logger Logger, repoArchive string, builds []Build) error - // Run runs the actual tests and provides the result. - Run(ctx context.Context, verbose bool, sshClient SSHClient, logger Logger, agentVersion string, prefix string, batch define.Batch, env map[string]string) (OSRunnerResult, error) - // Diagnostics gathers any diagnostics from the host. - Diagnostics(ctx context.Context, sshClient SSHClient, logger Logger, destination string) error -} - -// Logger is a simple logging interface used by each runner type. -type Logger interface { - // Logf logs the message for this runner. - Logf(format string, args ...any) -} - // Result is the complete result from the runner. type Result struct { // Tests is the number of tests ran. @@ -99,39 +48,25 @@ type State struct { Instances []StateInstance `yaml:"instances"` // Stacks store provisioned stacks. - Stacks []Stack `yaml:"stacks"` + Stacks []common.Stack `yaml:"stacks"` } // StateInstance is an instance stored in the state. type StateInstance struct { - Instance + common.Instance // Prepared set to true when the instance is prepared. Prepared bool `yaml:"prepared"` } -// Build describes a build and its paths. -type Build struct { - // Version of the Elastic Agent build. - Version string - // Type of OS this build is for. - Type string - // Arch is architecture this build is for. - Arch string - // Path is the path to the build. - Path string - // SHA512 is the path to the SHA512 file. - SHA512Path string -} - // Runner runs the tests on remote instances. type Runner struct { - cfg Config - logger Logger - ip InstanceProvisioner - sp StackProvisioner + cfg common.Config + logger common.Logger + ip common.InstanceProvisioner + sp common.StackProvisioner - batches []OSBatch + batches []common.OSBatch batchToStack map[string]stackRes batchToStackCh map[string]chan stackRes @@ -142,7 +77,7 @@ type Runner struct { } // NewRunner creates a new runner based on the provided batches. -func NewRunner(cfg Config, ip InstanceProvisioner, sp StackProvisioner, batches ...define.Batch) (*Runner, error) { +func NewRunner(cfg common.Config, ip common.InstanceProvisioner, sp common.StackProvisioner, batches ...define.Batch) (*Runner, error) { err := cfg.Validate() if err != nil { return nil, err @@ -152,6 +87,12 @@ func NewRunner(cfg Config, ip InstanceProvisioner, sp StackProvisioner, batches return nil, err } + osBatches, err := supported.CreateBatches(batches, platforms, cfg.Groups, cfg.Matrix, cfg.SingleTest) + if err != nil { + return nil, err + } + osBatches = filterSupportedOS(osBatches, ip) + logger := &runnerLogger{ writer: os.Stdout, timestamp: cfg.Timestamp, @@ -159,24 +100,6 @@ func NewRunner(cfg Config, ip InstanceProvisioner, sp StackProvisioner, batches ip.SetLogger(logger) sp.SetLogger(logger) - var osBatches []OSBatch - for _, b := range batches { - lbs, err := createBatches(b, platforms, cfg.Groups, cfg.Matrix) - if err != nil { - return nil, err - } - if lbs != nil { - osBatches = append(osBatches, lbs...) - } - } - if cfg.SingleTest != "" { - osBatches, err = filterSingleTest(osBatches, cfg.SingleTest) - if err != nil { - return nil, err - } - } - osBatches = filterSupportedOS(osBatches, ip) - r := &Runner{ cfg: cfg, logger: logger, @@ -195,7 +118,7 @@ func NewRunner(cfg Config, ip InstanceProvisioner, sp StackProvisioner, batches } // Logger returns the logger used by the runner. -func (r *Runner) Logger() Logger { +func (r *Runner) Logger() common.Logger { return r.logger } @@ -223,7 +146,7 @@ func (r *Runner) Run(ctx context.Context) (Result, error) { // only send to the provisioner the batches that need to be created var instances []StateInstance - var batches []OSBatch + var batches []common.OSBatch for _, b := range r.batches { if !b.Skip { i, ok := r.findInstance(b.ID) @@ -247,15 +170,15 @@ func (r *Runner) Run(ctx context.Context) (Result, error) { } } - var results map[string]OSRunnerResult + var results map[string]common.OSRunnerResult switch r.ip.Type() { - case ProvisionerTypeVM: + case common.ProvisionerTypeVM: // use SSH to perform all the required work on the instances results, err = r.runInstances(ctx, sshAuth, repoArchive, instances) if err != nil { return Result{}, err } - case ProvisionerTypeK8SCluster: + case common.ProvisionerTypeK8SCluster: results, err = r.runK8sInstances(ctx, instances) if err != nil { return Result{}, err @@ -274,12 +197,12 @@ func (r *Runner) Clean() error { r.stateMx.Lock() defer r.stateMx.Unlock() - var instances []Instance + var instances []common.Instance for _, i := range r.state.Instances { instances = append(instances, i.Instance) } r.state.Instances = nil - stacks := make([]Stack, len(r.state.Stacks)) + stacks := make([]common.Stack, len(r.state.Stacks)) copy(stacks, r.state.Stacks) r.state.Stacks = nil err := r.writeState() @@ -294,7 +217,7 @@ func (r *Runner) Clean() error { return r.ip.Clean(ctx, r.cfg, instances) }) for _, stack := range stacks { - g.Go(func(stack Stack) func() error { + g.Go(func(stack common.Stack) func() error { return func() error { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() @@ -305,8 +228,8 @@ func (r *Runner) Clean() error { return g.Wait() } -func (r *Runner) runK8sInstances(ctx context.Context, instances []StateInstance) (map[string]OSRunnerResult, error) { - results := make(map[string]OSRunnerResult) +func (r *Runner) runK8sInstances(ctx context.Context, instances []StateInstance) (map[string]common.OSRunnerResult, error) { + results := make(map[string]common.OSRunnerResult) var resultsMx sync.Mutex var err error for _, instance := range instances { @@ -367,9 +290,9 @@ func (r *Runner) runK8sInstances(ctx context.Context, instances []StateInstance) } // runInstances runs the batch on each instance in parallel. -func (r *Runner) runInstances(ctx context.Context, sshAuth ssh.AuthMethod, repoArchive string, instances []StateInstance) (map[string]OSRunnerResult, error) { +func (r *Runner) runInstances(ctx context.Context, sshAuth ssh.AuthMethod, repoArchive string, instances []StateInstance) (map[string]common.OSRunnerResult, error) { g, ctx := errgroup.WithContext(ctx) - results := make(map[string]OSRunnerResult) + results := make(map[string]common.OSRunnerResult) var resultsMx sync.Mutex for _, i := range instances { func(i StateInstance) { @@ -399,20 +322,20 @@ func (r *Runner) runInstances(ctx context.Context, sshAuth ssh.AuthMethod, repoA } // runInstance runs the batch on the machine. -func (r *Runner) runInstance(ctx context.Context, sshAuth ssh.AuthMethod, logger Logger, repoArchive string, batch OSBatch, instance StateInstance) (OSRunnerResult, error) { +func (r *Runner) runInstance(ctx context.Context, sshAuth ssh.AuthMethod, logger common.Logger, repoArchive string, batch common.OSBatch, instance StateInstance) (common.OSRunnerResult, error) { sshPrivateKeyPath, err := filepath.Abs(filepath.Join(r.cfg.StateDir, "id_rsa")) if err != nil { - return OSRunnerResult{}, fmt.Errorf("failed to determine OGC SSH private key path: %w", err) + return common.OSRunnerResult{}, fmt.Errorf("failed to determine OGC SSH private key path: %w", err) } logger.Logf("Starting SSH; connect with `ssh -i %s %s@%s`", sshPrivateKeyPath, instance.Username, instance.IP) - client := NewSSHClient(instance.IP, instance.Username, sshAuth, logger) + client := tssh.NewClient(instance.IP, instance.Username, sshAuth, logger) connectCtx, connectCancel := context.WithTimeout(ctx, 10*time.Minute) defer connectCancel() err = client.Connect(connectCtx) if err != nil { logger.Logf("Failed to connect to instance %s: %s", instance.IP, err) - return OSRunnerResult{}, fmt.Errorf("failed to connect to instance %s: %w", instance.Name, err) + return common.OSRunnerResult{}, fmt.Errorf("failed to connect to instance %s: %w", instance.Name, err) } defer client.Close() logger.Logf("Connected over SSH") @@ -423,14 +346,14 @@ func (r *Runner) runInstance(ctx context.Context, sshAuth ssh.AuthMethod, logger err = batch.OS.Runner.Prepare(ctx, client, logger, batch.OS.Arch, r.cfg.GOVersion) if err != nil { logger.Logf("Failed to prepare instance: %s", err) - return OSRunnerResult{}, fmt.Errorf("failed to prepare instance %s: %w", instance.Name, err) + return common.OSRunnerResult{}, fmt.Errorf("failed to prepare instance %s: %w", instance.Name, err) } // now its prepared, add to state instance.Prepared = true err = r.addOrUpdateInstance(instance) if err != nil { - return OSRunnerResult{}, fmt.Errorf("failed to save instance state %s: %w", instance.Name, err) + return common.OSRunnerResult{}, fmt.Errorf("failed to save instance state %s: %w", instance.Name, err) } } @@ -438,7 +361,7 @@ func (r *Runner) runInstance(ctx context.Context, sshAuth ssh.AuthMethod, logger err = batch.OS.Runner.Copy(ctx, client, logger, repoArchive, r.getBuilds(batch)) if err != nil { logger.Logf("Failed to copy files instance: %s", err) - return OSRunnerResult{}, fmt.Errorf("failed to copy files to instance %s: %w", instance.Name, err) + return common.OSRunnerResult{}, fmt.Errorf("failed to copy files to instance %s: %w", instance.Name, err) } // start with the ExtraEnv first preventing the other environment flags below // from being overwritten @@ -453,7 +376,7 @@ func (r *Runner) runInstance(ctx context.Context, sshAuth ssh.AuthMethod, logger logger.Logf("Waiting for stack to be ready...") stack, err := r.getStackForBatchID(batch.ID) if err != nil { - return OSRunnerResult{}, err + return common.OSRunnerResult{}, err } env["ELASTICSEARCH_HOST"] = stack.Elasticsearch env["ELASTICSEARCH_USERNAME"] = stack.Username @@ -472,7 +395,7 @@ func (r *Runner) runInstance(ctx context.Context, sshAuth ssh.AuthMethod, logger result, err := batch.OS.Runner.Run(ctx, r.cfg.VerboseMode, client, logger, r.cfg.AgentVersion, batch.ID, batch.Batch, env) if err != nil { logger.Logf("Failed to execute tests on instance: %s", err) - return OSRunnerResult{}, fmt.Errorf("failed to execute tests on instance %s: %w", instance.Name, err) + return common.OSRunnerResult{}, fmt.Errorf("failed to execute tests on instance %s: %w", instance.Name, err) } // fetch any diagnostics @@ -519,8 +442,8 @@ func (r *Runner) validate() error { } // getBuilds returns the build for the batch. -func (r *Runner) getBuilds(b OSBatch) []Build { - var builds []Build +func (r *Runner) getBuilds(b common.OSBatch) []common.Build { + var builds []common.Build formats := []string{"targz", "zip", "rpm", "deb"} binaryName := "elastic-agent" @@ -560,7 +483,7 @@ func (r *Runner) getBuilds(b OSBatch) []Build { continue } packageName := filepath.Join(r.cfg.BuildDir, fmt.Sprintf("%s-%s-%s", binaryName, r.cfg.AgentVersion, suffix)) - build := Build{ + build := common.Build{ Version: r.cfg.ReleaseVersion, Type: b.OS.Type, Arch: arch, @@ -631,15 +554,15 @@ func (r *Runner) createSSHKey(dir string) (ssh.AuthMethod, error) { r.logger.Logf("Create SSH keys to use for SSH") _ = os.Remove(privateKey) _ = os.Remove(publicKey) - pri, err := newSSHPrivateKey() + pri, err := tssh.NewPrivateKey() if err != nil { return nil, fmt.Errorf("failed to create ssh private key: %w", err) } - pubBytes, err := newSSHPublicKey(&pri.PublicKey) + pubBytes, err := tssh.NewPublicKey(&pri.PublicKey) if err != nil { return nil, fmt.Errorf("failed to create ssh public key: %w", err) } - priBytes := sshEncodeToPEM(pri) + priBytes := tssh.EncodeToPEM(pri) err = os.WriteFile(privateKey, priBytes, 0600) if err != nil { return nil, fmt.Errorf("failed to write ssh private key: %w", err) @@ -704,12 +627,12 @@ func (r *Runner) startStacks(ctx context.Context) error { for _, version := range versions { id := strings.Replace(version, ".", "", -1) requests = append(requests, stackReq{ - request: StackRequest{ID: id, Version: version}, + request: common.StackRequest{ID: id, Version: version}, stack: r.findStack(id), }) } - reportResult := func(version string, stack Stack, err error) { + reportResult := func(version string, stack common.Stack, err error) { r.batchToStackMx.Lock() defer r.batchToStackMx.Unlock() res := stackRes{ @@ -731,7 +654,7 @@ func (r *Runner) startStacks(ctx context.Context) error { for _, request := range requests { go func(ctx context.Context, req stackReq) { var err error - var stack Stack + var stack common.Stack if req.stack != nil { stack = *req.stack } else { @@ -771,7 +694,7 @@ func (r *Runner) startStacks(ctx context.Context) error { return nil } -func (r *Runner) getStackForBatchID(id string) (Stack, error) { +func (r *Runner) getStackForBatchID(id string) (common.Stack, error) { r.batchToStackMx.Lock() res, ok := r.batchToStack[id] if ok { @@ -780,7 +703,7 @@ func (r *Runner) getStackForBatchID(id string) (Stack, error) { } _, ok = r.batchToStackCh[id] if ok { - return Stack{}, fmt.Errorf("getStackForBatchID called twice; this is not allowed") + return common.Stack{}, fmt.Errorf("getStackForBatchID called twice; this is not allowed") } ch := make(chan stackRes, 1) r.batchToStackCh[id] = ch @@ -792,7 +715,7 @@ func (r *Runner) getStackForBatchID(id string) (Stack, error) { defer t.Stop() select { case <-t.C: - return Stack{}, fmt.Errorf("failed waiting for a response after 12 minutes") + return common.Stack{}, fmt.Errorf("failed waiting for a response after 12 minutes") case res = <-ch: return res.stack, res.err } @@ -803,7 +726,7 @@ func (r *Runner) findInstance(id string) (StateInstance, bool) { defer r.stateMx.Unlock() for _, existing := range r.state.Instances { if existing.Same(StateInstance{ - Instance: Instance{ID: id, Provisioner: r.ip.Name()}}) { + Instance: common.Instance{ID: id, Provisioner: r.ip.Name()}}) { return existing, true } } @@ -830,18 +753,18 @@ func (r *Runner) addOrUpdateInstance(instance StateInstance) error { return r.writeState() } -func (r *Runner) findStack(id string) *Stack { +func (r *Runner) findStack(id string) *common.Stack { r.stateMx.Lock() defer r.stateMx.Unlock() for _, existing := range r.state.Stacks { - if existing.Same(Stack{ID: id, Provisioner: r.sp.Name()}) { + if existing.Same(common.Stack{ID: id, Provisioner: r.sp.Name()}) { return &existing } } return nil } -func (r *Runner) addOrUpdateStack(stack Stack) error { +func (r *Runner) addOrUpdateStack(stack common.Stack) error { r.stateMx.Lock() defer r.stateMx.Unlock() @@ -891,7 +814,7 @@ func (r *Runner) getStatePath() string { return filepath.Join(r.cfg.StateDir, "state.yml") } -func (r *Runner) mergeResults(results map[string]OSRunnerResult) (Result, error) { +func (r *Runner) mergeResults(results map[string]common.OSRunnerResult) (Result, error) { var rawOutput bytes.Buffer var jsonOutput bytes.Buffer var suites JUnitTestSuites @@ -933,14 +856,7 @@ func (s StateInstance) Same(other StateInstance) bool { s.ID == other.ID } -// Same returns true if other is the same stack as this one. -// Two stacks are considered the same if their provisioner and ID are the same. -func (s Stack) Same(other Stack) bool { - return s.Provisioner == other.Provisioner && - s.ID == other.ID -} - -func mergePackageResult(pkg OSRunnerPackageResult, batchName string, sudo bool, rawOutput io.Writer, jsonOutput io.Writer, suites *JUnitTestSuites) error { +func mergePackageResult(pkg common.OSRunnerPackageResult, batchName string, sudo bool, rawOutput io.Writer, jsonOutput io.Writer, suites *JUnitTestSuites) error { suffix := "" sudoStr := "false" if sudo { @@ -949,7 +865,7 @@ func mergePackageResult(pkg OSRunnerPackageResult, batchName string, sudo bool, } if pkg.Output != nil { rawLogger := &runnerLogger{writer: rawOutput, timestamp: false} - pkgWriter := newPrefixOutput(rawLogger, fmt.Sprintf("%s(%s)%s: ", pkg.Name, batchName, suffix)) + pkgWriter := common.NewPrefixOutput(rawLogger, fmt.Sprintf("%s(%s)%s: ", pkg.Name, batchName, suffix)) _, err := pkgWriter.Write(pkg.Output) if err != nil { return fmt.Errorf("failed to write raw output from %s %s: %w", batchName, pkg.Name, err) @@ -986,167 +902,13 @@ func mergePackageResult(pkg OSRunnerPackageResult, batchName string, sudo bool, return nil } -func findBatchByID(id string, batches []OSBatch) (OSBatch, bool) { +func findBatchByID(id string, batches []common.OSBatch) (common.OSBatch, bool) { for _, batch := range batches { if batch.ID == id { return batch, true } } - return OSBatch{}, false -} - -func batchInGroups(batch define.Batch, groups []string) bool { - for _, g := range groups { - if batch.Group == g { - return true - } - } - return false -} - -func createBatches(batch define.Batch, platforms []define.OS, groups []string, matrix bool) ([]OSBatch, error) { - var batches []OSBatch - if len(groups) > 0 && !batchInGroups(batch, groups) { - return nil, nil - } - specifics, err := getSupported(batch.OS, platforms) - if errors.Is(err, ErrOSNotSupported) { - var s SupportedOS - s.OS.Type = batch.OS.Type - s.OS.Arch = batch.OS.Arch - s.OS.Distro = batch.OS.Distro - if s.OS.Distro == "" { - s.OS.Distro = "unknown" - } - if s.OS.Version == "" { - s.OS.Version = "unknown" - } - b := OSBatch{ - OS: s, - Batch: batch, - Skip: true, - } - b.ID = createBatchID(b) - batches = append(batches, b) - return batches, nil - } else if err != nil { - return nil, err - } - if matrix { - for _, s := range specifics { - b := OSBatch{ - OS: s, - Batch: batch, - Skip: false, - } - b.ID = createBatchID(b) - batches = append(batches, b) - } - } else { - b := OSBatch{ - OS: specifics[0], - Batch: batch, - Skip: false, - } - b.ID = createBatchID(b) - batches = append(batches, b) - } - return batches, nil -} - -func filterSingleTest(batches []OSBatch, singleTest string) ([]OSBatch, error) { - var filtered []OSBatch - for _, batch := range batches { - batch, ok := filterSingleTestBatch(batch, singleTest) - if ok { - filtered = append(filtered, batch) - } - } - if len(filtered) == 0 { - return nil, fmt.Errorf("test not found: %s", singleTest) - } - return filtered, nil -} - -func filterSingleTestBatch(batch OSBatch, testName string) (OSBatch, bool) { - for _, pt := range batch.Batch.Tests { - for _, t := range pt.Tests { - if t.Name == testName { - // filter batch to only run one test - batch.Batch.Tests = []define.BatchPackageTests{ - { - Name: pt.Name, - Tests: []define.BatchPackageTest{t}, - }, - } - batch.Batch.SudoTests = nil - // remove stack requirement when the test doesn't need a stack - if !t.Stack { - batch.Batch.Stack = nil - } - return batch, true - } - } - } - for _, pt := range batch.Batch.SudoTests { - for _, t := range pt.Tests { - if t.Name == testName { - // filter batch to only run one test - batch.Batch.SudoTests = []define.BatchPackageTests{ - { - Name: pt.Name, - Tests: []define.BatchPackageTest{t}, - }, - } - batch.Batch.Tests = nil - // remove stack requirement when the test doesn't need a stack - if !t.Stack { - batch.Batch.Stack = nil - } - return batch, true - } - } - } - return batch, false -} - -func filterSupportedOS(batches []OSBatch, provisioner InstanceProvisioner) []OSBatch { - var filtered []OSBatch - for _, batch := range batches { - if ok := provisioner.Supported(batch.OS.OS); ok { - filtered = append(filtered, batch) - } - } - return filtered -} - -// createBatchID creates a consistent/unique ID for the batch -// -// ID needs to be consistent so each execution of the runner always -// selects the same ID for each batch. -func createBatchID(batch OSBatch) string { - id := batch.OS.Type + "-" + batch.OS.Arch - if batch.OS.Type == define.Linux { - id += "-" + batch.OS.Distro - } - if batch.OS.Version != "" { - id += "-" + strings.Replace(batch.OS.Version, ".", "", -1) - } - if batch.OS.Type == define.Kubernetes && batch.OS.DockerVariant != "" { - id += "-" + batch.OS.DockerVariant - } - id += "-" + strings.Replace(batch.Batch.Group, ".", "", -1) - - // The batchID needs to be at most 63 characters long otherwise - // OGC will fail to instantiate the VM. - maxIDLen := 63 - if len(id) > maxIDLen { - hash := fmt.Sprintf("%x", md5.Sum([]byte(id))) - hashLen := utf8.RuneCountInString(hash) - id = id[:maxIDLen-hashLen-1] + "-" + hash - } - - return strings.ToLower(id) + return common.OSBatch{}, false } type runnerLogger struct { @@ -1163,20 +925,30 @@ func (l *runnerLogger) Logf(format string, args ...any) { } type batchLogger struct { - wrapped Logger + wrapped common.Logger prefix string } +func filterSupportedOS(batches []common.OSBatch, provisioner common.InstanceProvisioner) []common.OSBatch { + var filtered []common.OSBatch + for _, batch := range batches { + if ok := provisioner.Supported(batch.OS.OS); ok { + filtered = append(filtered, batch) + } + } + return filtered +} + func (b *batchLogger) Logf(format string, args ...any) { b.wrapped.Logf("(%s) %s", b.prefix, fmt.Sprintf(format, args...)) } type stackRes struct { - stack Stack + stack common.Stack err error } type stackReq struct { - request StackRequest - stack *Stack + request common.StackRequest + stack *common.Stack } diff --git a/pkg/testing/runner/runner_test.go b/pkg/testing/runner/runner_test.go index 08643650f1b..31baf041018 100644 --- a/pkg/testing/runner/runner_test.go +++ b/pkg/testing/runner/runner_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/elastic/elastic-agent/pkg/testing/common" "github.com/elastic/elastic-agent/pkg/testing/define" ) @@ -23,7 +24,7 @@ func TestNewRunner_Clean(t *testing.T) { err := os.MkdirAll(stateDir, 0755) require.NoError(t, err) - cfg := Config{ + cfg := common.Config{ AgentVersion: "8.10.0", StackVersion: "8.10.0-SNAPSHOT", BuildDir: filepath.Join(tmpdir, "build"), @@ -37,7 +38,7 @@ func TestNewRunner_Clean(t *testing.T) { r, err := NewRunner(cfg, ip, sp) require.NoError(t, err) - i1 := Instance{ + i1 := common.Instance{ ID: "id-1", Name: "name-1", Provisioner: ip.Name(), @@ -51,7 +52,7 @@ func TestNewRunner_Clean(t *testing.T) { Prepared: true, }) require.NoError(t, err) - i2 := Instance{ + i2 := common.Instance{ ID: "id-2", Name: "name-2", Provisioner: ip.Name(), @@ -65,7 +66,7 @@ func TestNewRunner_Clean(t *testing.T) { Prepared: true, }) require.NoError(t, err) - s1 := Stack{ + s1 := common.Stack{ ID: "id-1", Provisioner: sp.Name(), Version: "8.10.0", @@ -73,7 +74,7 @@ func TestNewRunner_Clean(t *testing.T) { } err = r.addOrUpdateStack(s1) require.NoError(t, err) - s2 := Stack{ + s2 := common.Stack{ ID: "id-2", Provisioner: sp.Name(), Version: "8.9.0", @@ -90,35 +91,35 @@ func TestNewRunner_Clean(t *testing.T) { err = r.Clean() require.NoError(t, err) - assert.ElementsMatch(t, ip.instances, []Instance{i1, i2}) - assert.ElementsMatch(t, sp.deletedStacks, []Stack{s1, s2}) + assert.ElementsMatch(t, ip.instances, []common.Instance{i1, i2}) + assert.ElementsMatch(t, sp.deletedStacks, []common.Stack{s1, s2}) } type fakeInstanceProvisioner struct { - batches []OSBatch - instances []Instance + batches []common.OSBatch + instances []common.Instance } func (p *fakeInstanceProvisioner) Name() string { return "fake" } -func (p *fakeInstanceProvisioner) Type() ProvisionerType { - return ProvisionerTypeVM +func (p *fakeInstanceProvisioner) Type() common.ProvisionerType { + return common.ProvisionerTypeVM } -func (p *fakeInstanceProvisioner) SetLogger(_ Logger) { +func (p *fakeInstanceProvisioner) SetLogger(_ common.Logger) { } func (p *fakeInstanceProvisioner) Supported(_ define.OS) bool { return true } -func (p *fakeInstanceProvisioner) Provision(_ context.Context, _ Config, batches []OSBatch) ([]Instance, error) { +func (p *fakeInstanceProvisioner) Provision(_ context.Context, _ common.Config, batches []common.OSBatch) ([]common.Instance, error) { p.batches = batches - var instances []Instance + var instances []common.Instance for _, batch := range batches { - instances = append(instances, Instance{ + instances = append(instances, common.Instance{ ID: batch.ID, Name: batch.ID, IP: "127.0.0.1", @@ -130,29 +131,29 @@ func (p *fakeInstanceProvisioner) Provision(_ context.Context, _ Config, batches return instances, nil } -func (p *fakeInstanceProvisioner) Clean(_ context.Context, _ Config, instances []Instance) error { +func (p *fakeInstanceProvisioner) Clean(_ context.Context, _ common.Config, instances []common.Instance) error { p.instances = instances return nil } type fakeStackProvisioner struct { mx sync.Mutex - requests []StackRequest - deletedStacks []Stack + requests []common.StackRequest + deletedStacks []common.Stack } func (p *fakeStackProvisioner) Name() string { return "fake" } -func (p *fakeStackProvisioner) SetLogger(_ Logger) { +func (p *fakeStackProvisioner) SetLogger(_ common.Logger) { } -func (p *fakeStackProvisioner) Create(_ context.Context, request StackRequest) (Stack, error) { +func (p *fakeStackProvisioner) Create(_ context.Context, request common.StackRequest) (common.Stack, error) { p.mx.Lock() defer p.mx.Unlock() p.requests = append(p.requests, request) - return Stack{ + return common.Stack{ ID: request.ID, Version: request.Version, Elasticsearch: "http://localhost:9200", @@ -164,12 +165,12 @@ func (p *fakeStackProvisioner) Create(_ context.Context, request StackRequest) ( }, nil } -func (p *fakeStackProvisioner) WaitForReady(_ context.Context, stack Stack) (Stack, error) { +func (p *fakeStackProvisioner) WaitForReady(_ context.Context, stack common.Stack) (common.Stack, error) { stack.Ready = true return stack, nil } -func (p *fakeStackProvisioner) Delete(_ context.Context, stack Stack) error { +func (p *fakeStackProvisioner) Delete(_ context.Context, stack common.Stack) error { p.mx.Lock() defer p.mx.Unlock() p.deletedStacks = append(p.deletedStacks, stack) diff --git a/pkg/testing/runner/ssh.go b/pkg/testing/ssh/client.go similarity index 70% rename from pkg/testing/runner/ssh.go rename to pkg/testing/ssh/client.go index 51dcc7f688a..831d325ba8c 100644 --- a/pkg/testing/runner/ssh.go +++ b/pkg/testing/ssh/client.go @@ -2,15 +2,11 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package runner +package ssh import ( "bytes" "context" - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "encoding/pem" "fmt" "io" "net" @@ -21,99 +17,21 @@ import ( "golang.org/x/crypto/ssh" ) -// newSSHPrivateKey creates RSA private key -func newSSHPrivateKey() (*rsa.PrivateKey, error) { - pk, err := rsa.GenerateKey(rand.Reader, 2056) - if err != nil { - return nil, err - } - err = pk.Validate() - if err != nil { - return nil, err - } - return pk, nil -} - -// sshEncodeToPEM encodes private key to PEM format -func sshEncodeToPEM(privateKey *rsa.PrivateKey) []byte { - der := x509.MarshalPKCS1PrivateKey(privateKey) - privBlock := pem.Block{ - Type: "RSA PRIVATE KEY", - Headers: nil, - Bytes: der, - } - return pem.EncodeToMemory(&privBlock) -} - -// newSSHPublicKey returns bytes for writing to .pub file -func newSSHPublicKey(pk *rsa.PublicKey) ([]byte, error) { - pub, err := ssh.NewPublicKey(pk) - if err != nil { - return nil, err - } - return ssh.MarshalAuthorizedKey(pub), nil -} - -type fileContentsOpts struct { - command string -} - -// FileContentsOpt provides an option to modify how fetching files from the remote host work. -type FileContentsOpt func(opts *fileContentsOpts) - -// WithContentFetchCommand changes the command to use for fetching the file contents. -func WithContentFetchCommand(command string) FileContentsOpt { - return func(opts *fileContentsOpts) { - opts.command = command - } -} - -// SSHClient is a *ssh.Client that provides a nice interface to work with. -type SSHClient interface { - // Connect connects to the host. - Connect(ctx context.Context) error - - // ConnectWithTimeout connects to the host with a timeout. - ConnectWithTimeout(ctx context.Context, timeout time.Duration) error - - // Close closes the client. - Close() error - - // Reconnect disconnects and reconnected to the host. - Reconnect(ctx context.Context) error - - // ReconnectWithTimeout disconnects and reconnected to the host with a timeout. - ReconnectWithTimeout(ctx context.Context, timeout time.Duration) error - - // NewSession opens a new Session for this host. - NewSession() (*ssh.Session, error) - - // Exec runs a command on the host. - Exec(ctx context.Context, cmd string, args []string, stdin io.Reader) ([]byte, []byte, error) - - // ExecWithRetry runs the command on loop waiting the interval between calls - ExecWithRetry(ctx context.Context, cmd string, args []string, interval time.Duration) ([]byte, []byte, error) - - // Copy copies the filePath to the host at dest. - Copy(filePath string, dest string) error - - // GetFileContents returns the file content. - GetFileContents(ctx context.Context, filename string, opts ...FileContentsOpt) ([]byte, error) - - // GetFileContentsOutput returns the file content writing to output. - GetFileContentsOutput(ctx context.Context, filename string, output io.Writer, opts ...FileContentsOpt) error +type logger interface { + // Logf logs the message for this runner. + Logf(format string, args ...any) } type sshClient struct { ip string username string auth ssh.AuthMethod - logger Logger + logger logger c *ssh.Client } -// NewSSHClient creates a new SSH client connection to the host. -func NewSSHClient(ip string, username string, sshAuth ssh.AuthMethod, logger Logger) SSHClient { +// NewClient creates a new SSH client connection to the host. +func NewClient(ip string, username string, sshAuth ssh.AuthMethod, logger logger) SSHClient { return &sshClient{ ip: ip, username: username, diff --git a/pkg/testing/ssh/file.go b/pkg/testing/ssh/file.go new file mode 100644 index 00000000000..f40d050c75f --- /dev/null +++ b/pkg/testing/ssh/file.go @@ -0,0 +1,19 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package ssh + +type fileContentsOpts struct { + command string +} + +// FileContentsOpt provides an option to modify how fetching files from the remote host work. +type FileContentsOpt func(opts *fileContentsOpts) + +// WithContentFetchCommand changes the command to use for fetching the file contents. +func WithContentFetchCommand(command string) FileContentsOpt { + return func(opts *fileContentsOpts) { + opts.command = command + } +} diff --git a/pkg/testing/ssh/interface.go b/pkg/testing/ssh/interface.go new file mode 100644 index 00000000000..00eccbe5c7a --- /dev/null +++ b/pkg/testing/ssh/interface.go @@ -0,0 +1,49 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package ssh + +import ( + "context" + "io" + "time" + + "golang.org/x/crypto/ssh" +) + +// SSHClient is a *ssh.Client that provides a nice interface to work with. +type SSHClient interface { + // Connect connects to the host. + Connect(ctx context.Context) error + + // ConnectWithTimeout connects to the host with a timeout. + ConnectWithTimeout(ctx context.Context, timeout time.Duration) error + + // Close closes the client. + Close() error + + // Reconnect disconnects and reconnected to the host. + Reconnect(ctx context.Context) error + + // ReconnectWithTimeout disconnects and reconnected to the host with a timeout. + ReconnectWithTimeout(ctx context.Context, timeout time.Duration) error + + // NewSession opens a new Session for this host. + NewSession() (*ssh.Session, error) + + // Exec runs a command on the host. + Exec(ctx context.Context, cmd string, args []string, stdin io.Reader) ([]byte, []byte, error) + + // ExecWithRetry runs the command on loop waiting the interval between calls + ExecWithRetry(ctx context.Context, cmd string, args []string, interval time.Duration) ([]byte, []byte, error) + + // Copy copies the filePath to the host at dest. + Copy(filePath string, dest string) error + + // GetFileContents returns the file content. + GetFileContents(ctx context.Context, filename string, opts ...FileContentsOpt) ([]byte, error) + + // GetFileContentsOutput returns the file content writing to output. + GetFileContentsOutput(ctx context.Context, filename string, output io.Writer, opts ...FileContentsOpt) error +} diff --git a/pkg/testing/ssh/keys.go b/pkg/testing/ssh/keys.go new file mode 100644 index 00000000000..5f53a88a0ed --- /dev/null +++ b/pkg/testing/ssh/keys.go @@ -0,0 +1,47 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package ssh + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + + "golang.org/x/crypto/ssh" +) + +// NewPrivateKey creates RSA private key +func NewPrivateKey() (*rsa.PrivateKey, error) { + pk, err := rsa.GenerateKey(rand.Reader, 2056) + if err != nil { + return nil, err + } + err = pk.Validate() + if err != nil { + return nil, err + } + return pk, nil +} + +// EncodeToPEM encodes private key to PEM format +func EncodeToPEM(privateKey *rsa.PrivateKey) []byte { + der := x509.MarshalPKCS1PrivateKey(privateKey) + privBlock := pem.Block{ + Type: "RSA PRIVATE KEY", + Headers: nil, + Bytes: der, + } + return pem.EncodeToMemory(&privBlock) +} + +// NewPublicKey returns bytes for writing to .pub file +func NewPublicKey(pk *rsa.PublicKey) ([]byte, error) { + pub, err := ssh.NewPublicKey(pk) + if err != nil { + return nil, err + } + return ssh.MarshalAuthorizedKey(pub), nil +} diff --git a/pkg/testing/supported/batch.go b/pkg/testing/supported/batch.go new file mode 100644 index 00000000000..a53cf6364c5 --- /dev/null +++ b/pkg/testing/supported/batch.go @@ -0,0 +1,183 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package supported + +import ( + "crypto/md5" + "errors" + "fmt" + "strings" + "unicode/utf8" + + "github.com/elastic/elastic-agent/pkg/testing/common" + "github.com/elastic/elastic-agent/pkg/testing/define" +) + +// CreateBatches creates the OSBatch set based on the defined supported OS's. +func CreateBatches(batches []define.Batch, platforms []define.OS, groups []string, matrix bool, singleTest string) ([]common.OSBatch, error) { + var err error + var osBatches []common.OSBatch + for _, b := range batches { + lbs, err := createBatchesFromBatch(b, platforms, groups, matrix) + if err != nil { + return nil, err + } + if lbs != nil { + osBatches = append(osBatches, lbs...) + } + } + if singleTest != "" { + osBatches, err = filterSingleTest(osBatches, singleTest) + if err != nil { + return nil, err + } + } + + return osBatches, nil +} + +func createBatchesFromBatch(batch define.Batch, platforms []define.OS, groups []string, matrix bool) ([]common.OSBatch, error) { + var batches []common.OSBatch + if len(groups) > 0 && !batchInGroups(batch, groups) { + return nil, nil + } + specifics, err := getSupported(batch.OS, platforms) + if errors.Is(err, ErrOSNotSupported) { + var s common.SupportedOS + s.OS.Type = batch.OS.Type + s.OS.Arch = batch.OS.Arch + s.OS.Distro = batch.OS.Distro + if s.OS.Distro == "" { + s.OS.Distro = "unknown" + } + if s.OS.Version == "" { + s.OS.Version = "unknown" + } + b := common.OSBatch{ + OS: s, + Batch: batch, + Skip: true, + } + b.ID = createBatchID(b) + batches = append(batches, b) + return batches, nil + } else if err != nil { + return nil, err + } + if matrix { + for _, s := range specifics { + b := common.OSBatch{ + OS: s, + Batch: batch, + Skip: false, + } + b.ID = createBatchID(b) + batches = append(batches, b) + } + } else { + b := common.OSBatch{ + OS: specifics[0], + Batch: batch, + Skip: false, + } + b.ID = createBatchID(b) + batches = append(batches, b) + } + return batches, nil +} + +func batchInGroups(batch define.Batch, groups []string) bool { + for _, g := range groups { + if batch.Group == g { + return true + } + } + return false +} + +func filterSingleTest(batches []common.OSBatch, singleTest string) ([]common.OSBatch, error) { + var filtered []common.OSBatch + for _, batch := range batches { + batch, ok := filterSingleTestBatch(batch, singleTest) + if ok { + filtered = append(filtered, batch) + } + } + if len(filtered) == 0 { + return nil, fmt.Errorf("test not found: %s", singleTest) + } + return filtered, nil +} + +func filterSingleTestBatch(batch common.OSBatch, testName string) (common.OSBatch, bool) { + for _, pt := range batch.Batch.Tests { + for _, t := range pt.Tests { + if t.Name == testName { + // filter batch to only run one test + batch.Batch.Tests = []define.BatchPackageTests{ + { + Name: pt.Name, + Tests: []define.BatchPackageTest{t}, + }, + } + batch.Batch.SudoTests = nil + // remove stack requirement when the test doesn't need a stack + if !t.Stack { + batch.Batch.Stack = nil + } + return batch, true + } + } + } + for _, pt := range batch.Batch.SudoTests { + for _, t := range pt.Tests { + if t.Name == testName { + // filter batch to only run one test + batch.Batch.SudoTests = []define.BatchPackageTests{ + { + Name: pt.Name, + Tests: []define.BatchPackageTest{t}, + }, + } + batch.Batch.Tests = nil + // remove stack requirement when the test doesn't need a stack + if !t.Stack { + batch.Batch.Stack = nil + } + return batch, true + } + } + } + return batch, false +} + +// createBatchID creates a consistent/unique ID for the batch +// +// ID needs to be consistent so each execution of the runner always +// selects the same ID for each batch. +func createBatchID(batch common.OSBatch) string { + id := batch.OS.Type + "-" + batch.OS.Arch + if batch.OS.Type == define.Linux { + id += "-" + batch.OS.Distro + } + if batch.OS.Version != "" { + id += "-" + strings.Replace(batch.OS.Version, ".", "", -1) + } + if batch.OS.Type == define.Kubernetes && batch.OS.DockerVariant != "" { + id += "-" + batch.OS.DockerVariant + } + id += "-" + strings.Replace(batch.Batch.Group, ".", "", -1) + + // The batchID needs to be at most 63 characters long otherwise + // OGC will fail to instantiate the VM. + maxIDLen := 63 + if len(id) > maxIDLen { + hash := fmt.Sprintf("%x", md5.Sum([]byte(id))) + hashLen := utf8.RuneCountInString(hash) + id = id[:maxIDLen-hashLen-1] + "-" + hash + } + + return strings.ToLower(id) +} diff --git a/pkg/testing/runner/supported.go b/pkg/testing/supported/supported.go similarity index 80% rename from pkg/testing/runner/supported.go rename to pkg/testing/supported/supported.go index 94bea62847a..1ced9674b26 100644 --- a/pkg/testing/runner/supported.go +++ b/pkg/testing/supported/supported.go @@ -2,15 +2,17 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package runner +package supported import ( "errors" "fmt" - "github.com/elastic/elastic-agent/pkg/testing/kubernetes" - + "github.com/elastic/elastic-agent/pkg/testing/common" "github.com/elastic/elastic-agent/pkg/testing/define" + "github.com/elastic/elastic-agent/pkg/testing/kubernetes" + "github.com/elastic/elastic-agent/pkg/testing/linux" + "github.com/elastic/elastic-agent/pkg/testing/windows" ) const ( @@ -24,138 +26,130 @@ var ( ErrOSNotSupported = errors.New("os/arch not currently supported") ) -// SupportedOS maps a OS definition to a OSRunner. -type SupportedOS struct { - define.OS - - // Runner is the runner to use for the OS. - Runner OSRunner -} - var ( // UbuntuAMD64_2404 - Ubuntu (amd64) 24.04 - UbuntuAMD64_2404 = SupportedOS{ + UbuntuAMD64_2404 = common.SupportedOS{ OS: define.OS{ Type: define.Linux, Arch: define.AMD64, Distro: Ubuntu, Version: "24.04", }, - Runner: DebianRunner{}, + Runner: linux.DebianRunner{}, } // UbuntuAMD64_2204 - Ubuntu (amd64) 22.04 - UbuntuAMD64_2204 = SupportedOS{ + UbuntuAMD64_2204 = common.SupportedOS{ OS: define.OS{ Type: define.Linux, Arch: define.AMD64, Distro: Ubuntu, Version: "22.04", }, - Runner: DebianRunner{}, + Runner: linux.DebianRunner{}, } // UbuntuAMD64_2004 - Ubuntu (amd64) 20.04 - UbuntuAMD64_2004 = SupportedOS{ + UbuntuAMD64_2004 = common.SupportedOS{ OS: define.OS{ Type: define.Linux, Arch: define.AMD64, Distro: Ubuntu, Version: "20.04", }, - Runner: DebianRunner{}, + Runner: linux.DebianRunner{}, } // UbuntuARM64_2404 - Ubuntu (arm64) 24.04 - UbuntuARM64_2404 = SupportedOS{ + UbuntuARM64_2404 = common.SupportedOS{ OS: define.OS{ Type: define.Linux, Arch: define.ARM64, Distro: Ubuntu, Version: "24.04", }, - Runner: DebianRunner{}, + Runner: linux.DebianRunner{}, } // UbuntuARM64_2204 - Ubuntu (arm64) 22.04 - UbuntuARM64_2204 = SupportedOS{ + UbuntuARM64_2204 = common.SupportedOS{ OS: define.OS{ Type: define.Linux, Arch: define.ARM64, Distro: Ubuntu, Version: "22.04", }, - Runner: DebianRunner{}, + Runner: linux.DebianRunner{}, } // UbuntuARM64_2004 - Ubuntu (arm64) 20.04 - UbuntuARM64_2004 = SupportedOS{ + UbuntuARM64_2004 = common.SupportedOS{ OS: define.OS{ Type: define.Linux, Arch: define.ARM64, Distro: Ubuntu, Version: "20.04", }, - Runner: DebianRunner{}, + Runner: linux.DebianRunner{}, } // RhelAMD64_8 - RedHat Enterprise Linux (amd64) 8 - RhelAMD64_8 = SupportedOS{ + RhelAMD64_8 = common.SupportedOS{ OS: define.OS{ Type: define.Linux, Arch: define.AMD64, Distro: Rhel, Version: "8", }, - Runner: RhelRunner{}, + Runner: linux.RhelRunner{}, } // WindowsAMD64_2022 - Windows (amd64) Server 2022 - WindowsAMD64_2022 = SupportedOS{ + WindowsAMD64_2022 = common.SupportedOS{ OS: define.OS{ Type: define.Windows, Arch: define.AMD64, Version: "2022", }, - Runner: WindowsRunner{}, + Runner: windows.WindowsRunner{}, } // WindowsAMD64_2022_Core - Windows (amd64) Server 2022 Core - WindowsAMD64_2022_Core = SupportedOS{ + WindowsAMD64_2022_Core = common.SupportedOS{ OS: define.OS{ Type: define.Windows, Arch: define.AMD64, Version: "2022-core", }, - Runner: WindowsRunner{}, + Runner: windows.WindowsRunner{}, } // WindowsAMD64_2019 - Windows (amd64) Server 2019 - WindowsAMD64_2019 = SupportedOS{ + WindowsAMD64_2019 = common.SupportedOS{ OS: define.OS{ Type: define.Windows, Arch: define.AMD64, Version: "2019", }, - Runner: WindowsRunner{}, + Runner: windows.WindowsRunner{}, } // WindowsAMD64_2019_Core - Windows (amd64) Server 2019 Core - WindowsAMD64_2019_Core = SupportedOS{ + WindowsAMD64_2019_Core = common.SupportedOS{ OS: define.OS{ Type: define.Windows, Arch: define.AMD64, Version: "2019-core", }, - Runner: WindowsRunner{}, + Runner: windows.WindowsRunner{}, } // WindowsAMD64_2016 - Windows (amd64) Server 2016 - WindowsAMD64_2016 = SupportedOS{ + WindowsAMD64_2016 = common.SupportedOS{ OS: define.OS{ Type: define.Windows, Arch: define.AMD64, Version: "2016", }, - Runner: WindowsRunner{}, + Runner: windows.WindowsRunner{}, } // WindowsAMD64_2016_Core - Windows (amd64) Server 2016 Core - WindowsAMD64_2016_Core = SupportedOS{ + WindowsAMD64_2016_Core = common.SupportedOS{ OS: define.OS{ Type: define.Windows, Arch: define.AMD64, Version: "2016-core", }, - Runner: WindowsRunner{}, + Runner: windows.WindowsRunner{}, } ) @@ -167,7 +161,7 @@ var ( // In the case that a batch is not specific on the version and/or distro the first // one in this list will be picked. So it's best to place the one that we want the // most testing at the top. -var supported = []SupportedOS{ +var supported = []common.SupportedOS{ UbuntuAMD64_2404, UbuntuAMD64_2204, UbuntuAMD64_2004, @@ -187,9 +181,9 @@ var supported = []SupportedOS{ // init injects the kubernetes support list into the support list above func init() { for _, k8sSupport := range kubernetes.GetSupported() { - supported = append(supported, SupportedOS{ + supported = append(supported, common.SupportedOS{ OS: k8sSupport, - Runner: KubernetesRunner{}, + Runner: kubernetes.Runner{}, }) } } @@ -213,8 +207,8 @@ func osMatch(specific define.OS, notSpecific define.OS) bool { // getSupported returns all the supported based on the provided OS profile while using // the provided platforms as a filter. -func getSupported(os define.OS, platforms []define.OS) ([]SupportedOS, error) { - var match []SupportedOS +func getSupported(os define.OS, platforms []define.OS) ([]common.SupportedOS, error) { + var match []common.SupportedOS for _, s := range supported { if osMatch(s.OS, os) && allowedByPlatforms(s.OS, platforms) { match = append(match, s) diff --git a/pkg/testing/runner/supported_test.go b/pkg/testing/supported/supported_test.go similarity index 88% rename from pkg/testing/runner/supported_test.go rename to pkg/testing/supported/supported_test.go index 579d0c12697..f25458bdb18 100644 --- a/pkg/testing/runner/supported_test.go +++ b/pkg/testing/supported/supported_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package runner +package supported import ( "errors" @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/elastic/elastic-agent/pkg/testing/common" "github.com/elastic/elastic-agent/pkg/testing/define" ) @@ -18,7 +19,7 @@ func TestGetSupported(t *testing.T) { Name string OS define.OS Platforms []define.OS - Results []SupportedOS + Results []common.SupportedOS Err error }{ { @@ -36,7 +37,7 @@ func TestGetSupported(t *testing.T) { Arch: define.AMD64, Distro: Ubuntu, }, - Results: []SupportedOS{ + Results: []common.SupportedOS{ UbuntuAMD64_2404, UbuntuAMD64_2204, UbuntuAMD64_2004, @@ -50,7 +51,7 @@ func TestGetSupported(t *testing.T) { Distro: Ubuntu, Version: "20.04", }, - Results: []SupportedOS{ + Results: []common.SupportedOS{ UbuntuAMD64_2004, }, }, @@ -75,7 +76,7 @@ func TestGetSupported(t *testing.T) { Version: "20.04", }, }, - Results: []SupportedOS{ + Results: []common.SupportedOS{ UbuntuAMD64_2004, }, }, @@ -86,7 +87,7 @@ func TestGetSupported(t *testing.T) { Arch: define.AMD64, Distro: Rhel, }, - Results: []SupportedOS{ + Results: []common.SupportedOS{ RhelAMD64_8, }, }, @@ -98,7 +99,7 @@ func TestGetSupported(t *testing.T) { Distro: Rhel, Version: "8", }, - Results: []SupportedOS{ + Results: []common.SupportedOS{ RhelAMD64_8, }, }, diff --git a/pkg/testing/runner/windows.go b/pkg/testing/windows/windows.go similarity index 83% rename from pkg/testing/runner/windows.go rename to pkg/testing/windows/windows.go index 8e77b350232..9275ebd51c8 100644 --- a/pkg/testing/runner/windows.go +++ b/pkg/testing/windows/windows.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package runner +package windows import ( "context" @@ -13,14 +13,16 @@ import ( "strings" "time" + "github.com/elastic/elastic-agent/pkg/testing/common" "github.com/elastic/elastic-agent/pkg/testing/define" + "github.com/elastic/elastic-agent/pkg/testing/ssh" ) // WindowsRunner is a handler for running tests on Windows type WindowsRunner struct{} // Prepare the test -func (WindowsRunner) Prepare(ctx context.Context, sshClient SSHClient, logger Logger, arch string, goVersion string) error { +func (WindowsRunner) Prepare(ctx context.Context, sshClient ssh.SSHClient, logger common.Logger, arch string, goVersion string) error { // install chocolatey logger.Logf("Installing chocolatey") chocoInstall := `"[System.Net.ServicePointManager]::SecurityProtocol = 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))"` @@ -69,7 +71,7 @@ func (WindowsRunner) Prepare(ctx context.Context, sshClient SSHClient, logger Lo } // Copy places the required files on the host. -func (WindowsRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logger, repoArchive string, builds []Build) error { +func (WindowsRunner) Copy(ctx context.Context, sshClient ssh.SSHClient, logger common.Logger, repoArchive string, builds []common.Build) error { // copy the archive and extract it on the host (tar exists and can extract zip on windows) logger.Logf("Copying repo") destRepoName := filepath.Base(repoArchive) @@ -108,7 +110,7 @@ func (WindowsRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logge return fmt.Errorf("failed to read local SHA52 contents %s: %w", build.SHA512Path, err) } hostSHA512Path := filepath.Base(build.SHA512Path) - hostSHA512, err := sshClient.GetFileContents(ctx, hostSHA512Path, WithContentFetchCommand("type")) + hostSHA512, err := sshClient.GetFileContents(ctx, hostSHA512Path, ssh.WithContentFetchCommand("type")) if err == nil { if string(localSHA512) == string(hostSHA512) { logger.Logf("Skipping copy agent build %s; already the same", filepath.Base(build.Path)) @@ -159,7 +161,7 @@ func (WindowsRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logge } // Run the test -func (WindowsRunner) Run(ctx context.Context, verbose bool, c SSHClient, logger Logger, agentVersion string, prefix string, batch define.Batch, env map[string]string) (OSRunnerResult, error) { +func (WindowsRunner) Run(ctx context.Context, verbose bool, c ssh.SSHClient, logger common.Logger, agentVersion string, prefix string, batch define.Batch, env map[string]string) (common.OSRunnerResult, error) { var tests []string for _, pkg := range batch.Tests { for _, test := range pkg.Tests { @@ -173,13 +175,13 @@ func (WindowsRunner) Run(ctx context.Context, verbose bool, c SSHClient, logger } } - var result OSRunnerResult + var result common.OSRunnerResult if len(tests) > 0 { script := toPowershellScript(agentVersion, prefix, verbose, tests, env) results, err := runTestsOnWindows(ctx, logger, "non-sudo", prefix, script, c, batch.SudoTests) if err != nil { - return OSRunnerResult{}, fmt.Errorf("error running non-sudo tests: %w", err) + return common.OSRunnerResult{}, fmt.Errorf("error running non-sudo tests: %w", err) } result.Packages = results } @@ -190,7 +192,7 @@ func (WindowsRunner) Run(ctx context.Context, verbose bool, c SSHClient, logger results, err := runTestsOnWindows(ctx, logger, "sudo", prefix, script, c, batch.SudoTests) if err != nil { - return OSRunnerResult{}, fmt.Errorf("error running sudo tests: %w", err) + return common.OSRunnerResult{}, fmt.Errorf("error running sudo tests: %w", err) } result.SudoPackages = results @@ -199,7 +201,7 @@ func (WindowsRunner) Run(ctx context.Context, verbose bool, c SSHClient, logger } // Diagnostics gathers any diagnostics from the host. -func (WindowsRunner) Diagnostics(ctx context.Context, sshClient SSHClient, logger Logger, destination string) error { +func (WindowsRunner) Diagnostics(ctx context.Context, sshClient ssh.SSHClient, logger common.Logger, destination string) error { diagnosticDir := "agent\\build\\diagnostics" stdOut, _, err := sshClient.Exec(ctx, "dir", []string{diagnosticDir, "/b"}, nil) if err != nil { @@ -224,7 +226,7 @@ func (WindowsRunner) Diagnostics(ctx context.Context, sshClient SSHClient, logge if err != nil { return fmt.Errorf("failed to create file %s: %w", dp, err) } - err = sshClient.GetFileContentsOutput(ctx, fp, out, WithContentFetchCommand("type")) + err = sshClient.GetFileContentsOutput(ctx, fp, out, ssh.WithContentFetchCommand("type")) _ = out.Close() if err != nil { return fmt.Errorf("failed to copy file from remote host to %s: %w", dp, err) @@ -233,7 +235,7 @@ func (WindowsRunner) Diagnostics(ctx context.Context, sshClient SSHClient, logge return nil } -func sshRunPowershell(ctx context.Context, sshClient SSHClient, cmd string) ([]byte, []byte, error) { +func sshRunPowershell(ctx context.Context, sshClient ssh.SSHClient, cmd string) ([]byte, []byte, error) { return sshClient.Exec(ctx, "powershell", []string{ "-NoProfile", "-InputFormat", "None", @@ -269,7 +271,7 @@ func toPowershellScript(agentVersion string, prefix string, verbose bool, tests return sb.String() } -func runTestsOnWindows(ctx context.Context, logger Logger, name string, prefix string, script string, sshClient SSHClient, tests []define.BatchPackageTests) ([]OSRunnerPackageResult, error) { +func runTestsOnWindows(ctx context.Context, logger common.Logger, name string, prefix string, script string, sshClient ssh.SSHClient, tests []define.BatchPackageTests) ([]common.OSRunnerPackageResult, error) { execTest := strings.NewReader(script) session, err := sshClient.NewSession() @@ -277,8 +279,8 @@ func runTestsOnWindows(ctx context.Context, logger Logger, name string, prefix s return nil, fmt.Errorf("failed to start session: %w", err) } - session.Stdout = newPrefixOutput(logger, fmt.Sprintf("Test output (%s) (stdout): ", name)) - session.Stderr = newPrefixOutput(logger, fmt.Sprintf("Test output (%s) (stderr): ", name)) + session.Stdout = common.NewPrefixOutput(logger, fmt.Sprintf("Test output (%s) (stdout): ", name)) + session.Stderr = common.NewPrefixOutput(logger, fmt.Sprintf("Test output (%s) (stderr): ", name)) session.Stdin = execTest // allowed to fail because tests might fail logger.Logf("Running %s tests...", name) @@ -289,7 +291,7 @@ func runTestsOnWindows(ctx context.Context, logger Logger, name string, prefix s // this seems to always return an error _ = session.Close() - var result []OSRunnerPackageResult + var result []common.OSRunnerPackageResult // fetch the contents for each package for _, pkg := range tests { resultPkg, err := getWindowsRunnerPackageResult(ctx, sshClient, pkg, prefix) @@ -305,22 +307,22 @@ func toWindowsPath(path string) string { return strings.ReplaceAll(path, "/", "\\") } -func getWindowsRunnerPackageResult(ctx context.Context, sshClient SSHClient, pkg define.BatchPackageTests, prefix string) (OSRunnerPackageResult, error) { +func getWindowsRunnerPackageResult(ctx context.Context, sshClient ssh.SSHClient, pkg define.BatchPackageTests, prefix string) (common.OSRunnerPackageResult, error) { var err error - var resultPkg OSRunnerPackageResult + var resultPkg common.OSRunnerPackageResult resultPkg.Name = pkg.Name outputPath := fmt.Sprintf("%%home%%\\agent\\build\\TEST-go-remote-%s.%s", prefix, filepath.Base(pkg.Name)) - resultPkg.Output, err = sshClient.GetFileContents(ctx, outputPath+".out", WithContentFetchCommand("type")) + resultPkg.Output, err = sshClient.GetFileContents(ctx, outputPath+".out", ssh.WithContentFetchCommand("type")) if err != nil { - return OSRunnerPackageResult{}, fmt.Errorf("failed to fetched test output at %s.out", outputPath) + return common.OSRunnerPackageResult{}, fmt.Errorf("failed to fetched test output at %s.out", outputPath) } - resultPkg.JSONOutput, err = sshClient.GetFileContents(ctx, outputPath+".out.json", WithContentFetchCommand("type")) + resultPkg.JSONOutput, err = sshClient.GetFileContents(ctx, outputPath+".out.json", ssh.WithContentFetchCommand("type")) if err != nil { - return OSRunnerPackageResult{}, fmt.Errorf("failed to fetched test output at %s.out.json", outputPath) + return common.OSRunnerPackageResult{}, fmt.Errorf("failed to fetched test output at %s.out.json", outputPath) } - resultPkg.XMLOutput, err = sshClient.GetFileContents(ctx, outputPath+".xml", WithContentFetchCommand("type")) + resultPkg.XMLOutput, err = sshClient.GetFileContents(ctx, outputPath+".xml", ssh.WithContentFetchCommand("type")) if err != nil { - return OSRunnerPackageResult{}, fmt.Errorf("failed to fetched test output at %s.xml", outputPath) + return common.OSRunnerPackageResult{}, fmt.Errorf("failed to fetched test output at %s.xml", outputPath) } return resultPkg, nil } From 72e57b29c4544e245dc5e709ae08ec2e73f73435 Mon Sep 17 00:00:00 2001 From: Geoff Rowland <70981735+rowlandgeoff@users.noreply.github.com> Date: Mon, 30 Sep 2024 15:01:15 -0400 Subject: [PATCH 05/20] Handle BUILDKITE_MESSAGE length in pre-command.ps1 (#5610) --- .buildkite/hooks/pre-command.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildkite/hooks/pre-command.ps1 b/.buildkite/hooks/pre-command.ps1 index 44a595dd918..e3a6257b2f1 100644 --- a/.buildkite/hooks/pre-command.ps1 +++ b/.buildkite/hooks/pre-command.ps1 @@ -1,3 +1,6 @@ +# Shorten BUILDKITE_MESSAGE if needed to avoid filling the Windows env var buffer +$env:BUILDKITE_MESSAGE = $env:BUILDKITE_MESSAGE.Substring(0, [System.Math]::Min(2048, $env:BUILDKITE_MESSAGE.Length)) + # Install gcc TODO: Move to the VM image choco install mingw Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1 From 701f8b9491da46d4aa2e1a211c970cfba9caef55 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Mon, 30 Sep 2024 16:56:37 -0400 Subject: [PATCH 06/20] Add elastic-agent-service container to packaging (#5349) Adds the elastic-agent-service container with the py-connectors component enabled. --- .buildkite/integration.pipeline.yml | 2 + dev-tools/mage/checksums.go | 10 +- dev-tools/mage/dockervariants.go | 6 + dev-tools/mage/manifest/manifest.go | 60 ++++---- dev-tools/mage/manifest/manifest_test.go | 17 ++- dev-tools/packaging/files/linux/connectors.sh | 6 + dev-tools/packaging/packages.yml | 43 ++++++ .../docker/Dockerfile.elastic-agent.tmpl | 14 +- magefile.go | 56 +++++--- pkg/testing/kubernetes/supported.go | 4 + specs/connectors.spec.yml | 17 +++ .../kubernetes_agent_service_test.go | 129 ++++++++++++++++++ .../kubernetes_agent_standalone_test.go | 83 +++++++---- .../integration/testdata/connectors.agent.yml | 13 ++ 14 files changed, 376 insertions(+), 84 deletions(-) create mode 100755 dev-tools/packaging/files/linux/connectors.sh create mode 100644 specs/connectors.spec.yml create mode 100644 testing/integration/kubernetes_agent_service_test.go create mode 100644 testing/integration/testdata/connectors.agent.yml diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml index cccd2ca1908..a11992264e5 100644 --- a/.buildkite/integration.pipeline.yml +++ b/.buildkite/integration.pipeline.yml @@ -116,7 +116,9 @@ steps: - "build/diagnostics/*" agents: provider: "gcp" + machineType: "c2-standard-16" image: "family/core-ubuntu-2204" + diskSizeGb: 400 notify: - github_commit_status: context: "buildkite/elastic-agent-extended-testing - Kubernetes Integration tests" diff --git a/dev-tools/mage/checksums.go b/dev-tools/mage/checksums.go index 318974be8d7..100d138195c 100644 --- a/dev-tools/mage/checksums.go +++ b/dev-tools/mage/checksums.go @@ -108,9 +108,9 @@ func ChecksumsWithManifest(requiredPackage string, versionedFlatPath string, ver // Only care about packages that match the required package constraint (os/arch) if strings.Contains(pkgName, requiredPackage) { // Iterate over the external binaries that we care about for packaging agent - for binary := range manifest.ExpectedBinaries { + for _, spec := range manifest.ExpectedBinaries { // If the individual package doesn't match the expected prefix, then continue - if !strings.HasPrefix(pkgName, binary) { + if !strings.HasPrefix(pkgName, spec.BinaryName) { continue } @@ -215,14 +215,14 @@ func getComponentVersion(componentName string, requiredPackage string, component // Iterate over all the packages in the component project for pkgName := range componentProject.Packages { // Only care about the external binaries that we want to package - for binary, project := range manifest.ExpectedBinaries { + for _, spec := range manifest.ExpectedBinaries { // If the given component name doesn't match the external binary component, skip - if componentName != project.Name { + if componentName != spec.ProjectName { continue } // Split the package name on the binary name prefix plus a dash - firstSplit := strings.Split(pkgName, binary+"-") + firstSplit := strings.Split(pkgName, spec.BinaryName+"-") if len(firstSplit) < 2 { continue } diff --git a/dev-tools/mage/dockervariants.go b/dev-tools/mage/dockervariants.go index da8f777d059..40b49be181d 100644 --- a/dev-tools/mage/dockervariants.go +++ b/dev-tools/mage/dockervariants.go @@ -17,6 +17,7 @@ const ( complete = "complete" completeWolfi = "complete-wolfi" cloud = "cloud" + service = "service" ) // DockerVariant defines the docker variant to build. @@ -31,6 +32,7 @@ const ( WolfiComplete Complete Cloud + Service ) // String returns the name of the docker variant type. @@ -50,6 +52,8 @@ func (typ DockerVariant) String() string { return complete case Cloud: return cloud + case Service: + return service default: return invalid } @@ -77,6 +81,8 @@ func (typ *DockerVariant) UnmarshalText(text []byte) error { *typ = Complete case cloud: *typ = Cloud + case service: + *typ = Service default: return fmt.Errorf("unknown docker variant: %v", string(text)) } diff --git a/dev-tools/mage/manifest/manifest.go b/dev-tools/mage/manifest/manifest.go index aaa10f6dcba..09ff95e27e6 100644 --- a/dev-tools/mage/manifest/manifest.go +++ b/dev-tools/mage/manifest/manifest.go @@ -94,20 +94,23 @@ var PlatformPackages = map[string]string{ // ExpectedBinaries is a map of binaries agent needs to their project in the unified-release manager. // The project names are those used in the "projects" list in the unified release manifest. // See the sample manifests in the testdata directory. -var ExpectedBinaries = map[string]BinarySpec{ - "agentbeat": {Name: "beats", Platforms: AllPlatforms}, - "apm-server": {Name: "apm-server", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}}}, - "cloudbeat": {Name: "cloudbeat", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}}, - "endpoint-security": {Name: "endpoint-dev", Platforms: AllPlatforms}, - "fleet-server": {Name: "fleet-server", Platforms: AllPlatforms}, - "pf-elastic-collector": {Name: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}}, - "pf-elastic-symbolizer": {Name: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}}, - "pf-host-agent": {Name: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}}, +var ExpectedBinaries = []BinarySpec{ + {BinaryName: "agentbeat", ProjectName: "beats", Platforms: AllPlatforms}, + {BinaryName: "apm-server", ProjectName: "apm-server", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}}}, + {BinaryName: "cloudbeat", ProjectName: "cloudbeat", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}}, + {BinaryName: "connectors", ProjectName: "connectors", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PythonWheel: true}, + {BinaryName: "endpoint-security", ProjectName: "endpoint-dev", Platforms: AllPlatforms}, + {BinaryName: "fleet-server", ProjectName: "fleet-server", Platforms: AllPlatforms}, + {BinaryName: "pf-elastic-collector", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}}, + {BinaryName: "pf-elastic-symbolizer", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}}, + {BinaryName: "pf-host-agent", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}}, } type BinarySpec struct { - Name string - Platforms []Platform + BinaryName string + ProjectName string + Platforms []Platform + PythonWheel bool } func (proj BinarySpec) SupportsPlatform(platform string) bool { @@ -119,6 +122,13 @@ func (proj BinarySpec) SupportsPlatform(platform string) bool { return false } +func (proj BinarySpec) GetPackageName(version string, platform string) string { + if proj.PythonWheel { + return fmt.Sprintf("%s-%s.zip", proj.BinaryName, version) + } + return fmt.Sprintf("%s-%s-%s", proj.BinaryName, version, PlatformPackages[platform]) +} + type Platform struct { OS string Arch string @@ -187,27 +197,27 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string errGrp, downloadsCtx := errgroup.WithContext(ctx) // for project, pkgs := range expectedProjectPkgs() { - for binary, project := range ExpectedBinaries { + for _, spec := range ExpectedBinaries { for _, platform := range platforms { targetPath := filepath.Join(dropPath) err := os.MkdirAll(targetPath, 0755) if err != nil { return fmt.Errorf("failed to create directory %s", targetPath) } - log.Printf("+++ Prepare to download project [%s] for [%s]", project.Name, platform) + log.Printf("+++ Prepare to download [%s] project [%s] for [%s]", spec.BinaryName, spec.ProjectName, platform) - if !project.SupportsPlatform(platform) { - log.Printf(">>>>>>>>> Binary [%s] does not support platform [%s] ", binary, platform) + if !spec.SupportsPlatform(platform) { + log.Printf(">>>>>>>>> Binary [%s] does not support platform [%s] ", spec.BinaryName, platform) continue } - pkgURL, err := resolveManifestPackage(projects[project.Name], binary, PlatformPackages[platform], majorMinorPatchVersion) + pkgURL, err := resolveManifestPackage(projects[spec.ProjectName], spec, majorMinorPatchVersion, platform) if err != nil { return err } for _, p := range pkgURL { - log.Printf(">>>>>>>>> Downloading [%s] [%s] ", binary, p) + log.Printf(">>>>>>>>> Downloading [%s] [%s] ", spec.BinaryName, p) pkgFilename := path.Base(p) downloadTarget := filepath.Join(targetPath, pkgFilename) if _, err := os.Stat(downloadTarget); err != nil { @@ -228,35 +238,35 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string return nil } -func resolveManifestPackage(project Project, binary string, platformPkg string, version string) ([]string, error) { +func resolveManifestPackage(project Project, spec BinarySpec, version string, platform string) ([]string, error) { var val Package var ok bool // Try the normal/easy case first - packageName := fmt.Sprintf("%s-%s-%s", binary, version, platformPkg) + packageName := spec.GetPackageName(version, platform) val, ok = project.Packages[packageName] if !ok { // If we didn't find it, it may be an Independent Agent Release, where // the opted-in projects will have a patch version one higher than // the rest of the projects, so we need to seek that out if mg.Verbose() { - log.Printf(">>>>>>>>>>> Looking for package [%s] of type [%s]", binary, platformPkg) + log.Printf(">>>>>>>>>>> Looking for package [%s] of type [%s]", spec.BinaryName, PlatformPackages[platform]) } var foundIt bool for pkgName := range project.Packages { - if strings.HasPrefix(pkgName, binary) { - firstSplit := strings.Split(pkgName, binary+"-") + if strings.HasPrefix(pkgName, spec.BinaryName) { + firstSplit := strings.Split(pkgName, spec.BinaryName+"-") if len(firstSplit) < 2 { continue } secondHalf := firstSplit[1] // Make sure we're finding one w/ the same required package type - if strings.Contains(secondHalf, platformPkg) { + if strings.Contains(secondHalf, PlatformPackages[platform]) { // Split again after the version with the required package string - secondSplit := strings.Split(secondHalf, "-"+platformPkg) + secondSplit := strings.Split(secondHalf, "-"+PlatformPackages[platform]) if len(secondSplit) < 2 { continue } @@ -268,7 +278,7 @@ func resolveManifestPackage(project Project, binary string, platformPkg string, } // Create a project/package key with the package, derived version, and required package - foundPkgKey := fmt.Sprintf("%s-%s-%s", binary, pkgVersion, platformPkg) + foundPkgKey := fmt.Sprintf("%s-%s-%s", spec.BinaryName, pkgVersion, PlatformPackages[platform]) if mg.Verbose() { log.Printf(">>>>>>>>>>> Looking for project package key: [%s]", foundPkgKey) } diff --git a/dev-tools/mage/manifest/manifest_test.go b/dev-tools/mage/manifest/manifest_test.go index 11f6e34d5f1..b975149aa27 100644 --- a/dev-tools/mage/manifest/manifest_test.go +++ b/dev-tools/mage/manifest/manifest_test.go @@ -135,15 +135,15 @@ func TestResolveManifestPackage(t *testing.T) { projects := manifestJson.Projects // Verify the component name is in the list of expected packages. - project, ok := ExpectedBinaries[tc.binary] + spec, ok := findBinarySpec(tc.binary) assert.True(t, ok) - if !project.SupportsPlatform(tc.platform) { - t.Logf("Project %s does not support platform %s", project.Name, tc.platform) + if !spec.SupportsPlatform(tc.platform) { + t.Logf("Project %s does not support platform %s", spec.ProjectName, tc.platform) return } - urlList, err := resolveManifestPackage(projects[tc.projectName], tc.binary, PlatformPackages[tc.platform], manifestJson.Version) + urlList, err := resolveManifestPackage(projects[tc.projectName], spec, manifestJson.Version, tc.platform) require.NoError(t, err) assert.Len(t, urlList, 3) @@ -153,3 +153,12 @@ func TestResolveManifestPackage(t *testing.T) { }) } } + +func findBinarySpec(name string) (BinarySpec, bool) { + for _, spec := range ExpectedBinaries { + if spec.BinaryName == name { + return spec, true + } + } + return BinarySpec{}, false +} diff --git a/dev-tools/packaging/files/linux/connectors.sh b/dev-tools/packaging/files/linux/connectors.sh new file mode 100755 index 00000000000..81f95e24372 --- /dev/null +++ b/dev-tools/packaging/files/linux/connectors.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +PY_AGENT_CLIENT_PATH=/usr/share/connectors +PYTHON_PATH=$PY_AGENT_CLIENT_PATH/.venv/bin/python +COMPONENT_PATH=$PY_AGENT_CLIENT_PATH/connectors/agent/cli.py +$PYTHON_PATH $COMPONENT_PATH diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 4a38a7657d1..6528ab36f91 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -291,6 +291,20 @@ shared: source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/agentbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' mode: 0755 + # service build is based on previous cloud variant + - &agent_docker_service_spec + docker_variant: 'service' + files: + 'data/service/connectors-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}.zip': + source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/connectors-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}.zip' + mode: 0755 + 'data/{{.BeatName}}-{{ commit_short }}/components/connectors': + source: '{{ elastic_beats_dir }}/dev-tools/packaging/files/linux/connectors.sh' + mode: 0755 + 'data/{{.BeatName}}-{{ commit_short }}/components/connectors.spec.yml': + source: '{{ elastic_beats_dir }}/specs/connectors.spec.yml' + mode: 0644 + # includes nodejs with @elastic/synthetics - &agent_docker_complete_spec <<: *agent_docker_spec @@ -1025,6 +1039,35 @@ specs: files: '{{.BeatName}}{{.BinaryExt}}': source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + #### Service specific docker images #### + - os: linux + arch: amd64 + types: [ docker ] + spec: + <<: *agent_docker_spec + # The service image is always based on Wolfi + <<: *docker_wolfi_spec + <<: *docker_builder_spec + <<: *agent_docker_cloud_spec + <<: *agent_docker_service_spec + <<: *elastic_license_for_binaries + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + - os: linux + arch: arm64 + types: [ docker ] + spec: + <<: *agent_docker_spec + # The service image is always based on Wolfi + <<: *docker_wolfi_arm_spec + <<: *docker_builder_arm_spec + <<: *agent_docker_cloud_spec + <<: *agent_docker_service_spec + <<: *elastic_license_for_binaries + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} - os: linux arch: amd64 types: [docker] diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl index ccede203a05..50894676a7f 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl @@ -43,7 +43,7 @@ RUN true && \ chmod 0775 {{ $beatHome}}/{{ $modulesd }} && \ {{- end }} -{{- if eq .Variant "cloud" }} +{{- if or (eq .Variant "cloud") (eq .Variant "service") }} mkdir -p /opt/agentbeat /opt/filebeat /opt/metricbeat && \ cp -f {{ $beatHome }}/data/cloud_downloads/filebeat.sh /opt/filebeat/filebeat && \ chmod +x /opt/filebeat/filebeat && \ @@ -170,13 +170,21 @@ RUN mkdir /licenses COPY --from=home {{ $beatHome }}/LICENSE.txt /licenses COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses -{{- if eq .Variant "cloud" }} +{{- if or (eq .Variant "cloud") (eq .Variant "service") }} COPY --from=home /opt /opt # Generate folder for a stub command that will be overwritten at runtime RUN mkdir /app && \ chown {{ .user }}:{{ .user }} /app {{- end }} +{{- if eq .Variant "service" }} +RUN apk add --no-cache git make python-3.11 py3.11-pip && \ + unzip {{ $beatHome }}/data/service/connectors-*.zip -d {{ $beatHome }}/data/service && \ + mv {{ $beatHome }}/data/service/elasticsearch_connectors-* /usr/share/connectors && \ + PYTHON=python3.11 make -C /usr/share/connectors clean install install-agent && \ + chmod 0755 {{ $beatHome }}/data/elastic-agent-*/components/connectors +{{- end }} + {{- if (and (eq .Variant "complete") (contains .from "ubuntu")) }} USER root ENV NODE_PATH={{ $beatHome }}/.node @@ -284,7 +292,7 @@ ENV LIBBEAT_MONITORING_CGROUPS_HIERARCHY_OVERRIDE=/ WORKDIR {{ $beatHome }} -{{- if eq .Variant "cloud" }} +{{- if or (eq .Variant "cloud") (eq .Variant "service") }} ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["/app/apm.sh"] # Generate a stub command that will be overwritten at runtime diff --git a/magefile.go b/magefile.go index fe5c55489eb..06ef0c1a2c3 100644 --- a/magefile.go +++ b/magefile.go @@ -533,11 +533,6 @@ func DownloadManifest(ctx context.Context) error { return errAtLeastOnePlatform } - var requiredPackages []string - for _, p := range platforms { - requiredPackages = append(requiredPackages, manifest.PlatformPackages[p]) - } - if e := manifest.DownloadComponents(ctx, devtools.ManifestURL, platforms, dropPath); e != nil { return fmt.Errorf("failed to download the manifest file, %w", e) } @@ -595,16 +590,9 @@ func FixDRADockerArtifacts() error { return nil } -func getPackageName(beat, version, pkg string) (string, string) { - if hasSnapshotEnv() { - version += "-SNAPSHOT" - } - return version, fmt.Sprintf("%s-%s-%s", beat, version, pkg) -} - func requiredPackagesPresent(basePath, beat, version string, requiredPackages []string) bool { for _, pkg := range requiredPackages { - _, packageName := getPackageName(beat, version, pkg) + packageName := fmt.Sprintf("%s-%s-%s", beat, version, pkg) path := filepath.Join(basePath, "build", "distributions", packageName) if _, err := os.Stat(path); err != nil { @@ -1023,6 +1011,10 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi } archivePath = movePackagesToArchive(dropPath, requiredPackages) + if hasSnapshotEnv() { + packageVersion = fmt.Sprintf("%s-SNAPSHOT", packageVersion) + } + os.Setenv(agentDropPath, dropPath) if devtools.ExternalBuild == true { @@ -1038,17 +1030,16 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi errGroup, ctx := errgroup.WithContext(context.Background()) completedDownloads := &atomic.Int32{} - for binary, project := range manifest.ExpectedBinaries { + for _, spec := range manifest.ExpectedBinaries { for _, platform := range platforms { - if !project.SupportsPlatform(platform) { - fmt.Printf("--- Binary %s does not support %s, download skipped\n", binary, platform) + if !spec.SupportsPlatform(platform) { + fmt.Printf("--- Binary %s does not support %s, download skipped\n", spec.BinaryName, platform) continue } - reqPackage := manifest.PlatformPackages[platform] - targetPath := filepath.Join(archivePath, reqPackage) + targetPath := filepath.Join(archivePath, manifest.PlatformPackages[platform]) os.MkdirAll(targetPath, 0755) - newVersion, packageName := getPackageName(binary, packageVersion, reqPackage) - errGroup.Go(downloadBinary(ctx, project.Name, packageName, binary, platform, newVersion, targetPath, completedDownloads)) + packageName := spec.GetPackageName(packageVersion, platform) + errGroup.Go(downloadBinary(ctx, spec.ProjectName, packageName, spec.BinaryName, platform, packageVersion, targetPath, completedDownloads)) } } @@ -1126,6 +1117,27 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi return archivePath, dropPath } +func removePythonWheels(matches []string, version string) []string { + if hasSnapshotEnv() { + version = fmt.Sprintf("%s-SNAPSHOT", version) + } + + var wheels []string + for _, spec := range manifest.ExpectedBinaries { + if spec.PythonWheel { + wheels = append(wheels, spec.GetPackageName(version, "")) + } + } + + cleaned := make([]string, 0, len(matches)) + for _, path := range matches { + if !slices.Contains(wheels, filepath.Base(path)) { + cleaned = append(cleaned, path) + } + } + return cleaned +} + // flattenDependencies will extract all the required packages collected in archivePath and dropPath in flatPath and // regenerate checksums func flattenDependencies(requiredPackages []string, packageVersion, archivePath, dropPath, flatPath string, manifestResponse *manifest.Build) { @@ -1149,6 +1161,10 @@ func flattenDependencies(requiredPackages []string, packageVersion, archivePath, } matches = append(matches, zipMatches...) + // never flatten any python wheels, the packages.yml and docker should handle + // those specifically so that the python wheels are installed into the container + matches = removePythonWheels(matches, packageVersion) + if mg.Verbose() { log.Printf("--- Extracting into the flat dir: %v", matches) } diff --git a/pkg/testing/kubernetes/supported.go b/pkg/testing/kubernetes/supported.go index 26f1bef3e6d..e7db5ba71c3 100644 --- a/pkg/testing/kubernetes/supported.go +++ b/pkg/testing/kubernetes/supported.go @@ -72,6 +72,10 @@ var variants = []struct { Name: "cloud", Image: "docker.elastic.co/beats-ci/elastic-agent-cloud", }, + { + Name: "service", + Image: "docker.elastic.co/beats-ci/elastic-agent-service", + }, } // GetSupported returns the list of supported OS types for Kubernetes. diff --git a/specs/connectors.spec.yml b/specs/connectors.spec.yml new file mode 100644 index 00000000000..0fb61f0bba2 --- /dev/null +++ b/specs/connectors.spec.yml @@ -0,0 +1,17 @@ +version: 2 +inputs: + - name: connectors-py + description: "Connectors Python" + platforms: + - linux/amd64 + - linux/arm64 + - container/amd64 + - container/arm64 + outputs: + - elasticsearch + command: + restart_monitoring_period: 5s + maximum_restarts_per_period: 1 + timeouts: + restart: 1s + args: [] diff --git a/testing/integration/kubernetes_agent_service_test.go b/testing/integration/kubernetes_agent_service_test.go new file mode 100644 index 00000000000..4a5ebdda2ad --- /dev/null +++ b/testing/integration/kubernetes_agent_service_test.go @@ -0,0 +1,129 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +//go:build integration + +package integration + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + + "github.com/elastic/elastic-agent/pkg/testing/define" +) + +func TestKubernetesAgentService(t *testing.T) { + info := define.Require(t, define.Requirements{ + Stack: &define.Stack{}, + Local: false, + Sudo: false, + OS: []define.OS{ + // only test the service container + {Type: define.Kubernetes, DockerVariant: "service"}, + }, + Group: define.Kubernetes, + }) + + agentImage := os.Getenv("AGENT_IMAGE") + require.NotEmpty(t, agentImage, "AGENT_IMAGE must be set") + + client, err := info.KubeClient() + require.NoError(t, err) + require.NotNil(t, client) + + testLogsBasePath := os.Getenv("K8S_TESTS_POD_LOGS_BASE") + require.NotEmpty(t, testLogsBasePath, "K8S_TESTS_POD_LOGS_BASE must be set") + + err = os.MkdirAll(filepath.Join(testLogsBasePath, t.Name()), 0755) + require.NoError(t, err, "failed to create test logs directory") + + namespace := info.Namespace + + esHost := os.Getenv("ELASTICSEARCH_HOST") + require.NotEmpty(t, esHost, "ELASTICSEARCH_HOST must be set") + + esAPIKey, err := generateESAPIKey(info.ESClient, namespace) + require.NoError(t, err, "failed to generate ES API key") + require.NotEmpty(t, esAPIKey, "failed to generate ES API key") + + renderedManifest, err := renderKustomize(agentK8SKustomize) + require.NoError(t, err, "failed to render kustomize") + + hasher := sha256.New() + hasher.Write([]byte(t.Name())) + testNamespace := strings.ToLower(base64.URLEncoding.EncodeToString(hasher.Sum(nil))) + testNamespace = noSpecialCharsRegexp.ReplaceAllString(testNamespace, "") + + k8sObjects, err := yamlToK8SObjects(bufio.NewReader(bytes.NewReader(renderedManifest))) + require.NoError(t, err, "failed to convert yaml to k8s objects") + + adjustK8SAgentManifests(k8sObjects, testNamespace, "elastic-agent-standalone", + func(container *corev1.Container) { + // set agent image + container.Image = agentImage + // set ImagePullPolicy to "Never" to avoid pulling the image + // as the image is already loaded by the kubernetes provisioner + container.ImagePullPolicy = "Never" + + // set Elasticsearch host and API key + for idx, env := range container.Env { + if env.Name == "ES_HOST" { + container.Env[idx].Value = esHost + container.Env[idx].ValueFrom = nil + } + if env.Name == "API_KEY" { + container.Env[idx].Value = esAPIKey + container.Env[idx].ValueFrom = nil + } + } + + // has a unique entrypoint and command because its ran in the cloud + // adjust the spec to run it correctly + container.Command = []string{"elastic-agent"} + container.Args = []string{"-c", "/etc/elastic-agent/agent.yml", "-e"} + }, + func(pod *corev1.PodSpec) { + for volumeIdx, volume := range pod.Volumes { + // need to update the volume path of the state directory + // to match the test namespace + if volume.Name == "elastic-agent-state" { + hostPathType := corev1.HostPathDirectoryOrCreate + pod.Volumes[volumeIdx].VolumeSource.HostPath = &corev1.HostPathVolumeSource{ + Type: &hostPathType, + Path: fmt.Sprintf("/var/lib/elastic-agent-standalone/%s/state", testNamespace), + } + } + } + }) + + // update the configmap to only run the connectors input + serviceAgentYAML, err := os.ReadFile(filepath.Join("testdata", "connectors.agent.yml")) + require.NoError(t, err) + for _, obj := range k8sObjects { + switch objWithType := obj.(type) { + case *corev1.ConfigMap: + _, ok := objWithType.Data["agent.yml"] + if ok { + objWithType.Data["agent.yml"] = string(serviceAgentYAML) + } + } + } + + ctx := context.Background() + + deployK8SAgent(t, ctx, client, k8sObjects, testNamespace, false, testLogsBasePath, map[string]bool{ + "connectors-py": true, + }) +} diff --git a/testing/integration/kubernetes_agent_standalone_test.go b/testing/integration/kubernetes_agent_standalone_test.go index 417a36c30c3..ddcbb559cca 100644 --- a/testing/integration/kubernetes_agent_standalone_test.go +++ b/testing/integration/kubernetes_agent_standalone_test.go @@ -15,6 +15,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os" "path/filepath" @@ -25,8 +26,6 @@ import ( "github.com/stretchr/testify/require" - "github.com/elastic/elastic-agent/pkg/testing/define" - "github.com/elastic/elastic-agent/pkg/testing/tools/fleettools" "github.com/elastic/go-elasticsearch/v8" appsv1 "k8s.io/api/apps/v1" @@ -47,6 +46,11 @@ import ( "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/cli" + + aclient "github.com/elastic/elastic-agent/pkg/control/v2/client" + atesting "github.com/elastic/elastic-agent/pkg/testing" + "github.com/elastic/elastic-agent/pkg/testing/define" + "github.com/elastic/elastic-agent/pkg/testing/tools/fleettools" ) const ( @@ -657,46 +661,71 @@ func deployK8SAgent(t *testing.T, ctx context.Context, client klient.Client, obj require.NotEmpty(t, agentPodName, "agent pod name is empty") - command := []string{"elastic-agent", "status"} + command := []string{"elastic-agent", "status", "--output=json"} + var status atesting.AgentStatusOutput var stdout, stderr bytes.Buffer var agentHealthyErr error // we will wait maximum 120 seconds for the agent to report healthy for i := 0; i < 120; i++ { + status = atesting.AgentStatusOutput{} // clear status output stdout.Reset() stderr.Reset() agentHealthyErr = client.Resources().ExecInPod(ctx, namespace, agentPodName, "elastic-agent-standalone", command, &stdout, &stderr) if agentHealthyErr == nil { - break + if uerr := json.Unmarshal(stdout.Bytes(), &status); uerr == nil { + if status.State == int(aclient.Healthy) { + // agent is healthy innner tests should now pass + if runInnerK8STests { + err := client.Resources().ExecInPod(ctx, namespace, agentPodName, "elastic-agent-standalone", + []string{"/usr/share/elastic-agent/k8s-inner-tests", "-test.v"}, &stdout, &stderr) + t.Log(stdout.String()) + if err != nil { + t.Log(stderr.String()) + } + require.NoError(t, err, "error at k8s inner tests execution") + } + + // validate that the components defined are also healthy if they should exist + componentsCorrect := true + for component, shouldBePresent := range componentPresence { + compState, ok := getComponentState(status, component) + if shouldBePresent { + if !ok { + // doesn't exist + componentsCorrect = false + } else if compState != int(aclient.Healthy) { + // not healthy + componentsCorrect = false + } + } else if ok { + // should not be present + // break instantly and fail (as it existing should never happen) + break + } + } + if componentsCorrect { + // agent health and components are correct + return + } + } + } } time.Sleep(time.Second * 1) } - statusString := stdout.String() - if agentHealthyErr != nil { - t.Errorf("elastic-agent never reported healthy: %v", agentHealthyErr) - t.Logf("stdout: %s\n", statusString) - t.Logf("stderr: %s\n", stderr.String()) - t.FailNow() - return - } - - stdout.Reset() - stderr.Reset() - - for component, shouldBePresent := range componentPresence { - isPresent := strings.Contains(statusString, component) - require.Equal(t, shouldBePresent, isPresent) - } + t.Errorf("elastic-agent never reported healthy: %+v", status) + t.Logf("stdout: %s\n", stdout.String()) + t.Logf("stderr: %s\n", stderr.String()) + t.FailNow() +} - if runInnerK8STests { - err := client.Resources().ExecInPod(ctx, namespace, agentPodName, "elastic-agent-standalone", - []string{"/usr/share/elastic-agent/k8s-inner-tests", "-test.v"}, &stdout, &stderr) - t.Log(stdout.String()) - if err != nil { - t.Log(stderr.String()) +func getComponentState(status atesting.AgentStatusOutput, componentName string) (int, bool) { + for _, comp := range status.Components { + if comp.Name == componentName { + return comp.State, true } - require.NoError(t, err, "error at k8s inner tests execution") } + return -1, false } // dumpLogs dumps the logs of all pods in the given namespace to the given target directory diff --git a/testing/integration/testdata/connectors.agent.yml b/testing/integration/testdata/connectors.agent.yml new file mode 100644 index 00000000000..5c3466ae8ae --- /dev/null +++ b/testing/integration/testdata/connectors.agent.yml @@ -0,0 +1,13 @@ +outputs: + default: + type: elasticsearch + hosts: + - >- + ${ES_HOST} + api_key: ${API_KEY} +agent: + monitoring: + enabled: false +inputs: + - id: connectors + type: connectors-py From ed98d51f11f1a0b33a9b2d67d6e61568f6aec439 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 30 Sep 2024 15:16:52 -0700 Subject: [PATCH 07/20] [CI] Soft fail merge coverage reports step (#5586) * Skip merge coverage reports step * Add debugging echo * Soft fail instead of skip * Soft fail instead of skip --- .buildkite/pipeline.yml | 2 ++ .buildkite/scripts/steps/merge.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 174096fb28a..60cba8fc57a 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -176,6 +176,7 @@ steps: - unit-tests - extended-windows allow_dependency_failure: true + soft_fail: true # Until https://github.com/elastic/ingest-dev/issues/4042 is resolved - group: "K8s tests" key: "k8s-tests" @@ -212,6 +213,7 @@ steps: retry: manual: allowed: true + soft_fail: true # Until https://github.com/elastic/ingest-dev/issues/4042 is resolved # Triggers a dynamic step: Sync K8s # Runs only on main and if k8s files are changed diff --git a/.buildkite/scripts/steps/merge.sh b/.buildkite/scripts/steps/merge.sh index 49b173a836b..bc7960419f9 100755 --- a/.buildkite/scripts/steps/merge.sh +++ b/.buildkite/scripts/steps/merge.sh @@ -4,6 +4,7 @@ # Usage: merge.sh ... Where is the id of the step that contains the coverage artifact.# set -euo pipefail +set -x # for debugging COV_ARTIFACT="coverage.out" MERGED_COV_FILE="build/TEST-go-unit.cov" From 0ebadad3b014c36f25d149cf3cb115de9e0a260b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Sep 2024 17:13:12 -0700 Subject: [PATCH 08/20] build(deps): bump github.com/elastic/elastic-agent-libs (#5559) Bumps [github.com/elastic/elastic-agent-libs](https://github.com/elastic/elastic-agent-libs) from 0.10.1 to 0.11.0. - [Release notes](https://github.com/elastic/elastic-agent-libs/releases) - [Commits](https://github.com/elastic/elastic-agent-libs/compare/v0.10.1...v0.11.0) --- updated-dependencies: - dependency-name: github.com/elastic/elastic-agent-libs dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 9fdb6913081..f2a510280ab 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1264,11 +1264,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.10.1 +Version: v0.11.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.10.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.11.0/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index 6f1160234c2..4ab8d20563d 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/dolmen-go/contextio v0.0.0-20200217195037-68fc5150bcd5 github.com/elastic/elastic-agent-autodiscover v0.8.2 github.com/elastic/elastic-agent-client/v7 v7.16.0 - github.com/elastic/elastic-agent-libs v0.10.1 + github.com/elastic/elastic-agent-libs v0.11.0 github.com/elastic/elastic-agent-system-metrics v0.11.2 github.com/elastic/elastic-transport-go/v8 v8.6.0 github.com/elastic/go-elasticsearch/v8 v8.15.0 diff --git a/go.sum b/go.sum index 4b9c2e43c58..6e0cf045b9e 100644 --- a/go.sum +++ b/go.sum @@ -241,8 +241,8 @@ github.com/elastic/elastic-agent-autodiscover v0.8.2 h1:Fs2FhR33AMBPfm5/jz4drVza github.com/elastic/elastic-agent-autodiscover v0.8.2/go.mod h1:VZnU53EVaFTxR8Xf6YsLN8FHD5DKQzHSPlKax9/4w+o= github.com/elastic/elastic-agent-client/v7 v7.16.0 h1:yKGq2+CxAuW8Kh0EoNl202tqAyQKfBcPRawVKs2Jve0= github.com/elastic/elastic-agent-client/v7 v7.16.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI= -github.com/elastic/elastic-agent-libs v0.10.1 h1:4MMqNQVPoQGqPMM9bEO1Q6d48QPhW+rIdV/0zt82ta4= -github.com/elastic/elastic-agent-libs v0.10.1/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M= +github.com/elastic/elastic-agent-libs v0.11.0 h1:m9rnNE3BkBF2XJoqubqEbu/kbtKEBZ7pHCjDlxfVRH0= +github.com/elastic/elastic-agent-libs v0.11.0/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M= github.com/elastic/elastic-agent-system-metrics v0.11.2 h1:UjSBcY6U3H3venB5zAPEFNjAb4Bb38EiVqaA4zALEnM= github.com/elastic/elastic-agent-system-metrics v0.11.2/go.mod h1:saqLKe9fuyuAo6IADAnnuy1kaBI7VNlxfwMo8KzSRyQ= github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA= From bbee90a291f150edbb0e05cf3c93fc6620871fb4 Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Tue, 1 Oct 2024 04:33:37 +0200 Subject: [PATCH 09/20] refactor: remove unused replace directives (#5590) goja is not used in elastic agent see go mod graph | grep groja --- go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.mod b/go.mod index 4ab8d20563d..65afb808bd0 100644 --- a/go.mod +++ b/go.mod @@ -453,8 +453,6 @@ require ( replace ( github.com/Shopify/sarama => github.com/elastic/sarama v1.19.1-0.20220310193331-ebc2b0d8eef3 - github.com/dop251/goja => github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20 - github.com/dop251/goja_nodejs => github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 // openshift removed all tags from their repo, use the pseudoversion from the release-3.9 branch HEAD // See https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/12d41f40b0d408b0167633d8095160d3343d46ac/go.mod#L38 github.com/openshift/api v3.9.0+incompatible => github.com/openshift/api v0.0.0-20180801171038-322a19404e37 From d0c7beab00a3a863154b01953993ba5f41ffaad9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 03:01:46 +0000 Subject: [PATCH 10/20] build(deps): bump github.com/elastic/go-sysinfo from 1.14.1 to 1.14.2 (#5603) * build(deps): bump github.com/elastic/go-sysinfo from 1.14.1 to 1.14.2 Bumps [github.com/elastic/go-sysinfo](https://github.com/elastic/go-sysinfo) from 1.14.1 to 1.14.2. - [Release notes](https://github.com/elastic/go-sysinfo/releases) - [Commits](https://github.com/elastic/go-sysinfo/compare/v1.14.1...v1.14.2) --- updated-dependencies: - dependency-name: github.com/elastic/go-sysinfo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update NOTICE.txt --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- NOTICE.txt | 8 ++++---- go.mod | 4 ++-- go.sum | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index f2a510280ab..876c417755d 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -2320,11 +2320,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-licenser@v0. -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-sysinfo -Version: v1.14.1 +Version: v1.14.2 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.14.1/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.14.2/LICENSE.txt: Apache License @@ -15131,11 +15131,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/sys -Version: v0.24.0 +Version: v0.25.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.24.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.25.0/LICENSE: Copyright 2009 The Go Authors. diff --git a/go.mod b/go.mod index 65afb808bd0..b683324f1b9 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/elastic/elastic-transport-go/v8 v8.6.0 github.com/elastic/go-elasticsearch/v8 v8.15.0 github.com/elastic/go-licenser v0.4.2 - github.com/elastic/go-sysinfo v1.14.1 + github.com/elastic/go-sysinfo v1.14.2 github.com/elastic/go-ucfg v0.8.8 github.com/elastic/mock-es v0.0.0-20240712014503-e5b47ece0015 github.com/elastic/opentelemetry-collector-components/processor/elasticinframetricsprocessor v0.11.0 @@ -64,7 +64,7 @@ require ( golang.org/x/crypto v0.26.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/sync v0.8.0 - golang.org/x/sys v0.24.0 + golang.org/x/sys v0.25.0 golang.org/x/term v0.23.0 golang.org/x/text v0.17.0 golang.org/x/time v0.5.0 diff --git a/go.sum b/go.sum index 6e0cf045b9e..d2943b24072 100644 --- a/go.sum +++ b/go.sum @@ -259,8 +259,8 @@ github.com/elastic/go-licenser v0.4.2 h1:bPbGm8bUd8rxzSswFOqvQh1dAkKGkgAmrPxbUi+ github.com/elastic/go-licenser v0.4.2/go.mod h1:W8eH6FaZDR8fQGm+7FnVa7MxI1b/6dAqxz+zPB8nm5c= github.com/elastic/go-structform v0.0.12 h1:HXpzlAKyej8T7LobqKDThUw7BMhwV6Db24VwxNtgxCs= github.com/elastic/go-structform v0.0.12/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4= -github.com/elastic/go-sysinfo v1.14.1 h1:BpY/Utfz75oKSpsQnbAJmmlnT3gBV9WFsopBEYgjhZY= -github.com/elastic/go-sysinfo v1.14.1/go.mod h1:FKUXnZWhnYI0ueO7jhsGV3uQJ5hiz8OqM5b3oGyaRr8= +github.com/elastic/go-sysinfo v1.14.2 h1:DeIy+pVfdRsd08Nx2Xjh+dUS+jrEEI7LGc29U/BKVWo= +github.com/elastic/go-sysinfo v1.14.2/go.mod h1:jPSuTgXG+dhhh0GKIyI2Cso+w5lPJ5PvVqKlL8LV/Hk= github.com/elastic/go-ucfg v0.8.8 h1:54KIF/2zFKfl0MzsSOCGOsZ3O2bnjFQJ0nDJcLhviyk= github.com/elastic/go-ucfg v0.8.8/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA= github.com/elastic/go-windows v1.0.2 h1:yoLLsAsV5cfg9FLhZ9EXZ2n2sQFKeDYrHenkcivY4vI= @@ -1536,8 +1536,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= -golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= From 218c984c7cf2612270baa170c82a2671f9fd31e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20=C5=9Awi=C4=85tek?= Date: Tue, 1 Oct 2024 10:43:33 +0200 Subject: [PATCH 11/20] Allow setting ES api key via env variable in container mode (#5536) Also remove defaults for username and password set this way. --- _meta/config/elastic-agent.docker.yml.tmpl | 5 +-- ...6144746-container-default-credentials.yaml | 32 +++++++++++++++++++ .../1726145045-container-api-key.yaml | 32 +++++++++++++++++++ elastic-agent.docker.yml | 5 +-- 4 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 changelog/fragments/1726144746-container-default-credentials.yaml create mode 100644 changelog/fragments/1726145045-container-api-key.yaml diff --git a/_meta/config/elastic-agent.docker.yml.tmpl b/_meta/config/elastic-agent.docker.yml.tmpl index 10f9c4147c8..d0c421c75b7 100644 --- a/_meta/config/elastic-agent.docker.yml.tmpl +++ b/_meta/config/elastic-agent.docker.yml.tmpl @@ -5,8 +5,9 @@ outputs: default: type: elasticsearch hosts: '${ELASTICSEARCH_HOSTS:http://elasticsearch:9200}' - username: '${ELASTICSEARCH_USERNAME:elastic}' - password: '${ELASTICSEARCH_PASSWORD:changeme}' + username: '${ELASTICSEARCH_USERNAME:}' + password: '${ELASTICSEARCH_PASSWORD:}' + api_key: '${ELASTICSEARCH_API_KEY:}' preset: balanced inputs: diff --git a/changelog/fragments/1726144746-container-default-credentials.yaml b/changelog/fragments/1726144746-container-default-credentials.yaml new file mode 100644 index 00000000000..560262122cf --- /dev/null +++ b/changelog/fragments/1726144746-container-default-credentials.yaml @@ -0,0 +1,32 @@ +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: breaking-change + +# Change summary; a 80ish characters long description of the change. +summary: Remove default credentials when running Elastic Agent in container mode + +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment. +description: ELASTICSEARCH_USERNAME and ELASTICSEARCH_PASSWORD now need to be explicitly set when running the agent in container mode + +# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. +component: elastic-agent + +# PR URL; optional; the PR number that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +#pr: https://github.com/owner/repo/1234 + +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +#issue: https://github.com/owner/repo/1234 diff --git a/changelog/fragments/1726145045-container-api-key.yaml b/changelog/fragments/1726145045-container-api-key.yaml new file mode 100644 index 00000000000..72cf4b5fc1c --- /dev/null +++ b/changelog/fragments/1726145045-container-api-key.yaml @@ -0,0 +1,32 @@ +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: enhancement + +# Change summary; a 80ish characters long description of the change. +summary: Support ELASTICSEARCH_API_KEY environment variable when running in container mode + +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment. +#description: + +# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. +component: elastic-agent + +# PR URL; optional; the PR number that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +#pr: https://github.com/owner/repo/1234 + +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +#issue: https://github.com/owner/repo/1234 diff --git a/elastic-agent.docker.yml b/elastic-agent.docker.yml index a0a6d5da8f6..d9089358600 100644 --- a/elastic-agent.docker.yml +++ b/elastic-agent.docker.yml @@ -5,8 +5,9 @@ outputs: default: type: elasticsearch hosts: '${ELASTICSEARCH_HOSTS:http://elasticsearch:9200}' - username: '${ELASTICSEARCH_USERNAME:elastic}' - password: '${ELASTICSEARCH_PASSWORD:changeme}' + username: '${ELASTICSEARCH_USERNAME:}' + password: '${ELASTICSEARCH_PASSWORD:}' + api_key: '${ELASTICSEARCH_API_KEY:}' preset: balanced inputs: From a9d54fb0c48005bf6055c6af7e855e3575fd8abe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20=C5=9Awi=C4=85tek?= Date: Tue, 1 Oct 2024 11:04:00 +0200 Subject: [PATCH 12/20] Fix linter errors (#5442) * Remove unused symbols * Fix bodycloser linter errors * Fix errcheck linter errors * Fix staticcheck linter errors * Use default configuration for nakedret linter * Fix various linter errors --- .golangci.yml | 4 --- dev-tools/cmd/buildfleetcfg/buildfleetcfg.go | 13 ++++++-- dev-tools/licenses/license_generate.go | 12 ++++++-- dev-tools/mage/check.go | 4 +-- dev-tools/mage/gotool/go.go | 4 --- dev-tools/mage/kubernetes/kubectl.go | 20 ------------- dev-tools/mage/target/common/package.go | 6 +++- .../monitoring/processes_cloud_test.go | 2 ++ .../artifact/download/headers_rtt_test.go | 2 +- internal/pkg/agent/errors/error_test.go | 12 +++++--- .../pkg/fleetapi/uploader/uploader_test.go | 12 ++++++++ pkg/component/fake/component/comp/dialer.go | 26 ---------------- .../fake/component/comp/dialer_windows.go | 26 ---------------- pkg/component/runtime/service_command.go | 30 +++++++++---------- pkg/control/v1/client/dial.go | 2 +- pkg/control/v1/client/dial_windows.go | 2 +- pkg/control/v2/client/dial.go | 2 +- pkg/control/v2/client/dial_windows.go | 2 +- testing/upgradetest/upgrader.go | 2 +- 19 files changed, 68 insertions(+), 115 deletions(-) delete mode 100644 pkg/component/fake/component/comp/dialer.go delete mode 100644 pkg/component/fake/component/comp/dialer_windows.go diff --git a/.golangci.yml b/.golangci.yml index 399b5f7e496..4c0c3bf0aff 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -124,10 +124,6 @@ linters-settings: # Select the Go version to target. The default is '1.13'. go: "1.22.7" - nakedret: - # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 - max-func-lines: 0 - nolintlint: # Enable to ensure that nolint directives are all used. Default is true. allow-unused: false diff --git a/dev-tools/cmd/buildfleetcfg/buildfleetcfg.go b/dev-tools/cmd/buildfleetcfg/buildfleetcfg.go index 9c4e294807d..612142382b1 100644 --- a/dev-tools/cmd/buildfleetcfg/buildfleetcfg.go +++ b/dev-tools/cmd/buildfleetcfg/buildfleetcfg.go @@ -83,7 +83,11 @@ func main() { return } - os.WriteFile(output, data, 0640) + err = os.WriteFile(output, data, 0640) + if err != nil { + fmt.Fprintf(os.Stderr, "Error while writing the file, err: %+v\n", err) + } + return } @@ -94,11 +98,11 @@ func gen(path string, l string) ([]byte, error) { } if len(files) > 1 { - return nil, fmt.Errorf("Can only embed a single configuration file") + return nil, fmt.Errorf("can only embed a single configuration file") } var buf bytes.Buffer - tmpl.Execute(&buf, struct { + err = tmpl.Execute(&buf, struct { Pack string Files []string License string @@ -107,6 +111,9 @@ func gen(path string, l string) ([]byte, error) { Files: files, License: l, }) + if err != nil { + return nil, err + } return format.Source(buf.Bytes()) } diff --git a/dev-tools/licenses/license_generate.go b/dev-tools/licenses/license_generate.go index 6a6a32c9177..5f063e51614 100644 --- a/dev-tools/licenses/license_generate.go +++ b/dev-tools/licenses/license_generate.go @@ -58,9 +58,12 @@ func main() { Headers["Elasticv2"] = string(content) var buf bytes.Buffer - Template.Execute(&buf, data{ + err = Template.Execute(&buf, data{ Licenses: Headers, }) + if err != nil { + panic(err) + } bs, err := format.Source(buf.Bytes()) if err != nil { @@ -68,8 +71,11 @@ func main() { } if output == "-" { - os.Stdout.Write(bs) + _, err = os.Stdout.Write(bs) } else { - os.WriteFile(output, bs, 0640) + err = os.WriteFile(output, bs, 0640) + } + if err != nil { + panic(err) } } diff --git a/dev-tools/mage/check.go b/dev-tools/mage/check.go index 3c95fe7cbb5..db93278ddc5 100644 --- a/dev-tools/mage/check.go +++ b/dev-tools/mage/check.go @@ -24,7 +24,7 @@ import ( // It checks the file permissions of python test cases and YAML files. // It checks .go source files using 'go vet'. func Check() error { - fmt.Println(">> check: Checking source code for common problems") //nolint:forbidigo // it's ok to use fmt.println in mage + fmt.Println(">> check: Checking source code for common problems") mg.Deps(GoVet, CheckYAMLNotExecutable, devtools.CheckNoChanges) @@ -69,7 +69,7 @@ func GoVet() error { // CheckLicenseHeaders checks license headers in .go files. func CheckLicenseHeaders() error { - fmt.Println(">> fmt - go-licenser: Checking for missing headers") //nolint:forbidigo // it's ok to use fmt.println in mage + fmt.Println(">> fmt - go-licenser: Checking for missing headers") mg.Deps(InstallGoLicenser) licenser := gotool.Licenser diff --git a/dev-tools/mage/gotool/go.go b/dev-tools/mage/gotool/go.go index 095226976f9..1f9448414a6 100644 --- a/dev-tools/mage/gotool/go.go +++ b/dev-tools/mage/gotool/go.go @@ -210,10 +210,6 @@ func runVGo(cmd string, args *Args) error { }, cmd, args) } -func runGo(cmd string, args *Args) error { - return execGoWith(sh.RunWith, cmd, args) -} - func execGoWith( fn func(map[string]string, string, ...string) error, cmd string, args *Args, diff --git a/dev-tools/mage/kubernetes/kubectl.go b/dev-tools/mage/kubernetes/kubectl.go index 84cc4371efa..b96c81ea165 100644 --- a/dev-tools/mage/kubernetes/kubectl.go +++ b/dev-tools/mage/kubernetes/kubectl.go @@ -96,23 +96,3 @@ func kubectlIn(env map[string]string, stdout, stderr io.Writer, input string, ar return c.Run() } - -func kubectlStart(env map[string]string, stdout, stderr io.Writer, args ...string) (*exec.Cmd, error) { - c := exec.Command("kubectl", args...) - c.Env = os.Environ() - for k, v := range env { - c.Env = append(c.Env, k+"="+v) - } - c.Stdout = stdout - c.Stderr = stderr - c.Stdin = nil - - if mg.Verbose() { - fmt.Println("exec:", "kubectl", strings.Join(args, " ")) - } - - if err := c.Start(); err != nil { - return nil, err - } - return c, nil -} diff --git a/dev-tools/mage/target/common/package.go b/dev-tools/mage/target/common/package.go index ca245e79635..c2237a6de0c 100644 --- a/dev-tools/mage/target/common/package.go +++ b/dev-tools/mage/target/common/package.go @@ -48,7 +48,11 @@ func PackageSystemTests() error { parent := filepath.Dir(targetFile) if !fileExists(parent) { fmt.Printf(">> creating parent dir: %s", parent) - os.Mkdir(parent, 0750) + err = os.Mkdir(parent, 0750) + if err != nil { + fmt.Printf(">> %s", err) + return err + } } err = devtools.Tar(systemTestsDir, targetFile) diff --git a/internal/pkg/agent/application/monitoring/processes_cloud_test.go b/internal/pkg/agent/application/monitoring/processes_cloud_test.go index 7f83c755ba1..f4d6bc904e3 100644 --- a/internal/pkg/agent/application/monitoring/processes_cloud_test.go +++ b/internal/pkg/agent/application/monitoring/processes_cloud_test.go @@ -90,6 +90,7 @@ func TestExpectedCloudProcessID(t *testing.T) { } for _, tc := range testcases { + tc := tc // make a copy to avoid implicit memory aliasing t.Run(tc.name, func(t *testing.T) { assert.Equal(t, tc.id, expectedCloudProcessID(&tc.component)) }) @@ -142,6 +143,7 @@ func TestMatchesCloudProcessID(t *testing.T) { } for _, tc := range testcases { + tc := tc // make a copy to avoid implicit memory aliasing t.Run(tc.name, func(t *testing.T) { assert.Equal(t, tc.matches, matchesCloudProcessID(&tc.component, tc.processID)) }) diff --git a/internal/pkg/agent/application/upgrade/artifact/download/headers_rtt_test.go b/internal/pkg/agent/application/upgrade/artifact/download/headers_rtt_test.go index 440b727fe35..faf80276eef 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/headers_rtt_test.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/headers_rtt_test.go @@ -29,7 +29,7 @@ func TestAddingHeaders(t *testing.T) { rtt := WithHeaders(c.Transport, Headers) c.Transport = rtt - resp, err := c.Get(server.URL) + resp, err := c.Get(server.URL) //nolint:noctx // this is fine in tests require.NoError(t, err) b, err := io.ReadAll(resp.Body) defer resp.Body.Close() diff --git a/internal/pkg/agent/errors/error_test.go b/internal/pkg/agent/errors/error_test.go index d45695f268a..e85e197ff8c 100644 --- a/internal/pkg/agent/errors/error_test.go +++ b/internal/pkg/agent/errors/error_test.go @@ -64,7 +64,8 @@ func TestErrorsWrap(t *testing.T) { ew := fmt.Errorf("wrapper: %w", ce) outer := New(ew) - outerCustom, ok := outer.(Error) + var outerCustom Error + ok := errors.As(outer, &outerCustom) if !ok { t.Error("expected Error") return @@ -109,7 +110,8 @@ func TestErrors(t *testing.T) { for _, tc := range cases { actualErr := New(tc.args...) - agentErr, ok := actualErr.(Error) + var agentErr Error + ok := errors.As(actualErr, &agentErr) if !ok { t.Errorf("[%s] expected Error", tc.id) continue @@ -151,7 +153,8 @@ func TestMetaFold(t *testing.T) { err3 := New("level3", err2, M("key1", "level3"), M("key3", "level3")) err4 := New("level4", err3) - resultingErr, ok := err4.(Error) + var resultingErr Error + ok := errors.As(err4, &resultingErr) if !ok { t.Fatal("error is not Error") } @@ -186,7 +189,8 @@ func TestMetaCallDoesNotModifyCollection(t *testing.T) { err3 := New("level3", err2, M("key1", "level3"), M("key3", "level3")) err4 := New("level4", err3) - resultingErr, ok := err4.(agentError) + var resultingErr agentError + ok := errors.As(err4, &resultingErr) if !ok { t.Fatal("error is not Error") } diff --git a/internal/pkg/fleetapi/uploader/uploader_test.go b/internal/pkg/fleetapi/uploader/uploader_test.go index 97dd2d6bfc9..11b4974da8e 100644 --- a/internal/pkg/fleetapi/uploader/uploader_test.go +++ b/internal/pkg/fleetapi/uploader/uploader_test.go @@ -135,6 +135,12 @@ func Test_retrySender_Send(t *testing.T) { assert.Equal(t, tc.err, err) assert.Equal(t, tc.status, resp.StatusCode) } + defer func() { + if resp != nil && resp.Body != nil { + err := resp.Body.Close() + assert.NoError(t, err) + } + }() sender.AssertExpectations(t) }) } @@ -166,6 +172,12 @@ func Test_retrySender_bodyValidation(t *testing.T) { wait: backoff, } resp, err := c.Send(context.Background(), "POST", "/", nil, nil, bytes.NewReader([]byte("abcd"))) + defer func() { + if resp != nil && resp.Body != nil { + err := resp.Body.Close() + assert.NoError(t, err) + } + }() require.NoError(t, err) assert.Equal(t, resp.StatusCode, 200) assert.Equal(t, []byte("abcd"), body1) diff --git a/pkg/component/fake/component/comp/dialer.go b/pkg/component/fake/component/comp/dialer.go deleted file mode 100644 index 9ea9b8e76c6..00000000000 --- a/pkg/component/fake/component/comp/dialer.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License 2.0; -// you may not use this file except in compliance with the Elastic License 2.0. - -//go:build !windows - -package comp - -import ( - "context" - "crypto/x509" - "net" - "strings" - - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" -) - -func dialContext(ctx context.Context, addr string, cp *x509.CertPool, serverName string) (*grpc.ClientConn, error) { - return grpc.DialContext(ctx, strings.TrimPrefix(addr, "unix://"), grpc.WithContextDialer(dialer), grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(cp, serverName))) -} - -func dialer(ctx context.Context, addr string) (net.Conn, error) { - var d net.Dialer - return d.DialContext(ctx, "unix", addr) -} diff --git a/pkg/component/fake/component/comp/dialer_windows.go b/pkg/component/fake/component/comp/dialer_windows.go deleted file mode 100644 index 56a2e269378..00000000000 --- a/pkg/component/fake/component/comp/dialer_windows.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License 2.0; -// you may not use this file except in compliance with the Elastic License 2.0. - -//go:build windows - -package comp - -import ( - "context" - "crypto/x509" - "net" - - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" - - "github.com/elastic/elastic-agent-libs/api/npipe" -) - -func dialContext(ctx context.Context, addr string, cp *x509.CertPool, serverName string) (*grpc.ClientConn, error) { - return grpc.DialContext(ctx, npipe.TransformString(addr), grpc.WithContextDialer(dialer), grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(cp, serverName))) -} - -func dialer(ctx context.Context, addr string) (net.Conn, error) { - return npipe.DialContext(npipe.TransformString(addr))(ctx, "", "") -} diff --git a/pkg/component/runtime/service_command.go b/pkg/component/runtime/service_command.go index 5009c755e53..702be287855 100644 --- a/pkg/component/runtime/service_command.go +++ b/pkg/component/runtime/service_command.go @@ -55,24 +55,22 @@ func executeCommand(ctx context.Context, log *logger.Logger, binaryPath string, // channel for the last error message from the stderr output errch := make(chan string, 1) ctxStderr := contextio.NewReader(ctx, proc.Stderr) - if ctxStderr != nil { - go func() { - var errText string - scanner := bufio.NewScanner(ctxStderr) - for scanner.Scan() { - line := scanner.Text() - if len(line) > 0 { - txt := strings.TrimSpace(line) - if len(txt) > 0 { - errText = txt - // Log error output line - log.Error(errText) - } + go func() { + var errText string + scanner := bufio.NewScanner(ctxStderr) + for scanner.Scan() { + line := scanner.Text() + if len(line) > 0 { + txt := strings.TrimSpace(line) + if len(txt) > 0 { + errText = txt + // Log error output line + log.Error(errText) } } - errch <- errText - }() - } + } + errch <- errText + }() procState := <-proc.Wait() if errors.Is(ctx.Err(), context.DeadlineExceeded) { diff --git a/pkg/control/v1/client/dial.go b/pkg/control/v1/client/dial.go index 92c871559a8..31db92bd713 100644 --- a/pkg/control/v1/client/dial.go +++ b/pkg/control/v1/client/dial.go @@ -18,7 +18,7 @@ import ( ) func dialContext(ctx context.Context) (*grpc.ClientConn, error) { - return grpc.DialContext( + return grpc.DialContext( //nolint:staticcheck // Only the deprecated version allows this call to be blocking ctx, strings.TrimPrefix(control.Address(), "unix://"), grpc.WithTransportCredentials(insecure.NewCredentials()), diff --git a/pkg/control/v1/client/dial_windows.go b/pkg/control/v1/client/dial_windows.go index 824578bed70..6cb76ac8637 100644 --- a/pkg/control/v1/client/dial_windows.go +++ b/pkg/control/v1/client/dial_windows.go @@ -19,7 +19,7 @@ import ( ) func dialContext(ctx context.Context) (*grpc.ClientConn, error) { - return grpc.DialContext( + return grpc.DialContext( //nolint:staticcheck // Only the deprecated version allows this call to be blocking ctx, control.Address(), grpc.WithTransportCredentials(insecure.NewCredentials()), diff --git a/pkg/control/v2/client/dial.go b/pkg/control/v2/client/dial.go index 49c022a4ce8..7647a4c9cee 100644 --- a/pkg/control/v2/client/dial.go +++ b/pkg/control/v2/client/dial.go @@ -21,7 +21,7 @@ func dialContext(ctx context.Context, address string, maxMsgSize int, opts ...gr grpc.WithContextDialer(dialer), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)), ) - return grpc.DialContext(ctx, address, opts...) + return grpc.DialContext(ctx, address, opts...) //nolint:staticcheck // Only the deprecated version allows this call to be blocking } func dialer(ctx context.Context, addr string) (net.Conn, error) { diff --git a/pkg/control/v2/client/dial_windows.go b/pkg/control/v2/client/dial_windows.go index 79d58af4faf..0ab339e1fdc 100644 --- a/pkg/control/v2/client/dial_windows.go +++ b/pkg/control/v2/client/dial_windows.go @@ -23,7 +23,7 @@ func dialContext(ctx context.Context, address string, maxMsgSize int, opts ...gr grpc.WithContextDialer(dialer), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)), ) - return grpc.DialContext(ctx, address, opts...) + return grpc.DialContext(ctx, address, opts...) //nolint:staticcheck // Only the deprecated version allows this call to be blocking } func dialer(ctx context.Context, addr string) (net.Conn, error) { diff --git a/testing/upgradetest/upgrader.go b/testing/upgradetest/upgrader.go index 07ed91856a1..7094e49fda5 100644 --- a/testing/upgradetest/upgrader.go +++ b/testing/upgradetest/upgrader.go @@ -516,7 +516,7 @@ func WaitHealthyAndVersion(ctx context.Context, f *atesting.Fixture, versionInfo // If we're in an upgrade process, the versions might not match // so we wait to see if we get to a stable version if errors.Is(err, ErrVerMismatch) { - logger.Logf("version mismatch, ignoring it, time until timeout: %s", deadline.Sub(time.Now())) + logger.Logf("version mismatch, ignoring it, time until timeout: %s", time.Until(deadline)) continue } if err == nil { From 23621df5ee6116b8d0d953b152c29b2fdae7ae03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paolo=20Chil=C3=A0?= Date: Tue, 1 Oct 2024 16:49:31 +0200 Subject: [PATCH 13/20] Include python wheel packages when moving downloaded dependencies archives (#5647) This commit modifies movePackagesToArchive() function to include python wheel packages introduced with commit 701f8b9 . This solves an issue when packaging docker images using a DROP_PATH env var: prior to this change, the python wheel archives present in DROP_PATH were not moved to `/archive/` failing to create an image for which the Dockerfile expects to install the python wheel package --- magefile.go | 56 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/magefile.go b/magefile.go index 06ef0c1a2c3..03aecaeb620 100644 --- a/magefile.go +++ b/magefile.go @@ -93,6 +93,8 @@ const ( agentCoreProjectName = "elastic-agent-core" helmChartPath = "./deploy/helm/elastic-agent" + + sha512FileExt = ".sha512" ) var ( @@ -947,16 +949,16 @@ func runAgent(ctx context.Context, env map[string]string) error { func packageAgent(ctx context.Context, platforms []string, dependenciesVersion string, manifestResponse *manifest.Build, agentPackaging, agentBinaryTarget mg.Fn) error { fmt.Println("--- Package Elastic-Agent") - requiredPackages := []string{} + platformPackageSuffixes := []string{} for _, p := range platforms { - requiredPackages = append(requiredPackages, manifest.PlatformPackages[p]) + platformPackageSuffixes = append(platformPackageSuffixes, manifest.PlatformPackages[p]) } if mg.Verbose() { - log.Printf("--- Packaging dependenciesVersion[%s], %+v \n", dependenciesVersion, requiredPackages) + log.Printf("--- Packaging dependenciesVersion[%s], %+v \n", dependenciesVersion, platformPackageSuffixes) } // download/copy all the necessary dependencies for packaging elastic-agent - archivePath, dropPath := collectPackageDependencies(platforms, dependenciesVersion, requiredPackages) + archivePath, dropPath := collectPackageDependencies(platforms, dependenciesVersion, platformPackageSuffixes) // cleanup after build defer os.RemoveAll(archivePath) @@ -972,7 +974,7 @@ func packageAgent(ctx context.Context, platforms []string, dependenciesVersion s defer os.RemoveAll(flatPath) // extract all dependencies from their archives into flat dir - flattenDependencies(requiredPackages, dependenciesVersion, archivePath, dropPath, flatPath, manifestResponse) + flattenDependencies(platformPackageSuffixes, dependenciesVersion, archivePath, dropPath, flatPath, manifestResponse) // package agent log.Println("--- Running packaging function") @@ -990,7 +992,7 @@ func packageAgent(ctx context.Context, platforms []string, dependenciesVersion s // NOTE: after the build is done the caller must: // - delete archivePath and dropPath contents // - unset AGENT_DROP_PATH environment variable -func collectPackageDependencies(platforms []string, packageVersion string, requiredPackages []string) (archivePath string, dropPath string) { +func collectPackageDependencies(platforms []string, packageVersion string, platformPackageSuffixes []string) (archivePath string, dropPath string) { dropPath, found := os.LookupEnv(agentDropPath) @@ -1009,7 +1011,7 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi if mg.Verbose() { log.Printf(">> Creating drop-in folder %+v \n", dropPath) } - archivePath = movePackagesToArchive(dropPath, requiredPackages) + archivePath = movePackagesToArchive(dropPath, platformPackageSuffixes, packageVersion) if hasSnapshotEnv() { packageVersion = fmt.Sprintf("%s-SNAPSHOT", packageVersion) @@ -1061,7 +1063,7 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi packagesCopied := 0 - if !requiredPackagesPresent(pwd, b, packageVersion, requiredPackages) { + if !requiredPackagesPresent(pwd, b, packageVersion, platformPackageSuffixes) { fmt.Printf("--- Package %s\n", pwd) cmd := exec.Command("mage", "package") cmd.Dir = pwd @@ -1079,7 +1081,7 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi // copy to new drop sourcePath := filepath.Join(pwd, "build", "distributions") - for _, rp := range requiredPackages { + for _, rp := range platformPackageSuffixes { files, err := filepath.Glob(filepath.Join(sourcePath, "*"+rp+"*")) if err != nil { panic(err) @@ -1112,7 +1114,7 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi } } } else { - archivePath = movePackagesToArchive(dropPath, requiredPackages) + archivePath = movePackagesToArchive(dropPath, platformPackageSuffixes, packageVersion) } return archivePath, dropPath } @@ -1161,6 +1163,10 @@ func flattenDependencies(requiredPackages []string, packageVersion, archivePath, } matches = append(matches, zipMatches...) + if mg.Verbose() { + log.Printf("--- Unfiltered dependencies to flatten in %s : %v", targetPath, matches) + } + // never flatten any python wheels, the packages.yml and docker should handle // those specifically so that the python wheels are installed into the container matches = removePythonWheels(matches, packageVersion) @@ -1426,7 +1432,7 @@ func downloadDRAArtifacts(ctx context.Context, manifestUrl string, downloadDir s } // download the SHA to check integrity - artifactSHADownloadPath := filepath.Join(draDownloadDir, pkgName+".sha512") + artifactSHADownloadPath := filepath.Join(draDownloadDir, pkgName+sha512FileExt) err = manifest.DownloadPackage(errCtx, pkgDesc.ShaURL, artifactSHADownloadPath) if err != nil { return fmt.Errorf("downloading SHA for %q: %w", pkgName, err) @@ -1564,7 +1570,7 @@ func appendComponentChecksums(versionedDropPath string, checksums map[string]str } // movePackagesToArchive Create archive folder and move any pre-existing artifacts into it. -func movePackagesToArchive(dropPath string, requiredPackages []string) string { +func movePackagesToArchive(dropPath string, platformPackageSuffixes []string, packageVersion string) string { archivePath := filepath.Join(dropPath, "archives") os.MkdirAll(archivePath, 0755) @@ -1580,8 +1586,14 @@ func movePackagesToArchive(dropPath string, requiredPackages []string) string { matches = append(matches, zipMatches...) for _, f := range matches { - for _, rp := range requiredPackages { - if !strings.Contains(f, rp) { + for _, packageSuffix := range platformPackageSuffixes { + if mg.Verbose() { + log.Printf("--- Evaluating moving dependency %s to archive path %s\n", f, archivePath) + } + if !strings.Contains(f, packageSuffix) && !isPythonWheelPackage(f, packageVersion) { + if mg.Verbose() { + log.Printf("--- Skipped moving dependency %s to archive path\n", f) + } continue } @@ -1596,7 +1608,7 @@ func movePackagesToArchive(dropPath string, requiredPackages []string) string { continue } - targetPath := filepath.Join(archivePath, rp, filepath.Base(f)) + targetPath := filepath.Join(archivePath, packageSuffix, filepath.Base(f)) targetDir := filepath.Dir(targetPath) if err := os.MkdirAll(targetDir, 0750); err != nil { fmt.Printf("warning: failed to create directory %s: %s", targetDir, err) @@ -1604,12 +1616,26 @@ func movePackagesToArchive(dropPath string, requiredPackages []string) string { if err := os.Rename(f, targetPath); err != nil { panic(fmt.Errorf("failed renaming file: %w", err)) } + if mg.Verbose() { + log.Printf("--- Moved dependency in archive path %s => %s\n", f, targetPath) + } } } return archivePath } +func isPythonWheelPackage(f string, packageVersion string) bool { + fileBaseName := filepath.Base(f) + for _, spec := range manifest.ExpectedBinaries { + packageName := spec.GetPackageName(packageVersion, "") + if spec.PythonWheel && (fileBaseName == packageName || fileBaseName == packageName+sha512FileExt) { + return true + } + } + return false +} + func selectedPackageTypes() string { if len(devtools.SelectedPackageTypes) == 0 { return "" From 3fe574119e8f0729fa2338afb6f1b350f3e6743f Mon Sep 17 00:00:00 2001 From: Michel Laterman <82832767+michel-laterman@users.noreply.github.com> Date: Tue, 1 Oct 2024 09:01:12 -0700 Subject: [PATCH 14/20] TestManager_StartStopComponent add test logs, alternate manager ready signal (#5353) use channel instead of bool for ready signal and close when ready, add test logs --- pkg/component/runtime/manager.go | 7 +++---- pkg/component/runtime/manager_fake_input_test.go | 5 ++++- pkg/component/runtime/manager_test.go | 12 +++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/component/runtime/manager.go b/pkg/component/runtime/manager.go index 5266b2bc1a9..6e34e99ce9c 100644 --- a/pkg/component/runtime/manager.go +++ b/pkg/component/runtime/manager.go @@ -16,7 +16,6 @@ import ( "path" "strings" "sync" - "sync/atomic" "time" "github.com/gofrs/uuid/v5" @@ -109,7 +108,7 @@ type Manager struct { grpcConfig *configuration.GRPCConfig // Set when the RPC server is ready to receive requests, for use by tests. - serverReady *atomic.Bool + serverReady chan struct{} // updateChan forwards component model updates from the public Update method // to the internal run loop. @@ -192,7 +191,7 @@ func NewManager( errCh: make(chan error), monitor: monitor, grpcConfig: grpcConfig, - serverReady: &atomic.Bool{}, + serverReady: make(chan struct{}), doneChan: make(chan struct{}), runAsOtel: runAsOtel, } @@ -343,7 +342,7 @@ LOOP: } func (m *Manager) serverLoop(ctx context.Context, listener net.Listener, server *grpc.Server) { - m.serverReady.Store(true) + close(m.serverReady) for ctx.Err() == nil { err := server.Serve(listener) if err != nil && ctx.Err() == nil { diff --git a/pkg/component/runtime/manager_fake_input_test.go b/pkg/component/runtime/manager_fake_input_test.go index ffd3768c9c5..22b960cc258 100644 --- a/pkg/component/runtime/manager_fake_input_test.go +++ b/pkg/component/runtime/manager_fake_input_test.go @@ -2703,6 +2703,7 @@ func (suite *FakeInputSuite) TestManager_StartStopComponent() { default: } + t.Log("Apply component config 1") m.Update(component.Model{Components: components}) err = <-m.errCh require.NoError(t, err, "expected no error from the manager when applying"+ @@ -2720,6 +2721,7 @@ func (suite *FakeInputSuite) TestManager_StartStopComponent() { 200*time.Millisecond, "component %s did not start", comp0ID) + t.Log("Apply component config 2") m.Update(component.Model{Components: components2}) err = <-m.errCh require.NoError(t, err, "expected no error from the manager when applying"+ @@ -2740,6 +2742,7 @@ func (suite *FakeInputSuite) TestManager_StartStopComponent() { // component 1 started, we can stop the manager cancel() + t.Log("Verify behaviour") comp0StartLogs := logs.FilterMessageSnippet( fmt.Sprintf("Starting component %q", comp0ID)).TakeAll() comp0StopLogs := logs.FilterMessageSnippet( @@ -2761,7 +2764,7 @@ func (suite *FakeInputSuite) TestManager_StartStopComponent() { assert.NoError(t, err, "Manager.Run returned and error") if t.Failed() { - t.Logf("manager logs:") + t.Log("manager logs:") for _, l := range logs.TakeAll() { t.Log(l) } diff --git a/pkg/component/runtime/manager_test.go b/pkg/component/runtime/manager_test.go index b04f07fae7b..0bfaf75ff4e 100644 --- a/pkg/component/runtime/manager_test.go +++ b/pkg/component/runtime/manager_test.go @@ -164,14 +164,12 @@ func (*testMonitoringManager) Cleanup(string) error // waitForReady waits until the RPC server is ready to be used. func waitForReady(ctx context.Context, m *Manager) error { - for !m.serverReady.Load() { - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(100 * time.Millisecond): - } + select { + case <-ctx.Done(): + return ctx.Err() + case <-m.serverReady: + return nil } - return nil } // [gRPC:8.15] Uncomment this test only after Agent/Endpoint switches fully to local gRPC, post 8.14 From 95021e1287a2a637df9c84c0e4bff0bfc661f90c Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 1 Oct 2024 12:16:10 -0700 Subject: [PATCH 15/20] Clarify that Agent package used by integration tests relies on Beats unified release (#5638) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Clarify that Agent package used by integration tests relies on Beats unified release * Update docs/test-framework-dev-guide.md Co-authored-by: Paolo Chilà --------- Co-authored-by: Paolo Chilà --- docs/test-framework-dev-guide.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/test-framework-dev-guide.md b/docs/test-framework-dev-guide.md index ff837a09cb4..b449120cbd0 100644 --- a/docs/test-framework-dev-guide.md +++ b/docs/test-framework-dev-guide.md @@ -12,9 +12,16 @@ provides a high level overview of the testing framework. ### Dependencies -Go version should be at least the same than the one in [.go-version](https://github.com/elastic/elastic-agent/blob/main/.go-version) file at the root of this repository +#### Go version +Go version should be at least the same than the one in [.go-version](https://github.com/elastic/elastic-agent/blob/main/.go-version) file at the root of this repository. -[GCloud CLI](https://cloud.google.com/sdk/gcloud) + +### GCloud CLI +The integration testing framework spins up resources in GCP. To achieve this, it needs the +[GCloud CLI](https://cloud.google.com/sdk/gcloud) to be installed on the system where the tests are initiated from. + +### Beats +The Elastic Agent package that is used for integration tests packages Beats built from the Unified Release (as opposed to DRA). There is no explicit action needed for this prerequisite but just keep in mind that if any Agent integration tests rely on certain Beats features or bugfixes, they may not be available in the integration tests yet because a unified release containing those features or bugfixes may not have happened yet. ### Configuration From 9d0b751285f96383b7b2a9a0ced58235a81dcb7b Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 25 Sep 2024 14:46:30 -0700 Subject: [PATCH 16/20] Revert forcing Agent version to be 8.16.0 --- .buildkite/scripts/steps/beats_tests.sh | 2 +- .buildkite/scripts/steps/integration-package.sh | 2 +- .buildkite/scripts/steps/integration_tests.sh | 2 +- .buildkite/scripts/steps/k8s-extended-tests.sh | 4 ++-- testing/integration/upgrade_broken_package_test.go | 2 -- 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.buildkite/scripts/steps/beats_tests.sh b/.buildkite/scripts/steps/beats_tests.sh index d2dde8d4031..05fb5b47e2a 100755 --- a/.buildkite/scripts/steps/beats_tests.sh +++ b/.buildkite/scripts/steps/beats_tests.sh @@ -28,7 +28,7 @@ run_test_for_beat(){ export WORKSPACE=$(pwd) set +e - AGENT_STACK_VERSION="8.16.0-SNAPSHOT" TEST_INTEG_CLEAN_ON_EXIT=true TEST_PLATFORMS="linux/amd64" STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage integration:testBeatServerless $beat_name + TEST_INTEG_CLEAN_ON_EXIT=true TEST_PLATFORMS="linux/amd64" STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage integration:testBeatServerless $beat_name TESTS_EXIT_STATUS=$? set -e diff --git a/.buildkite/scripts/steps/integration-package.sh b/.buildkite/scripts/steps/integration-package.sh index f244329884b..0d2f358c056 100644 --- a/.buildkite/scripts/steps/integration-package.sh +++ b/.buildkite/scripts/steps/integration-package.sh @@ -3,4 +3,4 @@ set -euo pipefail source .buildkite/scripts/common.sh -AGENT_PACKAGE_VERSION=8.16.0 PACKAGES=tar.gz,zip,rpm,deb PLATFORMS=linux/amd64,linux/arm64,windows/amd64 SNAPSHOT=true EXTERNAL=true DEV=true mage package \ No newline at end of file +PACKAGES=tar.gz,zip,rpm,deb PLATFORMS=linux/amd64,linux/arm64,windows/amd64 SNAPSHOT=true EXTERNAL=true DEV=true mage package diff --git a/.buildkite/scripts/steps/integration_tests.sh b/.buildkite/scripts/steps/integration_tests.sh index d909f8ad555..834da1cd4c6 100755 --- a/.buildkite/scripts/steps/integration_tests.sh +++ b/.buildkite/scripts/steps/integration_tests.sh @@ -19,7 +19,7 @@ fi # Run integration tests set +e -AGENT_VERSION="8.16.0-SNAPSHOT" AGENT_STACK_VERSION="${STACK_VERSION}" TEST_INTEG_CLEAN_ON_EXIT=true STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage $MAGE_TARGET $MAGE_SUBTARGET +AGENT_STACK_VERSION="${STACK_VERSION}" TEST_INTEG_CLEAN_ON_EXIT=true STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage $MAGE_TARGET $MAGE_SUBTARGET TESTS_EXIT_STATUS=$? set -e diff --git a/.buildkite/scripts/steps/k8s-extended-tests.sh b/.buildkite/scripts/steps/k8s-extended-tests.sh index 450c47e4d0d..e3d78b64003 100644 --- a/.buildkite/scripts/steps/k8s-extended-tests.sh +++ b/.buildkite/scripts/steps/k8s-extended-tests.sh @@ -25,8 +25,8 @@ else exit 10 fi -AGENT_PACKAGE_VERSION="8.16.0" DEV=true SNAPSHOT=true EXTERNAL=true PACKAGES=docker mage -v package -AGENT_VERSION="8.16.0-SNAPSHOT" TEST_INTEG_CLEAN_ON_EXIT=true INSTANCE_PROVISIONER=kind STACK_PROVISIONER=stateful SNAPSHOT=true mage integration:kubernetesMatrix +DEV=true SNAPSHOT=true EXTERNAL=true PACKAGES=docker mage -v package +TEST_INTEG_CLEAN_ON_EXIT=true INSTANCE_PROVISIONER=kind STACK_PROVISIONER=stateful SNAPSHOT=true mage integration:kubernetesMatrix TESTS_EXIT_STATUS=$? set -e diff --git a/testing/integration/upgrade_broken_package_test.go b/testing/integration/upgrade_broken_package_test.go index c1413c2575b..5ece19ddbd8 100644 --- a/testing/integration/upgrade_broken_package_test.go +++ b/testing/integration/upgrade_broken_package_test.go @@ -32,8 +32,6 @@ func TestUpgradeBrokenPackageVersion(t *testing.T) { Sudo: true, // requires Agent installation }) - t.Skip("This test cannot succeed with a AGENT_PACKAGE_VERSION override. Check contents of .buildkite/scripts/steps/beats_tests.sh") - ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute)) defer cancel() From e06e78666dc774b60ec0a2d398ec320a32feb601 Mon Sep 17 00:00:00 2001 From: Michael Katsoulis Date: Wed, 2 Oct 2024 09:44:56 +0300 Subject: [PATCH 17/20] Adjust memory requests and limits for elastic-agent when run in Kubernetes cluster (#5614) * Adjust memory requests and limits for elastic-agent when run in Kubernetes cluster --- ...1727271745-resource-limits-kubernetes.yaml | 32 +++++++++++++++++++ .../base/elastic-agent-managed-daemonset.yaml | 4 +-- .../elastic-agent-standalone-daemonset.yaml | 4 +-- .../base/elastic-agent-managed-daemonset.yaml | 4 +-- .../elastic-agent-managed-statefulset.yaml | 4 +-- .../elastic-agent-standalone-daemonset.yaml | 4 +-- .../elastic-agent-standalone-statefulset.yaml | 4 +-- .../elastic-agent-managed-kubernetes.yaml | 4 +-- .../elastic-agent-managed-daemonset.yaml | 4 +-- .../elastic-agent-standalone-kubernetes.yaml | 4 +-- .../elastic-agent-standalone-daemonset.yaml | 4 +-- .../base/elastic-agent-managed/daemonset.yaml | 4 +-- .../elastic-agent-standalone/daemonset.yaml | 4 +-- .../elastic-agent-managed-gke-autopilot.yaml | 2 +- ...lastic-agent-standalone-gke-autopilot.yaml | 2 +- ...ent-standalone-kubernetes-side-leader.yaml | 4 +-- ...agent-standalone-statefulset-side-ksm.yaml | 4 +-- 17 files changed, 62 insertions(+), 30 deletions(-) create mode 100644 changelog/fragments/1727271745-resource-limits-kubernetes.yaml diff --git a/changelog/fragments/1727271745-resource-limits-kubernetes.yaml b/changelog/fragments/1727271745-resource-limits-kubernetes.yaml new file mode 100644 index 00000000000..3dfc7323bdc --- /dev/null +++ b/changelog/fragments/1727271745-resource-limits-kubernetes.yaml @@ -0,0 +1,32 @@ +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: feature + +# Change summary; a 80ish characters long description of the change. +summary: Increase Elastic Agent memory requests and limits when deployed in Kubernetes. + +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment. +#description: + +# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. +component: elastic-agent + +# PR URL; optional; the PR number that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +pr: https://github.com/elastic/elastic-agent/pull/5614 + +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +#issue: https://github.com/owner/repo/1234 diff --git a/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml b/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml index 9de04903069..ac3346c7fa0 100644 --- a/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml +++ b/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml @@ -83,10 +83,10 @@ spec: # - SYS_ADMIN resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: proc mountPath: /hostfs/proc diff --git a/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml b/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml index c1de6c0c11a..afb0f10a47a 100644 --- a/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml +++ b/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml @@ -90,10 +90,10 @@ spec: # - SYS_ADMIN resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: datastreams mountPath: /etc/elastic-agent/agent.yml diff --git a/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml b/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml index 2d67eac1407..5ca97d104f2 100644 --- a/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml +++ b/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml @@ -83,10 +83,10 @@ spec: # - SYS_ADMIN resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: proc mountPath: /hostfs/proc diff --git a/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-managed/extra/elastic-agent-managed-statefulset.yaml b/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-managed/extra/elastic-agent-managed-statefulset.yaml index dcaf2c3095a..d8667ab7a81 100644 --- a/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-managed/extra/elastic-agent-managed-statefulset.yaml +++ b/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-managed/extra/elastic-agent-managed-statefulset.yaml @@ -83,10 +83,10 @@ spec: # - SYS_ADMIN resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: proc mountPath: /hostfs/proc diff --git a/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml b/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml index 9dcd7672a6d..2e7f32e5106 100644 --- a/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml +++ b/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml @@ -90,10 +90,10 @@ spec: # - SYS_ADMIN resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: datastreams mountPath: /etc/elastic-agent/agent.yml diff --git a/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/extra/elastic-agent-standalone-statefulset.yaml b/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/extra/elastic-agent-standalone-statefulset.yaml index dd211a8cbd4..ea4c85562c9 100644 --- a/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/extra/elastic-agent-standalone-statefulset.yaml +++ b/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/extra/elastic-agent-standalone-statefulset.yaml @@ -90,10 +90,10 @@ spec: # - SYS_ADMIN resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: datastreams mountPath: /etc/elastic-agent/agent.yml diff --git a/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml b/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml index cae617b3af6..604787eb1d8 100644 --- a/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml +++ b/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml @@ -83,10 +83,10 @@ spec: # - SYS_ADMIN resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: proc mountPath: /hostfs/proc diff --git a/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml b/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml index 61939c5a72b..98ba1ab8102 100644 --- a/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml +++ b/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml @@ -83,10 +83,10 @@ spec: # - SYS_ADMIN resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: proc mountPath: /hostfs/proc diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml index 30073bd2f02..d529657bd9a 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml @@ -759,10 +759,10 @@ spec: # - SYS_ADMIN resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: datastreams mountPath: /etc/elastic-agent/agent.yml diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml index 908ac0124f5..7ffec40c9d0 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml @@ -90,10 +90,10 @@ spec: # - SYS_ADMIN resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: datastreams mountPath: /etc/elastic-agent/agent.yml diff --git a/dev-tools/kubernetes/base/elastic-agent-managed/daemonset.yaml b/dev-tools/kubernetes/base/elastic-agent-managed/daemonset.yaml index da2e4911756..cd97cbd86f7 100644 --- a/dev-tools/kubernetes/base/elastic-agent-managed/daemonset.yaml +++ b/dev-tools/kubernetes/base/elastic-agent-managed/daemonset.yaml @@ -38,10 +38,10 @@ spec: runAsUser: 0 resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: proc mountPath: /hostfs/proc diff --git a/dev-tools/kubernetes/base/elastic-agent-standalone/daemonset.yaml b/dev-tools/kubernetes/base/elastic-agent-standalone/daemonset.yaml index 2d126056cb0..2978da9de06 100644 --- a/dev-tools/kubernetes/base/elastic-agent-standalone/daemonset.yaml +++ b/dev-tools/kubernetes/base/elastic-agent-standalone/daemonset.yaml @@ -54,10 +54,10 @@ spec: runAsUser: 0 resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: datastreams mountPath: /etc/elastic-agent/agent.yml diff --git a/docs/manifests/elastic-agent-managed-gke-autopilot.yaml b/docs/manifests/elastic-agent-managed-gke-autopilot.yaml index c2ec4299be8..7ab4ef2074c 100644 --- a/docs/manifests/elastic-agent-managed-gke-autopilot.yaml +++ b/docs/manifests/elastic-agent-managed-gke-autopilot.yaml @@ -73,7 +73,7 @@ spec: resources: limits: #cpu: 200m # Keep this commented. We dont set the cpu limit to avoid scheduling problems of agent in autopilot scenarios - memory: 700Mi + memory: 1Gi ephemeral-storage: "500Mi" volumeMounts: - name: varlog diff --git a/docs/manifests/elastic-agent-standalone-gke-autopilot.yaml b/docs/manifests/elastic-agent-standalone-gke-autopilot.yaml index e4dc83fad10..e48e4b59cad 100644 --- a/docs/manifests/elastic-agent-standalone-gke-autopilot.yaml +++ b/docs/manifests/elastic-agent-standalone-gke-autopilot.yaml @@ -384,7 +384,7 @@ spec: resources: limits: # cpu: 200m # Keep this commented. We dont set the cpu limit to avoid scheduling problems of agent in autopilot scenarios - memory: 700Mi + memory: 1Gi ephemeral-storage: "500Mi" volumeMounts: - name: datastreams diff --git a/docs/manifests/kustomize-autosharding/elastic-agent-standalone-kubernetes-side-leader.yaml b/docs/manifests/kustomize-autosharding/elastic-agent-standalone-kubernetes-side-leader.yaml index e571b8e02a8..97ccd945594 100644 --- a/docs/manifests/kustomize-autosharding/elastic-agent-standalone-kubernetes-side-leader.yaml +++ b/docs/manifests/kustomize-autosharding/elastic-agent-standalone-kubernetes-side-leader.yaml @@ -602,10 +602,10 @@ spec: runAsUser: 0 resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: datastreams mountPath: /etc/elastic-agent/agent.yml diff --git a/docs/manifests/kustomize-autosharding/elastic-agent-standalone-statefulset-side-ksm.yaml b/docs/manifests/kustomize-autosharding/elastic-agent-standalone-statefulset-side-ksm.yaml index 3361542416e..344e5c1d6b5 100644 --- a/docs/manifests/kustomize-autosharding/elastic-agent-standalone-statefulset-side-ksm.yaml +++ b/docs/manifests/kustomize-autosharding/elastic-agent-standalone-statefulset-side-ksm.yaml @@ -409,10 +409,10 @@ spec: runAsUser: 0 resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: datastreams mountPath: /etc/elastic-agent/agent.yml From b8a685c29c73accd381d19a08960cbddab4b34fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paolo=20Chil=C3=A0?= Date: Wed, 2 Oct 2024 11:44:54 +0200 Subject: [PATCH 18/20] Fix docker image build when multiple platforms are specified (#5658) Force merging to unblock the unified release process --- dev-tools/mage/dockerbuilder.go | 2 +- magefile.go | 47 +++++++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index f99f4f1dcdb..0e0bd2078ca 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -46,7 +46,7 @@ func (b *dockerBuilder) Build() error { } if err := b.copyFiles(); err != nil { - return err + return fmt.Errorf("error copying files for docker variant %q: %w", b.DockerVariant, err) } if err := b.prepareBuild(); err != nil { diff --git a/magefile.go b/magefile.go index 03aecaeb620..670920cf128 100644 --- a/magefile.go +++ b/magefile.go @@ -1537,7 +1537,7 @@ func downloadBinary(ctx context.Context, project string, packageName string, bin } compl.Add(1) - fmt.Printf("Done downloading %s\n", packageName) + fmt.Printf("Done downloading %s into %s\n", packageName, targetPath) return nil } } @@ -1590,7 +1590,8 @@ func movePackagesToArchive(dropPath string, platformPackageSuffixes []string, pa if mg.Verbose() { log.Printf("--- Evaluating moving dependency %s to archive path %s\n", f, archivePath) } - if !strings.Contains(f, packageSuffix) && !isPythonWheelPackage(f, packageVersion) { + // if the matched file name does not contain the platform suffix and it's not a platform-independent package, skip it + if !strings.Contains(f, packageSuffix) && !isPlatformIndependentPackage(f, packageVersion) { if mg.Verbose() { log.Printf("--- Skipped moving dependency %s to archive path\n", f) } @@ -1613,9 +1614,18 @@ func movePackagesToArchive(dropPath string, platformPackageSuffixes []string, pa if err := os.MkdirAll(targetDir, 0750); err != nil { fmt.Printf("warning: failed to create directory %s: %s", targetDir, err) } - if err := os.Rename(f, targetPath); err != nil { - panic(fmt.Errorf("failed renaming file: %w", err)) + + // Platform-independent packages need to be placed in the archive sub-folders for all platforms, copy instead of moving + if isPlatformIndependentPackage(f, packageVersion) { + if err := copyFile(f, targetPath); err != nil { + panic(fmt.Errorf("failed copying file: %w", err)) + } + } else { + if err := os.Rename(f, targetPath); err != nil { + panic(fmt.Errorf("failed renaming file: %w", err)) + } } + if mg.Verbose() { log.Printf("--- Moved dependency in archive path %s => %s\n", f, targetPath) } @@ -1625,10 +1635,37 @@ func movePackagesToArchive(dropPath string, platformPackageSuffixes []string, pa return archivePath } -func isPythonWheelPackage(f string, packageVersion string) bool { +func copyFile(src, dst string) error { + srcStat, err := os.Stat(src) + if err != nil { + return fmt.Errorf("stat source file %q: %w", src, err) + } + + srcF, err := os.Open(src) + if err != nil { + return fmt.Errorf("opening source file %q: %w", src, err) + } + defer srcF.Close() + + dstF, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, srcStat.Mode()|os.ModePerm) + if err != nil { + return fmt.Errorf("opening/creating destination file %q: %w", dst, err) + } + defer dstF.Close() + + _, err = io.Copy(dstF, srcF) + if err != nil { + return fmt.Errorf("copying file %q to %q: %w", src, dst, err) + } + + return nil +} + +func isPlatformIndependentPackage(f string, packageVersion string) bool { fileBaseName := filepath.Base(f) for _, spec := range manifest.ExpectedBinaries { packageName := spec.GetPackageName(packageVersion, "") + // as of now only python wheels packages are platform-independent if spec.PythonWheel && (fileBaseName == packageName || fileBaseName == packageName+sha512FileExt) { return true } From c57a24efab37cad135373b4790565aaeab851612 Mon Sep 17 00:00:00 2001 From: Julien Lind Date: Wed, 2 Oct 2024 13:37:50 +0200 Subject: [PATCH 19/20] Revert "Skip TestAPMConfig. (#5625)" (#5655) This reverts commit 1eae51c7f6ae588401bd0c9d1ba6094cddb8f459. --- testing/integration/apm_propagation_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/testing/integration/apm_propagation_test.go b/testing/integration/apm_propagation_test.go index 000a7641916..6325f941259 100644 --- a/testing/integration/apm_propagation_test.go +++ b/testing/integration/apm_propagation_test.go @@ -55,9 +55,6 @@ func TestAPMConfig(t *testing.T) { Group: Default, Stack: &define.Stack{}, }) - - t.Skip("https://github.com/elastic/elastic-agent/issues/5624; apm-server not working correctly") - f, err := define.NewFixtureFromLocalBuild(t, define.Version()) require.NoError(t, err) From 483bc675f348a1f50c8975015f6839fe9e993110 Mon Sep 17 00:00:00 2001 From: Julien Lind Date: Wed, 2 Oct 2024 13:54:39 +0200 Subject: [PATCH 20/20] Update Makefile to remove reviewers (#5663) * Update Makefile to remove reviewers --- deploy/kubernetes/Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy/kubernetes/Makefile b/deploy/kubernetes/Makefile index 42000e52bbe..bd67fa7b6b2 100644 --- a/deploy/kubernetes/Makefile +++ b/deploy/kubernetes/Makefile @@ -91,8 +91,7 @@ else --label automation \ --label release_note:skip \ --base main \ - --head $(ELASTIC_AGENT_BRANCH) \ - --reviewer elastic/obs-cloudnative-monitoring + --head $(ELASTIC_AGENT_BRANCH) endif else