diff --git a/changelog/fragments/1727271745-resource-limits-kubernetes.yaml b/changelog/fragments/1727271745-resource-limits-kubernetes.yaml new file mode 100644 index 00000000000..3dfc7323bdc --- /dev/null +++ b/changelog/fragments/1727271745-resource-limits-kubernetes.yaml @@ -0,0 +1,32 @@ +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: feature + +# Change summary; a 80ish characters long description of the change. +summary: Increase Elastic Agent memory requests and limits when deployed in Kubernetes. + +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment. +#description: + +# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. +component: elastic-agent + +# PR URL; optional; the PR number that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +pr: https://github.com/elastic/elastic-agent/pull/5614 + +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +#issue: https://github.com/owner/repo/1234 diff --git a/deploy/kubernetes/Makefile b/deploy/kubernetes/Makefile index 42000e52bbe..bd67fa7b6b2 100644 --- a/deploy/kubernetes/Makefile +++ b/deploy/kubernetes/Makefile @@ -91,8 +91,7 @@ else --label automation \ --label release_note:skip \ --base main \ - --head $(ELASTIC_AGENT_BRANCH) \ - --reviewer elastic/obs-cloudnative-monitoring + --head $(ELASTIC_AGENT_BRANCH) endif else diff --git a/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml b/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml index 9de04903069..ac3346c7fa0 100644 --- a/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml +++ b/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml @@ -83,10 +83,10 @@ spec: # - SYS_ADMIN resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: proc mountPath: /hostfs/proc diff --git a/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml b/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml index c1de6c0c11a..afb0f10a47a 100644 --- a/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml +++ b/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml @@ -90,10 +90,10 @@ spec: # - SYS_ADMIN resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: datastreams mountPath: /etc/elastic-agent/agent.yml diff --git a/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml b/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml index 2d67eac1407..5ca97d104f2 100644 --- a/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml +++ b/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml @@ -83,10 +83,10 @@ spec: # - SYS_ADMIN resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: proc mountPath: /hostfs/proc diff --git a/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-managed/extra/elastic-agent-managed-statefulset.yaml b/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-managed/extra/elastic-agent-managed-statefulset.yaml index dcaf2c3095a..d8667ab7a81 100644 --- a/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-managed/extra/elastic-agent-managed-statefulset.yaml +++ b/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-managed/extra/elastic-agent-managed-statefulset.yaml @@ -83,10 +83,10 @@ spec: # - SYS_ADMIN resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: proc mountPath: /hostfs/proc diff --git a/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml b/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml index 9dcd7672a6d..2e7f32e5106 100644 --- a/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml +++ b/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml @@ -90,10 +90,10 @@ spec: # - SYS_ADMIN resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: datastreams mountPath: /etc/elastic-agent/agent.yml diff --git a/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/extra/elastic-agent-standalone-statefulset.yaml b/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/extra/elastic-agent-standalone-statefulset.yaml index dd211a8cbd4..ea4c85562c9 100644 --- a/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/extra/elastic-agent-standalone-statefulset.yaml +++ b/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/extra/elastic-agent-standalone-statefulset.yaml @@ -90,10 +90,10 @@ spec: # - SYS_ADMIN resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: datastreams mountPath: /etc/elastic-agent/agent.yml diff --git a/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml b/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml index cae617b3af6..604787eb1d8 100644 --- a/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml +++ b/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml @@ -83,10 +83,10 @@ spec: # - SYS_ADMIN resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: proc mountPath: /hostfs/proc diff --git a/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml b/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml index 61939c5a72b..98ba1ab8102 100644 --- a/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml +++ b/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml @@ -83,10 +83,10 @@ spec: # - SYS_ADMIN resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: proc mountPath: /hostfs/proc diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml index 30073bd2f02..d529657bd9a 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml @@ -759,10 +759,10 @@ spec: # - SYS_ADMIN resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: datastreams mountPath: /etc/elastic-agent/agent.yml diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml index 908ac0124f5..7ffec40c9d0 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml @@ -90,10 +90,10 @@ spec: # - SYS_ADMIN resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: datastreams mountPath: /etc/elastic-agent/agent.yml diff --git a/dev-tools/kubernetes/base/elastic-agent-managed/daemonset.yaml b/dev-tools/kubernetes/base/elastic-agent-managed/daemonset.yaml index da2e4911756..cd97cbd86f7 100644 --- a/dev-tools/kubernetes/base/elastic-agent-managed/daemonset.yaml +++ b/dev-tools/kubernetes/base/elastic-agent-managed/daemonset.yaml @@ -38,10 +38,10 @@ spec: runAsUser: 0 resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: proc mountPath: /hostfs/proc diff --git a/dev-tools/kubernetes/base/elastic-agent-standalone/daemonset.yaml b/dev-tools/kubernetes/base/elastic-agent-standalone/daemonset.yaml index 2d126056cb0..2978da9de06 100644 --- a/dev-tools/kubernetes/base/elastic-agent-standalone/daemonset.yaml +++ b/dev-tools/kubernetes/base/elastic-agent-standalone/daemonset.yaml @@ -54,10 +54,10 @@ spec: runAsUser: 0 resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: datastreams mountPath: /etc/elastic-agent/agent.yml diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index f99f4f1dcdb..0e0bd2078ca 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -46,7 +46,7 @@ func (b *dockerBuilder) Build() error { } if err := b.copyFiles(); err != nil { - return err + return fmt.Errorf("error copying files for docker variant %q: %w", b.DockerVariant, err) } if err := b.prepareBuild(); err != nil { diff --git a/docs/manifests/elastic-agent-managed-gke-autopilot.yaml b/docs/manifests/elastic-agent-managed-gke-autopilot.yaml index c2ec4299be8..7ab4ef2074c 100644 --- a/docs/manifests/elastic-agent-managed-gke-autopilot.yaml +++ b/docs/manifests/elastic-agent-managed-gke-autopilot.yaml @@ -73,7 +73,7 @@ spec: resources: limits: #cpu: 200m # Keep this commented. We dont set the cpu limit to avoid scheduling problems of agent in autopilot scenarios - memory: 700Mi + memory: 1Gi ephemeral-storage: "500Mi" volumeMounts: - name: varlog diff --git a/docs/manifests/elastic-agent-standalone-gke-autopilot.yaml b/docs/manifests/elastic-agent-standalone-gke-autopilot.yaml index e4dc83fad10..e48e4b59cad 100644 --- a/docs/manifests/elastic-agent-standalone-gke-autopilot.yaml +++ b/docs/manifests/elastic-agent-standalone-gke-autopilot.yaml @@ -384,7 +384,7 @@ spec: resources: limits: # cpu: 200m # Keep this commented. We dont set the cpu limit to avoid scheduling problems of agent in autopilot scenarios - memory: 700Mi + memory: 1Gi ephemeral-storage: "500Mi" volumeMounts: - name: datastreams diff --git a/docs/manifests/kustomize-autosharding/elastic-agent-standalone-kubernetes-side-leader.yaml b/docs/manifests/kustomize-autosharding/elastic-agent-standalone-kubernetes-side-leader.yaml index e571b8e02a8..97ccd945594 100644 --- a/docs/manifests/kustomize-autosharding/elastic-agent-standalone-kubernetes-side-leader.yaml +++ b/docs/manifests/kustomize-autosharding/elastic-agent-standalone-kubernetes-side-leader.yaml @@ -602,10 +602,10 @@ spec: runAsUser: 0 resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: datastreams mountPath: /etc/elastic-agent/agent.yml diff --git a/docs/manifests/kustomize-autosharding/elastic-agent-standalone-statefulset-side-ksm.yaml b/docs/manifests/kustomize-autosharding/elastic-agent-standalone-statefulset-side-ksm.yaml index 3361542416e..344e5c1d6b5 100644 --- a/docs/manifests/kustomize-autosharding/elastic-agent-standalone-statefulset-side-ksm.yaml +++ b/docs/manifests/kustomize-autosharding/elastic-agent-standalone-statefulset-side-ksm.yaml @@ -409,10 +409,10 @@ spec: runAsUser: 0 resources: limits: - memory: 700Mi + memory: 1Gi requests: cpu: 100m - memory: 400Mi + memory: 500Mi volumeMounts: - name: datastreams mountPath: /etc/elastic-agent/agent.yml diff --git a/magefile.go b/magefile.go index 03aecaeb620..670920cf128 100644 --- a/magefile.go +++ b/magefile.go @@ -1537,7 +1537,7 @@ func downloadBinary(ctx context.Context, project string, packageName string, bin } compl.Add(1) - fmt.Printf("Done downloading %s\n", packageName) + fmt.Printf("Done downloading %s into %s\n", packageName, targetPath) return nil } } @@ -1590,7 +1590,8 @@ func movePackagesToArchive(dropPath string, platformPackageSuffixes []string, pa if mg.Verbose() { log.Printf("--- Evaluating moving dependency %s to archive path %s\n", f, archivePath) } - if !strings.Contains(f, packageSuffix) && !isPythonWheelPackage(f, packageVersion) { + // if the matched file name does not contain the platform suffix and it's not a platform-independent package, skip it + if !strings.Contains(f, packageSuffix) && !isPlatformIndependentPackage(f, packageVersion) { if mg.Verbose() { log.Printf("--- Skipped moving dependency %s to archive path\n", f) } @@ -1613,9 +1614,18 @@ func movePackagesToArchive(dropPath string, platformPackageSuffixes []string, pa if err := os.MkdirAll(targetDir, 0750); err != nil { fmt.Printf("warning: failed to create directory %s: %s", targetDir, err) } - if err := os.Rename(f, targetPath); err != nil { - panic(fmt.Errorf("failed renaming file: %w", err)) + + // Platform-independent packages need to be placed in the archive sub-folders for all platforms, copy instead of moving + if isPlatformIndependentPackage(f, packageVersion) { + if err := copyFile(f, targetPath); err != nil { + panic(fmt.Errorf("failed copying file: %w", err)) + } + } else { + if err := os.Rename(f, targetPath); err != nil { + panic(fmt.Errorf("failed renaming file: %w", err)) + } } + if mg.Verbose() { log.Printf("--- Moved dependency in archive path %s => %s\n", f, targetPath) } @@ -1625,10 +1635,37 @@ func movePackagesToArchive(dropPath string, platformPackageSuffixes []string, pa return archivePath } -func isPythonWheelPackage(f string, packageVersion string) bool { +func copyFile(src, dst string) error { + srcStat, err := os.Stat(src) + if err != nil { + return fmt.Errorf("stat source file %q: %w", src, err) + } + + srcF, err := os.Open(src) + if err != nil { + return fmt.Errorf("opening source file %q: %w", src, err) + } + defer srcF.Close() + + dstF, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, srcStat.Mode()|os.ModePerm) + if err != nil { + return fmt.Errorf("opening/creating destination file %q: %w", dst, err) + } + defer dstF.Close() + + _, err = io.Copy(dstF, srcF) + if err != nil { + return fmt.Errorf("copying file %q to %q: %w", src, dst, err) + } + + return nil +} + +func isPlatformIndependentPackage(f string, packageVersion string) bool { fileBaseName := filepath.Base(f) for _, spec := range manifest.ExpectedBinaries { packageName := spec.GetPackageName(packageVersion, "") + // as of now only python wheels packages are platform-independent if spec.PythonWheel && (fileBaseName == packageName || fileBaseName == packageName+sha512FileExt) { return true } diff --git a/testing/integration/apm_propagation_test.go b/testing/integration/apm_propagation_test.go index 000a7641916..6325f941259 100644 --- a/testing/integration/apm_propagation_test.go +++ b/testing/integration/apm_propagation_test.go @@ -55,9 +55,6 @@ func TestAPMConfig(t *testing.T) { Group: Default, Stack: &define.Stack{}, }) - - t.Skip("https://github.com/elastic/elastic-agent/issues/5624; apm-server not working correctly") - f, err := define.NewFixtureFromLocalBuild(t, define.Version()) require.NoError(t, err)