Merge branch 'main' into revert-816-force

elastic · Oct 2, 2024 · f2215d9 · f2215d9
2 parents 9d0b751 + 483bc67
commit f2215d9
Show file tree

Hide file tree

Showing 21 changed files with 106 additions and 41 deletions.
diff --git a/changelog/fragments/1727271745-resource-limits-kubernetes.yaml b/changelog/fragments/1727271745-resource-limits-kubernetes.yaml
@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: Increase Elastic Agent memory requests and limits when deployed in Kubernetes.
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/5614
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234
diff --git a/deploy/kubernetes/Makefile b/deploy/kubernetes/Makefile
@@ -91,8 +91,7 @@ else
 		--label automation \
 		--label release_note:skip \
 		--base main \
-		--head $(ELASTIC_AGENT_BRANCH) \
-		--reviewer elastic/obs-cloudnative-monitoring
+		--head $(ELASTIC_AGENT_BRANCH)
 endif
 
 else

diff --git a/...c-agent-kustomize/default/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml b/...c-agent-kustomize/default/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml
@@ -83,10 +83,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: proc
               mountPath: /hostfs/proc

diff --git a/...t-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml b/...t-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml
@@ -90,10 +90,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml

diff --git a/...ustomize/ksm-autosharding/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml b/...ustomize/ksm-autosharding/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml
@@ -83,10 +83,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: proc
               mountPath: /hostfs/proc

diff --git a/...omize/ksm-autosharding/elastic-agent-managed/extra/elastic-agent-managed-statefulset.yaml b/...omize/ksm-autosharding/elastic-agent-managed/extra/elastic-agent-managed-statefulset.yaml
@@ -83,10 +83,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: proc
               mountPath: /hostfs/proc

diff --git a/...ze/ksm-autosharding/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml b/...ze/ksm-autosharding/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml
@@ -90,10 +90,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml

diff --git a/...ksm-autosharding/elastic-agent-standalone/extra/elastic-agent-standalone-statefulset.yaml b/...ksm-autosharding/elastic-agent-standalone/extra/elastic-agent-standalone-statefulset.yaml
@@ -90,10 +90,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml

diff --git a/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml b/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml
@@ -83,10 +83,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: proc
               mountPath: /hostfs/proc

diff --git a/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml b/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml
@@ -83,10 +83,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: proc
               mountPath: /hostfs/proc

diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml
@@ -759,10 +759,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml

diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml
@@ -90,10 +90,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml

diff --git a/dev-tools/kubernetes/base/elastic-agent-managed/daemonset.yaml b/dev-tools/kubernetes/base/elastic-agent-managed/daemonset.yaml
@@ -38,10 +38,10 @@ spec:
             runAsUser: 0
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: proc
               mountPath: /hostfs/proc

diff --git a/dev-tools/kubernetes/base/elastic-agent-standalone/daemonset.yaml b/dev-tools/kubernetes/base/elastic-agent-standalone/daemonset.yaml
@@ -54,10 +54,10 @@ spec:
             runAsUser: 0
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml

diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go
@@ -46,7 +46,7 @@ func (b *dockerBuilder) Build() error {
 	}
 
 	if err := b.copyFiles(); err != nil {
-		return err
+		return fmt.Errorf("error copying files for docker variant %q: %w", b.DockerVariant, err)
 	}
 
 	if err := b.prepareBuild(); err != nil {

diff --git a/docs/manifests/elastic-agent-managed-gke-autopilot.yaml b/docs/manifests/elastic-agent-managed-gke-autopilot.yaml
@@ -73,7 +73,7 @@ spec:
           resources:
             limits:
               #cpu: 200m # Keep this commented. We dont set the cpu limit to avoid scheduling problems of agent in autopilot scenarios
-              memory: 700Mi
+              memory: 1Gi
               ephemeral-storage: "500Mi"
           volumeMounts:
             - name: varlog

diff --git a/docs/manifests/elastic-agent-standalone-gke-autopilot.yaml b/docs/manifests/elastic-agent-standalone-gke-autopilot.yaml
@@ -384,7 +384,7 @@ spec:
           resources:
             limits:
               # cpu: 200m # Keep this commented. We dont set the cpu limit to avoid scheduling problems of agent in autopilot scenarios
-              memory: 700Mi
+              memory: 1Gi
               ephemeral-storage: "500Mi"
           volumeMounts:
             - name: datastreams

diff --git a/docs/manifests/kustomize-autosharding/elastic-agent-standalone-kubernetes-side-leader.yaml b/docs/manifests/kustomize-autosharding/elastic-agent-standalone-kubernetes-side-leader.yaml
@@ -602,10 +602,10 @@ spec:
             runAsUser: 0
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml

diff --git a/docs/manifests/kustomize-autosharding/elastic-agent-standalone-statefulset-side-ksm.yaml b/docs/manifests/kustomize-autosharding/elastic-agent-standalone-statefulset-side-ksm.yaml
@@ -409,10 +409,10 @@ spec:
             runAsUser: 0
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml

diff --git a/magefile.go b/magefile.go
@@ -1537,7 +1537,7 @@ func downloadBinary(ctx context.Context, project string, packageName string, bin
 		}
 
 		compl.Add(1)
-		fmt.Printf("Done downloading %s\n", packageName)
+		fmt.Printf("Done downloading %s into %s\n", packageName, targetPath)
 		return nil
 	}
 }
@@ -1590,7 +1590,8 @@ func movePackagesToArchive(dropPath string, platformPackageSuffixes []string, pa
 			if mg.Verbose() {
 				log.Printf("--- Evaluating moving dependency %s to archive path %s\n", f, archivePath)
 			}
-			if !strings.Contains(f, packageSuffix) && !isPythonWheelPackage(f, packageVersion) {
+			// if the matched file name does not contain the platform suffix and it's not a platform-independent package, skip it
+			if !strings.Contains(f, packageSuffix) && !isPlatformIndependentPackage(f, packageVersion) {
 				if mg.Verbose() {
 					log.Printf("--- Skipped moving dependency %s to archive path\n", f)
 				}
@@ -1613,9 +1614,18 @@ func movePackagesToArchive(dropPath string, platformPackageSuffixes []string, pa
 			if err := os.MkdirAll(targetDir, 0750); err != nil {
 				fmt.Printf("warning: failed to create directory %s: %s", targetDir, err)
 			}
-			if err := os.Rename(f, targetPath); err != nil {
-				panic(fmt.Errorf("failed renaming file: %w", err))
+
+			// Platform-independent packages need to be placed in the archive sub-folders for all platforms, copy instead of moving
+			if isPlatformIndependentPackage(f, packageVersion) {
+				if err := copyFile(f, targetPath); err != nil {
+					panic(fmt.Errorf("failed copying file: %w", err))
+				}
+			} else {
+				if err := os.Rename(f, targetPath); err != nil {
+					panic(fmt.Errorf("failed renaming file: %w", err))
+				}
 			}
+
 			if mg.Verbose() {
 				log.Printf("--- Moved dependency in archive path %s => %s\n", f, targetPath)
 			}
@@ -1625,10 +1635,37 @@ func movePackagesToArchive(dropPath string, platformPackageSuffixes []string, pa
 	return archivePath
 }
 
-func isPythonWheelPackage(f string, packageVersion string) bool {
+func copyFile(src, dst string) error {
+	srcStat, err := os.Stat(src)
+	if err != nil {
+		return fmt.Errorf("stat source file %q: %w", src, err)
+	}
+
+	srcF, err := os.Open(src)
+	if err != nil {
+		return fmt.Errorf("opening source file %q: %w", src, err)
+	}
+	defer srcF.Close()
+
+	dstF, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, srcStat.Mode()|os.ModePerm)
+	if err != nil {
+		return fmt.Errorf("opening/creating destination file %q: %w", dst, err)
+	}
+	defer dstF.Close()
+
+	_, err = io.Copy(dstF, srcF)
+	if err != nil {
+		return fmt.Errorf("copying file %q to %q: %w", src, dst, err)
+	}
+
+	return nil
+}
+
+func isPlatformIndependentPackage(f string, packageVersion string) bool {
 	fileBaseName := filepath.Base(f)
 	for _, spec := range manifest.ExpectedBinaries {
 		packageName := spec.GetPackageName(packageVersion, "")
+		// as of now only python wheels packages are platform-independent
 		if spec.PythonWheel && (fileBaseName == packageName || fileBaseName == packageName+sha512FileExt) {
 			return true
 		}

diff --git a/testing/integration/apm_propagation_test.go b/testing/integration/apm_propagation_test.go
@@ -55,9 +55,6 @@ func TestAPMConfig(t *testing.T) {
 		Group: Default,
 		Stack: &define.Stack{},
 	})
-
-	t.Skip("https://github.com/elastic/elastic-agent/issues/5624; apm-server not working correctly")
-
 	f, err := define.NewFixtureFromLocalBuild(t, define.Version())
 	require.NoError(t, err)