Merge branch 'revert-816-force' of github.com:ycombinator/elastic-age…

…nt into revert-816-force
elastic · Oct 2, 2024 · 8ac8a12 · 8ac8a12
2 parents cd7ee88 + f2215d9
commit 8ac8a12
Show file tree

Hide file tree

Showing 101 changed files with 1,942 additions and 983 deletions.
diff --git a/.buildkite/hooks/pre-command.ps1 b/.buildkite/hooks/pre-command.ps1
@@ -1,3 +1,6 @@
+# Shorten BUILDKITE_MESSAGE if needed to avoid filling the Windows env var buffer
+$env:BUILDKITE_MESSAGE = $env:BUILDKITE_MESSAGE.Substring(0, [System.Math]::Min(2048, $env:BUILDKITE_MESSAGE.Length))
+
 # Install gcc TODO: Move to the VM image
 choco install mingw
 Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1

diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml
@@ -106,7 +106,7 @@ steps:
   - label: "Kubernetes Integration tests"
     key: "k8s-integration-tests"
     env:
-      K8S_VERSION: "v1.30.2"
+      K8S_VERSION: "v1.31.0"
       KIND_VERSION: "v0.20.0"
     command: ".buildkite/scripts/steps/k8s-extended-tests.sh"
     artifact_paths:
@@ -116,7 +116,9 @@ steps:
       - "build/diagnostics/*"
     agents:
       provider: "gcp"
+      machineType: "c2-standard-16"
       image: "family/core-ubuntu-2204"
+      diskSizeGb: 400
     notify:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Kubernetes Integration tests"
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -176,6 +176,7 @@ steps:
       - unit-tests
       - extended-windows
     allow_dependency_failure: true
+    soft_fail: true # Until https://github.com/elastic/ingest-dev/issues/4042 is resolved
 
   - group: "K8s tests"
     key: "k8s-tests"
@@ -191,10 +192,10 @@ steps:
         matrix:
           setup:
             k8s_version:
+              - "1.31.0"
               - "1.30.0"
               - "1.29.4"
               - "1.28.9"
-              - "1.27.13"
         retry:
           manual:
             allowed: true
@@ -212,6 +213,7 @@ steps:
     retry:
       manual:
         allowed: true
+    soft_fail: true # Until https://github.com/elastic/ingest-dev/issues/4042 is resolved
 
   # Triggers a dynamic step: Sync K8s
   # Runs only on main and if k8s files are changed

diff --git a/.buildkite/scripts/steps/merge.sh b/.buildkite/scripts/steps/merge.sh
@@ -4,6 +4,7 @@
 # Usage: merge.sh <step1> <step2> ... Where <step> is the id of the step that contains the coverage artifact.#  
 
 set -euo pipefail
+set -x # for debugging
 
 COV_ARTIFACT="coverage.out"
 MERGED_COV_FILE="build/TEST-go-unit.cov"

diff --git a/.golangci.yml b/.golangci.yml
@@ -124,10 +124,6 @@ linters-settings:
     # Select the Go version to target. The default is '1.13'.
     go: "1.22.7"
 
-  nakedret:
-    # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
-    max-func-lines: 0
-
   nolintlint:
     # Enable to ensure that nolint directives are all used. Default is true.
     allow-unused: false

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -1264,11 +1264,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.10.1
+Version: v0.11.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.10.1/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.11.0/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -2320,11 +2320,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-licenser@v0.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-sysinfo
-Version: v1.14.1
+Version: v1.14.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.14.1/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.14.2/LICENSE.txt:
 
 
                                  Apache License
@@ -15131,11 +15131,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/sys
-Version: v0.24.0
+Version: v0.25.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.24.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.25.0/LICENSE:
 
 Copyright 2009 The Go Authors.
 

diff --git a/_meta/config/elastic-agent.docker.yml.tmpl b/_meta/config/elastic-agent.docker.yml.tmpl
@@ -5,8 +5,9 @@ outputs:
   default:
     type: elasticsearch
     hosts: '${ELASTICSEARCH_HOSTS:http://elasticsearch:9200}'
-    username: '${ELASTICSEARCH_USERNAME:elastic}'
-    password: '${ELASTICSEARCH_PASSWORD:changeme}'
+    username: '${ELASTICSEARCH_USERNAME:}'
+    password: '${ELASTICSEARCH_PASSWORD:}'
+    api_key: '${ELASTICSEARCH_API_KEY:}'
     preset: balanced
 
 inputs:

diff --git a/changelog/fragments/1726144746-container-default-credentials.yaml b/changelog/fragments/1726144746-container-default-credentials.yaml
@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: breaking-change
+
+# Change summary; a 80ish characters long description of the change.
+summary: Remove default credentials when running Elastic Agent in container mode
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: ELASTICSEARCH_USERNAME and ELASTICSEARCH_PASSWORD now need to be explicitly set when running the agent in container mode
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234
diff --git a/changelog/fragments/1726145045-container-api-key.yaml b/changelog/fragments/1726145045-container-api-key.yaml
@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: enhancement
+
+# Change summary; a 80ish characters long description of the change.
+summary: Support ELASTICSEARCH_API_KEY environment variable when running in container mode
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234
diff --git a/changelog/fragments/1727271745-resource-limits-kubernetes.yaml b/changelog/fragments/1727271745-resource-limits-kubernetes.yaml
@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: Increase Elastic Agent memory requests and limits when deployed in Kubernetes.
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/5614
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234
diff --git a/deploy/kubernetes/Makefile b/deploy/kubernetes/Makefile
@@ -91,8 +91,7 @@ else
 		--label automation \
 		--label release_note:skip \
 		--base main \
-		--head $(ELASTIC_AGENT_BRANCH) \
-		--reviewer elastic/obs-cloudnative-monitoring
+		--head $(ELASTIC_AGENT_BRANCH)
 endif
 
 else

diff --git a/...c-agent-kustomize/default/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml b/...c-agent-kustomize/default/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml
@@ -83,10 +83,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: proc
               mountPath: /hostfs/proc

diff --git a/...t-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml b/...t-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml
@@ -90,10 +90,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml

diff --git a/...ustomize/ksm-autosharding/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml b/...ustomize/ksm-autosharding/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml
@@ -83,10 +83,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: proc
               mountPath: /hostfs/proc

diff --git a/...omize/ksm-autosharding/elastic-agent-managed/extra/elastic-agent-managed-statefulset.yaml b/...omize/ksm-autosharding/elastic-agent-managed/extra/elastic-agent-managed-statefulset.yaml
@@ -83,10 +83,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: proc
               mountPath: /hostfs/proc

diff --git a/...ze/ksm-autosharding/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml b/...ze/ksm-autosharding/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml
@@ -90,10 +90,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml

diff --git a/...ksm-autosharding/elastic-agent-standalone/extra/elastic-agent-standalone-statefulset.yaml b/...ksm-autosharding/elastic-agent-standalone/extra/elastic-agent-standalone-statefulset.yaml
@@ -90,10 +90,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml

diff --git a/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml b/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml
@@ -83,10 +83,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: proc
               mountPath: /hostfs/proc

diff --git a/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml b/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml
@@ -83,10 +83,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: proc
               mountPath: /hostfs/proc

diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml
@@ -759,10 +759,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml

diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml
@@ -90,10 +90,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml