feat: run filebeat to grab the Docker logs

elastic · Dec 14, 2022 · a24c481 · a24c481
1 parent 219bdaa
commit a24c481
Show file tree

Hide file tree

Showing 6 changed files with 125 additions and 26 deletions.
diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
@@ -177,6 +177,16 @@ pipeline {
               }
             }
           }
+          post {
+              always {
+                script{
+                  def stackIP = getNodeIp('stack')
+                  sh(label: 'Grab logs', script:"make -C .ci fetch-test-reports NODE_IP_ADDRESS=${stackIP} NODE_LABEL=debian_10_amd64")
+                  archiveArtifacts(allowEmptyArchive: true,
+                    artifacts: "outputs/${stackIP}/TEST-*.xml, outputs/${stackIP}/TEST-*.json, outputs/${stackIP}/TEST-*.json.html, outputs/${stackIP}/docker-logs/*")
+                }
+              }
+          }
         }
 
         stage('End-To-End Tests') {
@@ -556,7 +566,12 @@ def generateFunctionalTestStep(Map args = [:]){
             }
             junit2otel(traceName: 'junit-e2e-tests', allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/${testRunnerIP}/TEST-*.xml")
             archiveArtifacts allowEmptyArchive: true,
-              artifacts: "outputs/${testRunnerIP}/TEST-*.xml, outputs/${testRunnerIP}/TEST-*.json, outputs/${testRunnerIP}/TEST-*.json.html"
+              artifacts: "outputs/${testRunnerIP}/TEST-*.xml, outputs/${testRunnerIP}/TEST-*.json, outputs/${testRunnerIP}/TEST-*.json.html, outputs/${testRunnerIP}/docker-logs/*"
+
+            def stackIP = getNodeIp('stack')
+            sh(label: 'Grab logs', script:"make -C .ci fetch-test-reports NODE_IP_ADDRESS=${stackIP} NODE_LABEL=debian_10_amd64")
+            archiveArtifacts(allowEmptyArchive: true,
+              artifacts: "outputs/${stackIP}/TEST-*.xml, outputs/${stackIP}/TEST-*.json, outputs/${stackIP}/TEST-*.json.html, outputs/${stackIP}/docker-logs/*")
           }
         }
       }

diff --git a/.ci/ansible/fetch-test-reports.yml b/.ci/ansible/fetch-test-reports.yml
@@ -19,7 +19,7 @@
           paths:
             - 'group_vars'
   vars:
-    ansible_python_interpreter: "auto"
+    ansible_python_interpreter: "python3"
     ansible_shell_type: "{{ nodeShellType | default('sh') }}"
     ansible_user: "{{ nodeUser }}"
     pip_package: "python3-pip"

diff --git a/.ci/ansible/tasks/fetch_test_reports.yml b/.ci/ansible/tasks/fetch_test_reports.yml
@@ -1,13 +1,15 @@
 ---
 - name: Find the Test reports to copy/fetch
   become: no
-  ansible.builtin.find: 
+  ansible.builtin.find:
     paths: "{{ e2e_base_dir }}outputs"
     file_type: file
-    use_regex: yes   
+    use_regex: yes
+    recurse: yes
     patterns:
-      - '^TEST.*json$'
-      - '^TEST.*xml$'
+      - "^TEST.*json$"
+      - "^TEST.*xml$"
+      - "^.*.ndjson$"
   register: files_2_fetch
   tags:
     - fetch-reports
@@ -16,13 +18,15 @@
 
 - name: Find the Test reports to copy/fetch (Windows)
   become: no
-  ansible.windows.win_find: 
+  ansible.windows.win_find:
     paths: "{{ e2e_base_dir }}outputs"
     file_type: file
-    use_regex: yes   
+    use_regex: yes
+    recurse: yes
     patterns:
-      - '^TEST.*json$'
-      - '^TEST.*xml$'
+      - "^TEST.*json$"
+      - "^TEST.*xml$"
+      - "^.*.ndjson$"
   register: files_2_fetch_win
   tags:
     - fetch-reports

diff --git a/.ci/ansible/tasks/setup_test_script.yml b/.ci/ansible/tasks/setup_test_script.yml
@@ -136,27 +136,29 @@
     mode: '0777'
     dest: "{{ e2e_base_dir }}.ci/scripts/functional-test.sh"
     content: |
-       #!/usr/bin/env bash
-       set -euxo pipefail
+      #!/usr/bin/env bash
+      set -euxo pipefail
 
-       BASE_DIR="/home/{{ansible_user}}/e2e-testing"
-       SUITE="{{ lookup('env', 'SUITE') or 'fleet' }}"
-       REPORT_PREFIX="{{ lookup('env', 'REPORT_PREFIX') or 'junit' }}"
+      BASE_DIR="/home/{{ansible_user}}/e2e-testing"
+      SUITE="{{ lookup('env', 'SUITE') or 'fleet' }}"
+      REPORT_PREFIX="{{ lookup('env', 'REPORT_PREFIX') or 'junit' }}"
 
-       export PATH="$PATH:/usr/local/go/bin"
+      export PATH="$PATH:/usr/local/go/bin"
 
-       BASE_DIR=${BASE_DIR} "${BASE_DIR}/.ci/scripts/install-test-dependencies.sh" "${SUITE}"
+      BASE_DIR=${BASE_DIR} "${BASE_DIR}/.ci/scripts/install-test-dependencies.sh" "${SUITE}"
 
-       REPORT_PREFIX=$(echo "$REPORT_PREFIX" | sed -r 's/[ @~]+//g')
-       SEED="$(date +%Y-%m-%d-%H:%M:%S)"
-       REPORT="{{ e2e_base_dir }}outputs/TEST-${REPORT_PREFIX}-{{ runId }}-${SEED}"
-       echo "REPORT=\"${REPORT}"\" >> {{ e2e_home_dir }}.env
+      OUTPUT_DIR="{{ e2e_base_dir }}/outputs/docker-logs" OUTPUT_FILE="docker-logs-${REPORT_PREFIX}-{{ runId }}-${SUITE}" "${BASE_DIR}/.ci/scripts/run_filebeat.sh"
 
-       echo "Removing previous test files in the case the workspace is reused"
-       rm -f {{ e2e_base_dir }}outputs/TEST-*.*
+      REPORT_PREFIX=$(echo "$REPORT_PREFIX" | sed -r 's/[ @~]+//g')
+      SEED="$(date +%Y-%m-%d-%H:%M:%S)"
+      REPORT="{{ e2e_base_dir }}outputs/TEST-${REPORT_PREFIX}-{{ runId }}-${SEED}"
+      echo "REPORT=\"${REPORT}"\" >> {{ e2e_home_dir }}.env
 
-       TAGS="{{ lookup('env', 'TAGS') }}" \
-         FORMAT="pretty,cucumber:${REPORT}.json,junit:${REPORT}.xml" \
-         make --no-print-directory -C "{{ e2e_base_dir }}e2e/_suites/${SUITE}" functional-test
+      echo "Removing previous test files in the case the workspace is reused"
+      rm -f {{ e2e_base_dir }}outputs/TEST-*.*
+
+      TAGS="{{ lookup('env', 'TAGS') }}" \
+        FORMAT="pretty,cucumber:${REPORT}.json,junit:${REPORT}.xml" \
+        make --no-print-directory -C "{{ e2e_base_dir }}e2e/_suites/${SUITE}" functional-test
   tags:
     - scripts
diff --git a/.ci/scripts/functional-test.sh b/.ci/scripts/functional-test.sh
@@ -16,6 +16,7 @@ set -euxo pipefail
 #   - BEAT_VERSION - that's the version of the Beat to be tested. Default is stored in '.stack-version'.
 #   - ELASTIC_AGENT_VERSION - that's the version of the Elastic Agent to be tested. Default is stored in '.stack-version'.
 #
+# NOTE: this script is replaced in runtime by .ci/ansible/tasks/setup_test_script.yml
 
 BASE_VERSION="$(cat $(pwd)/.stack-version)"
 
@@ -33,6 +34,8 @@ REPORT_PREFIX=${REPORT_PREFIX:-"${SUITE}_${GOARCH}_${TAGS}"}
 rm -rf outputs || true
 mkdir -p outputs
 
+OUTPUT_DIR=$(pwd)/outputs/docker-logs .ci/scripts/run_filebeat.sh
+
 REPORT_PREFIX=$(echo "$REPORT_PREFIX" | sed -r 's/[ @~]+//g')
 REPORT="$(pwd)/outputs/TEST-${REPORT_PREFIX}"
 

diff --git a/.ci/scripts/run_filebeat.sh b/.ci/scripts/run_filebeat.sh
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+set -eu
+OUTPUT_DIR=${OUTPUT_DIR:-'/tmp/filebeat'}
+OUTPUT_FILE=${OUTPUT_FILE:-'docker'}
+CONFIG_PATH=${CONFIG_PATH:-'/tmp/filebeat.yml'}
+DOCKER_IMAGE=${DOCKER_IMAGE:-'docker.elastic.co/beats/filebeat:8.5.3'}
+
+echo "OUTPUT_DIR=${OUTPUT_DIR}"
+echo "OUTPUT_FILE=${OUTPUT_FILE}"
+echo "CONFIG_PATH=${CONFIG_PATH}"
+echo "DOCKER_IMAGE=${DOCKER_IMAGE}"
+
+docker ps --filter label="name=filebeat" -q | xargs docker kill || true
+
+mkdir -p "${OUTPUT_DIR}"
+
+cat <<EOF > "${CONFIG_PATH}"
+---
+filebeat.autodiscover:
+  providers:
+    - type: docker
+      templates:
+        - config:
+          - type: container
+            paths:
+              - /var/lib/docker/containers/\${data.docker.container.id}/*.log
+processors:
+  - add_host_metadata: ~
+  - add_cloud_metadata: ~
+  - add_docker_metadata: ~
+  - add_kubernetes_metadata: ~
+output.file:
+  path: "/output"
+  filename: ${OUTPUT_FILE}
+  permissions: 0644
+  codec.format:
+    string: '[%{[container.name]}][%{[container.image.name]}][%{[container.id]}][%{[@timestamp]}] %{[message]}'
+EOF
+
+echo "INFO: Run filebeat"
+docker run \
+  --detach \
+  -v "${OUTPUT_DIR}:/output" \
+  -v "${CONFIG_PATH}:/usr/share/filebeat/filebeat.yml" \
+  -u 0:0 \
+  -v /var/lib/docker/containers:/var/lib/docker/containers \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  -e OUTPUT_FILE="${OUTPUT_FILE}" \
+  "${DOCKER_IMAGE}" \
+    --strict.perms=false \
+    -environment container \
+    -E http.enabled=true > filebeat_docker_id
+
+ID=$(docker ps --filter label="name=filebeat" -q)
+URL=${2:-"http://localhost:5066/stats?pretty"}
+
+echo "INFO: print existing docker context"
+docker ps -a || true
+
+sleep 10
+
+echo "INFO: wait for the docker container to be available"
+N=0
+until docker exec "${ID}" curl -sSfI --retry 10 --retry-delay 5 --max-time 5 "${URL}"
+do
+  sleep 5
+  if [ "${N}" -gt 6 ]; then
+    echo "ERROR: print docker inspect"
+    docker inspect "${ID}"
+    echo "ERROR: docker container is not available"
+    docker logs "${ID}"
+    break;
+  fi
+  N=$((N + 1))
+done