diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index f30d8483f4..826c2c3689 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -177,6 +177,16 @@ pipeline { } } } + post { + always { + script{ + def stackIP = getNodeIp('stack') + sh(label: 'Grab logs', script:"make -C .ci fetch-test-reports NODE_IP_ADDRESS=${stackIP} NODE_LABEL=debian_10_amd64") + archiveArtifacts(allowEmptyArchive: true, + artifacts: "outputs/${stackIP}/TEST-*.xml, outputs/${stackIP}/TEST-*.json, outputs/${stackIP}/TEST-*.json.html, outputs/${stackIP}/docker-logs/*") + } + } + } } stage('End-To-End Tests') { @@ -556,7 +566,12 @@ def generateFunctionalTestStep(Map args = [:]){ } junit2otel(traceName: 'junit-e2e-tests', allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/${testRunnerIP}/TEST-*.xml") archiveArtifacts allowEmptyArchive: true, - artifacts: "outputs/${testRunnerIP}/TEST-*.xml, outputs/${testRunnerIP}/TEST-*.json, outputs/${testRunnerIP}/TEST-*.json.html" + artifacts: "outputs/${testRunnerIP}/TEST-*.xml, outputs/${testRunnerIP}/TEST-*.json, outputs/${testRunnerIP}/TEST-*.json.html, outputs/${testRunnerIP}/docker-logs/*" + + def stackIP = getNodeIp('stack') + sh(label: 'Grab logs', script:"make -C .ci fetch-test-reports NODE_IP_ADDRESS=${stackIP} NODE_LABEL=debian_10_amd64") + archiveArtifacts(allowEmptyArchive: true, + artifacts: "outputs/${stackIP}/TEST-*.xml, outputs/${stackIP}/TEST-*.json, outputs/${stackIP}/TEST-*.json.html, outputs/${stackIP}/docker-logs/*") } } } diff --git a/.ci/ansible/fetch-test-reports.yml b/.ci/ansible/fetch-test-reports.yml index 95aa7b634a..36d4e2e60e 100644 --- a/.ci/ansible/fetch-test-reports.yml +++ b/.ci/ansible/fetch-test-reports.yml @@ -19,7 +19,7 @@ paths: - 'group_vars' vars: - ansible_python_interpreter: "auto" + ansible_python_interpreter: "python3" ansible_shell_type: "{{ nodeShellType | default('sh') }}" ansible_user: "{{ nodeUser }}" pip_package: "python3-pip" diff --git a/.ci/ansible/tasks/fetch_test_reports.yml b/.ci/ansible/tasks/fetch_test_reports.yml index 771a08a743..cf5992c466 100644 --- a/.ci/ansible/tasks/fetch_test_reports.yml +++ b/.ci/ansible/tasks/fetch_test_reports.yml @@ -1,13 +1,15 @@ --- - name: Find the Test reports to copy/fetch become: no - ansible.builtin.find: + ansible.builtin.find: paths: "{{ e2e_base_dir }}outputs" file_type: file - use_regex: yes + use_regex: yes + recurse: yes patterns: - - '^TEST.*json$' - - '^TEST.*xml$' + - "^TEST.*json$" + - "^TEST.*xml$" + - "^.*.ndjson$" register: files_2_fetch tags: - fetch-reports @@ -16,13 +18,15 @@ - name: Find the Test reports to copy/fetch (Windows) become: no - ansible.windows.win_find: + ansible.windows.win_find: paths: "{{ e2e_base_dir }}outputs" file_type: file - use_regex: yes + use_regex: yes + recurse: yes patterns: - - '^TEST.*json$' - - '^TEST.*xml$' + - "^TEST.*json$" + - "^TEST.*xml$" + - "^.*.ndjson$" register: files_2_fetch_win tags: - fetch-reports diff --git a/.ci/ansible/tasks/setup_test_script.yml b/.ci/ansible/tasks/setup_test_script.yml index 8b70367cc6..7e4d5f6621 100644 --- a/.ci/ansible/tasks/setup_test_script.yml +++ b/.ci/ansible/tasks/setup_test_script.yml @@ -136,27 +136,29 @@ mode: '0777' dest: "{{ e2e_base_dir }}.ci/scripts/functional-test.sh" content: | - #!/usr/bin/env bash - set -euxo pipefail + #!/usr/bin/env bash + set -euxo pipefail - BASE_DIR="/home/{{ansible_user}}/e2e-testing" - SUITE="{{ lookup('env', 'SUITE') or 'fleet' }}" - REPORT_PREFIX="{{ lookup('env', 'REPORT_PREFIX') or 'junit' }}" + BASE_DIR="/home/{{ansible_user}}/e2e-testing" + SUITE="{{ lookup('env', 'SUITE') or 'fleet' }}" + REPORT_PREFIX="{{ lookup('env', 'REPORT_PREFIX') or 'junit' }}" - export PATH="$PATH:/usr/local/go/bin" + export PATH="$PATH:/usr/local/go/bin" - BASE_DIR=${BASE_DIR} "${BASE_DIR}/.ci/scripts/install-test-dependencies.sh" "${SUITE}" + BASE_DIR=${BASE_DIR} "${BASE_DIR}/.ci/scripts/install-test-dependencies.sh" "${SUITE}" - REPORT_PREFIX=$(echo "$REPORT_PREFIX" | sed -r 's/[ @~]+//g') - SEED="$(date +%Y-%m-%d-%H:%M:%S)" - REPORT="{{ e2e_base_dir }}outputs/TEST-${REPORT_PREFIX}-{{ runId }}-${SEED}" - echo "REPORT=\"${REPORT}"\" >> {{ e2e_home_dir }}.env + OUTPUT_DIR="{{ e2e_base_dir }}/outputs/docker-logs" OUTPUT_FILE="docker-logs-${REPORT_PREFIX}-{{ runId }}-${SUITE}" "${BASE_DIR}/.ci/scripts/run_filebeat.sh" - echo "Removing previous test files in the case the workspace is reused" - rm -f {{ e2e_base_dir }}outputs/TEST-*.* + REPORT_PREFIX=$(echo "$REPORT_PREFIX" | sed -r 's/[ @~]+//g') + SEED="$(date +%Y-%m-%d-%H:%M:%S)" + REPORT="{{ e2e_base_dir }}outputs/TEST-${REPORT_PREFIX}-{{ runId }}-${SEED}" + echo "REPORT=\"${REPORT}"\" >> {{ e2e_home_dir }}.env - TAGS="{{ lookup('env', 'TAGS') }}" \ - FORMAT="pretty,cucumber:${REPORT}.json,junit:${REPORT}.xml" \ - make --no-print-directory -C "{{ e2e_base_dir }}e2e/_suites/${SUITE}" functional-test + echo "Removing previous test files in the case the workspace is reused" + rm -f {{ e2e_base_dir }}outputs/TEST-*.* + + TAGS="{{ lookup('env', 'TAGS') }}" \ + FORMAT="pretty,cucumber:${REPORT}.json,junit:${REPORT}.xml" \ + make --no-print-directory -C "{{ e2e_base_dir }}e2e/_suites/${SUITE}" functional-test tags: - scripts diff --git a/.ci/scripts/functional-test.sh b/.ci/scripts/functional-test.sh index 63c9673189..5ec422e69e 100755 --- a/.ci/scripts/functional-test.sh +++ b/.ci/scripts/functional-test.sh @@ -16,6 +16,7 @@ set -euxo pipefail # - BEAT_VERSION - that's the version of the Beat to be tested. Default is stored in '.stack-version'. # - ELASTIC_AGENT_VERSION - that's the version of the Elastic Agent to be tested. Default is stored in '.stack-version'. # +# NOTE: this script is replaced in runtime by .ci/ansible/tasks/setup_test_script.yml BASE_VERSION="$(cat $(pwd)/.stack-version)" @@ -33,6 +34,8 @@ REPORT_PREFIX=${REPORT_PREFIX:-"${SUITE}_${GOARCH}_${TAGS}"} rm -rf outputs || true mkdir -p outputs +OUTPUT_DIR=$(pwd)/outputs/docker-logs .ci/scripts/run_filebeat.sh + REPORT_PREFIX=$(echo "$REPORT_PREFIX" | sed -r 's/[ @~]+//g') REPORT="$(pwd)/outputs/TEST-${REPORT_PREFIX}" diff --git a/.ci/scripts/run_filebeat.sh b/.ci/scripts/run_filebeat.sh new file mode 100755 index 0000000000..991a6c7872 --- /dev/null +++ b/.ci/scripts/run_filebeat.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -eu +OUTPUT_DIR=${OUTPUT_DIR:-'/tmp/filebeat'} +OUTPUT_FILE=${OUTPUT_FILE:-'docker'} +CONFIG_PATH=${CONFIG_PATH:-'/tmp/filebeat.yml'} +DOCKER_IMAGE=${DOCKER_IMAGE:-'docker.elastic.co/beats/filebeat:8.5.3'} + +echo "OUTPUT_DIR=${OUTPUT_DIR}" +echo "OUTPUT_FILE=${OUTPUT_FILE}" +echo "CONFIG_PATH=${CONFIG_PATH}" +echo "DOCKER_IMAGE=${DOCKER_IMAGE}" + +docker ps --filter label="name=filebeat" -q | xargs docker kill || true + +mkdir -p "${OUTPUT_DIR}" + +cat < "${CONFIG_PATH}" +--- +filebeat.autodiscover: + providers: + - type: docker + templates: + - config: + - type: container + paths: + - /var/lib/docker/containers/\${data.docker.container.id}/*.log +processors: + - add_host_metadata: ~ + - add_cloud_metadata: ~ + - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ +output.file: + path: "/output" + filename: ${OUTPUT_FILE} + permissions: 0644 + codec.format: + string: '[%{[container.name]}][%{[container.image.name]}][%{[container.id]}][%{[@timestamp]}] %{[message]}' +EOF + +echo "INFO: Run filebeat" +docker run \ + --detach \ + -v "${OUTPUT_DIR}:/output" \ + -v "${CONFIG_PATH}:/usr/share/filebeat/filebeat.yml" \ + -u 0:0 \ + -v /var/lib/docker/containers:/var/lib/docker/containers \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -e OUTPUT_FILE="${OUTPUT_FILE}" \ + "${DOCKER_IMAGE}" \ + --strict.perms=false \ + -environment container \ + -E http.enabled=true > filebeat_docker_id + +ID=$(docker ps --filter label="name=filebeat" -q) +URL=${2:-"http://localhost:5066/stats?pretty"} + +echo "INFO: print existing docker context" +docker ps -a || true + +sleep 10 + +echo "INFO: wait for the docker container to be available" +N=0 +until docker exec "${ID}" curl -sSfI --retry 10 --retry-delay 5 --max-time 5 "${URL}" +do + sleep 5 + if [ "${N}" -gt 6 ]; then + echo "ERROR: print docker inspect" + docker inspect "${ID}" + echo "ERROR: docker container is not available" + docker logs "${ID}" + break; + fi + N=$((N + 1)) +done \ No newline at end of file