Add SDE+ASAN CI dimension (#1254)

Adding a CI dimension to support SDE+ASAN on Ubuntu 22.04. No memory
leaks were detected with the additional coverage, but some of these
tests hang and never exit when kicked off by the go test runner. The
current workaround is to set a 20 minute timer on the tests. Once the
timer is up and the process hasn't exited, we kill the process and
check stdout to see if the tests have actually passed or not. This
workaround is only applied to SDE as of now.
I've created CryptoAlg-2154 to help us investigate reasons this is
happening for SDE+ASAN, but it shouldn't block us from adding
new coverage for this.

CMakeLists.txt

@@ -1062,7 +1062,7 @@ if(BUILD_TESTING)
     add_custom_target(
         run_tests_with_sde
         COMMAND ${GO_EXECUTABLE} run util/all_tests.go -build-dir
-                ${PROJECT_BINARY_DIR} -sde true -sde-path "$ENV{SDEROOT}/sde"
+                ${PROJECT_BINARY_DIR} -sde=true -sde-path="$ENV{SDEROOT}/sde"
         WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
         DEPENDS all_tests
         ${MAYBE_USES_TERMINAL})

crypto/dynamic_loading_test.c

@@ -105,13 +105,15 @@ int main(int argc, char *argv[]) {
     exit(1);
   }
 
-  printf("PASS\n");
+  fprintf(stdout, "PASS\n");
+  fflush(stdout);
   return 0;
 }
 #else
 
 int main(int argc, char **argv) {
-  printf("PASS\n");
+  fprintf(stdout, "PASS\n");
+  fflush(stdout);
   return 0;
 }
 

tests/ci/cdk/cdk/codebuild/github_ci_linux_x86_omnibus.yaml

@@ -105,6 +105,26 @@ batch:
         variables:
           AWS_LC_CI_TARGET: "tests/ci/run_fips_tests.sh"
 
+    - identifier: amazonlinux2_gcc7x_intel_sde_x86_64
+      buildspec: ./tests/ci/codebuild/common/run_simple_target.yml
+      env:
+        type: LINUX_CONTAINER
+        privileged-mode: true
+        compute-type: BUILD_GENERAL1_2XLARGE
+        image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86:amazonlinux-2_gcc-7x_intel-sde_latest
+        variables:
+          AWS_LC_CI_TARGET: "tests/ci/run_tests_with_sde.sh"
+
+    - identifier: ubuntu2204_clang14x_sde_asan_x86_64
+      buildspec: ./tests/ci/codebuild/common/run_simple_target.yml
+      env:
+        type: LINUX_CONTAINER
+        privileged-mode: true
+        compute-type: BUILD_GENERAL1_2XLARGE
+        image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86:ubuntu-22.04_clang-14x-sde_latest
+        variables:
+          AWS_LC_CI_TARGET: "tests/ci/run_tests_with_sde_asan.sh"
+
     - identifier: ubuntu2204_gcc11x_x86_64
       buildspec: ./tests/ci/codebuild/common/run_simple_target.yml
       env:
@@ -341,14 +361,6 @@ batch:
         variables:
           AWS_LC_CI_TARGET: "tests/ci/run_prefix_tests.sh"
 
-    - identifier: amazonlinux2_gcc7x_intel_sde_x86_64
-      buildspec: ./tests/ci/codebuild/linux-x86/amazonlinux-2_gcc-7x_intel-sde.yml
-      env:
-        type: LINUX_CONTAINER
-        privileged-mode: true
-        compute-type: BUILD_GENERAL1_2XLARGE
-        image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86:amazonlinux-2_gcc-7x_intel-sde_latest
-
     - identifier: amazonlinux2023_gcc11x_x86_64_valgrind
       buildspec: ./tests/ci/codebuild/common/run_simple_target.yml
       env:

tests/ci/common_posix_setup.sh

@@ -221,6 +221,16 @@ function print_executable_information {
   fi
 }
 
+function sde_getenforce_check {
+  # Based on Intel SDE README, SELinux should be turned off to allow pin to work.
+  # https://software.intel.com/content/www/us/en/develop/articles/intel-software-development-emulator.html#system-configuration
+  if [[ "$(getenforce)" == 'Disabled' ]]; then
+    echo "SELinux is disabled. Disabling SELinux is needed by sde to allow pin work.";
+  else
+    echo "SELinux should be turned off to allow sde pin to work." && exit 1;
+  fi
+}
+
 print_executable_information "cmake" "--version" "CMake version"
 print_executable_information "cmake3" "--version" "CMake version (cmake3 executable)"
 print_executable_information "go" "version" "Go version"

tests/ci/docker_images/linux-x86/amazonlinux-2_gcc-7x-intel-sde/Dockerfile

@@ -5,6 +5,9 @@ FROM amazonlinux-2:gcc-7x
 
 SHELL ["/bin/bash", "-c"]
 
+ENV SDE_VERSION_TAG=sde-external-9.21.1-2023-04-24-lin
+ENV SDE_MIRROR_URL="https://downloadmirror.intel.com/777395/sde-external-9.21.1-2023-04-24-lin.tar.xz"
+
 # Enable the EPEL repository on Amazon Linux 2 before installing packages
 # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/add-repositories.html
 
@@ -23,9 +26,7 @@ RUN set -ex && \
     # Install Intel® Software Development Emulator
     # This emulator is needed when running BoringSSL/AWS-LC code under Intel's SDE for each supported chip (like ice lake).
     # https://software.intel.com/content/www/us/en/develop/articles/intel-software-development-emulator.html#system-configuration
-    wget https://downloadmirror.intel.com/684899/sde-external-9.0.0-2021-11-07-lin.tar.xz && \
-    tar -xf sde-external-9.0.0-2021-11-07-lin.tar.xz && \
-    rm sde-external-9.0.0-2021-11-07-lin.tar.xz && \
+    wget ${SDE_MIRROR_URL} && tar -xf "${SDE_VERSION_TAG}.tar.xz" && rm "${SDE_VERSION_TAG}.tar.xz" && \
     yum clean packages && \
     yum clean metadata && \
     yum clean all && \
@@ -34,5 +35,5 @@ RUN set -ex && \
 
 ENV CC=gcc
 ENV CXX=g++
-ENV SDEROOT=/sde-external-9.0.0-2021-11-07-lin
+ENV SDEROOT="/${SDE_VERSION_TAG}"
 ENV PATH="$SDEROOT:$PATH"

tests/ci/docker_images/linux-x86/build_images.sh

@@ -25,6 +25,7 @@ docker build -t ubuntu-20.04:clang-10x ubuntu-20.04_clang-10x
 docker build -t ubuntu-20.04:android -f ubuntu-20.04_android/Dockerfile ../
 docker build -t ubuntu-20.04:clang-7x-bm-framework ubuntu-20.04_clang-7x-bm-framework
 docker build -t ubuntu-22.04:base -f ubuntu-22.04_base/Dockerfile ../dependencies
+docker build -t ubuntu-22.04:clang-14x-sde ubuntu-22.04_clang-14x-sde
 docker build -t ubuntu-22.04:gcc-11x ubuntu-22.04_gcc-11x
 docker build -t ubuntu-22.04:gcc-12x ubuntu-22.04_gcc-12x
 docker build -t amazonlinux-2:base -f amazonlinux-2_base/Dockerfile ../dependencies

tests/ci/docker_images/linux-x86/push_images.sh

@@ -27,13 +27,14 @@ tag_and_push_img 'ubuntu-20.04:clang-7x-bm-framework' "${ECS_REPO}:ubuntu-20.04_
 tag_and_push_img 'ubuntu-20.04:clang-10x_formal-verification' "${ECS_REPO}:ubuntu-20.04_clang-10x_formal-verification"
 tag_and_push_img 'ubuntu-20.04:gcc-7x' "${ECS_REPO}:ubuntu-20.04_gcc-7x"
 tag_and_push_img 'ubuntu-20.04:gcc-8x' "${ECS_REPO}:ubuntu-20.04_gcc-8x"
+tag_and_push_img 'ubuntu-22.04:clang-14x-sde' "${ECS_REPO}:ubuntu-22.04_clang-14x-sde"
 tag_and_push_img 'ubuntu-22.04:gcc-11x' "${ECS_REPO}:ubuntu-22.04_gcc-11x"
 tag_and_push_img 'ubuntu-22.04:gcc-12x' "${ECS_REPO}:ubuntu-22.04_gcc-12x"
 tag_and_push_img 'centos-7:gcc-4x' "${ECS_REPO}:centos-7_gcc-4x"
 tag_and_push_img 'centos-8:gcc-8x' "${ECS_REPO}:centos-8_gcc-8x"
 tag_and_push_img 'amazonlinux-2:gcc-7x' "${ECS_REPO}:amazonlinux-2_gcc-7x"
-tag_and_push_img 'amazonlinux-2:clang-7x' "${ECS_REPO}:amazonlinux-2_clang-7x"
 tag_and_push_img 'amazonlinux-2:gcc-7x-intel-sde' "${ECS_REPO}:amazonlinux-2_gcc-7x_intel-sde"
+tag_and_push_img 'amazonlinux-2:clang-7x' "${ECS_REPO}:amazonlinux-2_clang-7x"
 tag_and_push_img 'amazonlinux-2023:gcc-11x' "${ECS_REPO}:amazonlinux-2023_gcc-11x"
 tag_and_push_img 'amazonlinux-2023:clang-15x' "${ECS_REPO}:amazonlinux-2023_clang-15x"
 tag_and_push_img 'amazonlinux-2023:clang-15x-sanitizer' "${ECS_REPO}:amazonlinux-2023_clang-15x_sanitizer"

tests/ci/docker_images/linux-x86/ubuntu-22.04_clang-14x-sde/Dockerfile

@@ -0,0 +1,42 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0 OR ISC
+
+FROM ubuntu-22.04:base
+
+SHELL ["/bin/bash", "-c"]
+
+ENV SDE_VERSION_TAG=sde-external-9.21.1-2023-04-24-lin
+ENV SDE_MIRROR_URL="https://downloadmirror.intel.com/777395/sde-external-9.21.1-2023-04-24-lin.tar.xz"
+
+RUN set -ex && \
+    apt-get update && \
+    apt-get -y --no-install-recommends upgrade && \
+    apt-get -y --no-install-recommends install \
+    clang-14 clang++-14 \
+    # This provides command `getenforce`, which can tell the current status of SELinux.
+    # Based on Interl SDE README, SELinux should be turned off to allow pin to work.
+    selinux-basics \
+    wget \
+    xz-utils \
+    tar && \
+    # Install Intel® Software Development Emulator
+    # This emulator is needed when running BoringSSL/AWS-LC code under Intel's SDE for each supported chip (like ice lake).
+    # https://software.intel.com/content/www/us/en/develop/articles/intel-software-development-emulator.html#system-configuration
+    wget ${SDE_MIRROR_URL} && tar -xf "${SDE_VERSION_TAG}.tar.xz" && rm "${SDE_VERSION_TAG}.tar.xz" && \
+    apt-get autoremove --purge -y && \
+    apt-get clean && \
+    apt-get autoclean && \
+    rm -rf /var/lib/apt/lists/* && \
+    rm -rf /tmp/*
+
+ENV CC=clang-14
+ENV CXX=clang++-14
+
+# Notes: There are two SDE executables for Linux, sde64 and sde.
+#
+# Intel SDE supports running applications in their native mode. This means if the application is 64 bit application then SDE
+# will run it in 64 bit mode, but if the application is compiled to 32 bits then SDE will run it in 32 bit mode.
+# The executable sde64 is available only in Linux because most Linux distributions do not include the 32 bit runtime. It is
+# used only when 32 bit applications are not supported on the host system.
+ENV SDEROOT="/${SDE_VERSION_TAG}"
+ENV PATH="$SDEROOT:$PATH"

tests/ci/run_tests_with_sde.sh

@@ -1,9 +1,12 @@
-#!/bin/bash -ex
+#!/usr/bin/env bash
+set -ex
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0 OR ISC
 
 source tests/ci/common_posix_setup.sh
 
+sde_getenforce_check
+
 echo "Testing AWS-LC in debug mode under Intel's SDE."
 build_and_test_with_sde
 

tests/ci/run_tests_with_sde_asan.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -ex
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0 OR ISC
+
+source tests/ci/common_posix_setup.sh
+
+sde_getenforce_check
+
+echo "Testing AWS-LC in debug mode under Intel's SDE with address sanitizer."
+build_and_test_with_sde -DASAN=1
+
+echo "Testing AWS-LC in release mode under Intel's SDE with address sanitizer."
+build_and_test_with_sde -DCMAKE_BUILD_TYPE=Release -DASAN=1

util/all_tests.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"flag"
 	"fmt"
@@ -29,6 +30,7 @@ import (
 	"strings"
 	"sync"
 	"syscall"
+	"time"
 
 	"boringssl.googlesource.com/boringssl/util/testconfig"
 	"boringssl.googlesource.com/boringssl/util/testresult"
@@ -135,7 +137,7 @@ func gdbOf(path string, args ...string) *exec.Cmd {
 	return exec.Command("xterm", xtermArgs...)
 }
 
-func sdeOf(cpu, path string, args ...string) *exec.Cmd {
+func sdeOf(cpu, path string, args ...string) (*exec.Cmd, context.CancelFunc) {
 	sdeArgs := []string{"-" + cpu}
 	// The kernel's vdso code for gettimeofday sometimes uses the RDTSCP
 	// instruction. Although SDE has a -chip_check_vsyscall flag that
@@ -147,12 +149,18 @@ func sdeOf(cpu, path string, args ...string) *exec.Cmd {
 	}
 	sdeArgs = append(sdeArgs, "--", path)
 	sdeArgs = append(sdeArgs, args...)
-	return exec.Command(*sdePath, sdeArgs...)
+
+	// TODO(CryptoAlg-2154):SDE+ASAN tests will hang without exiting if tests pass for an unknown reason.
+	// Current workaround is to manually cancel the run after 20 minutes and check the output.
+	ctx, cancel := context.WithTimeout(context.Background(), 1200*time.Second)
+
+	return exec.CommandContext(ctx, *sdePath, sdeArgs...), cancel
 }
 
 var (
 	errMoreMallocs = errors.New("child process did not exhaust all allocation calls")
 	errTestSkipped = errors.New("test was skipped")
+	errTestHanging = errors.New("test hangs without exiting")
 )
 
 func runTestOnce(test test, mallocNumToFail int64) (passed bool, err error) {
@@ -164,14 +172,22 @@ func runTestOnce(test test, mallocNumToFail int64) (passed bool, err error) {
 		args = append(args, "--no_unwind_tests")
 	}
 	var cmd *exec.Cmd
+	var cancel context.CancelFunc
+	cancelled := false
 	if *useValgrind {
 		cmd = valgrindOf(false, test.ValgrindSupp, prog, args...)
 	} else if *useCallgrind {
 		cmd = callgrindOf(prog, args...)
 	} else if *useGDB {
 		cmd = gdbOf(prog, args...)
 	} else if *useSDE {
-		cmd = sdeOf(test.cpu, prog, args...)
+		cmd, cancel = sdeOf(test.cpu, prog, args...)
+		defer cancel()
+
+		cmd.Cancel = func() error {
+			cancelled = true
+			return cmd.Process.Kill()
+		}
 	} else {
 		cmd = exec.Command(prog, args...)
 	}
@@ -201,6 +217,7 @@ func runTestOnce(test test, mallocNumToFail int64) (passed bool, err error) {
 	if err := cmd.Start(); err != nil {
 		return false, err
 	}
+
 	if err := cmd.Wait(); err != nil {
 		if exitError, ok := err.(*exec.ExitError); ok {
 			switch exitError.Sys().(syscall.WaitStatus).ExitStatus() {
@@ -210,28 +227,36 @@ func runTestOnce(test test, mallocNumToFail int64) (passed bool, err error) {
 				fmt.Print(string(outBuf.Bytes()))
 				return false, errTestSkipped
 			}
+			if cancelled {
+				return testPass(outBuf), errTestHanging
+			}
 		}
 		fmt.Print(string(outBuf.Bytes()))
 		return false, err
 	}
 
+
+	return testPass(outBuf), nil
+}
+
+func testPass(outBuf bytes.Buffer) bool {
 	// Account for Windows line-endings.
 	stdout := bytes.Replace(outBuf.Bytes(), []byte("\r\n"), []byte("\n"), -1)
 
 	if bytes.HasSuffix(stdout, []byte("PASS\n")) &&
 		(len(stdout) == 5 || stdout[len(stdout)-6] == '\n') {
-		return true, nil
+		return true
 	}
 
 	// Also accept a googletest-style pass line. This is left here in
 	// transition until the tests are all converted and this script made
 	// unnecessary.
 	if bytes.Contains(stdout, []byte("\n[  PASSED  ]")) {
-		return true, nil
+		return true
 	}
 
 	fmt.Print(string(outBuf.Bytes()))
-	return false, nil
+	return false
 }
 
 func runTest(test test) (bool, error) {
@@ -412,6 +437,17 @@ func main() {
 			fmt.Printf("%s was skipped\n", args[0])
 			skipped = append(skipped, test)
 			testOutput.AddSkip(test.longName())
+		} else if testResult.Error == errTestHanging {
+			if !testResult.Passed {
+				fmt.Printf("%s\n", test.longName())
+				fmt.Printf("%s was left hanging without finishing.\n", args[0])
+				failed = append(failed, test)
+				testOutput.AddResult(test.longName(), "FAIL")
+			} else {
+				fmt.Printf("%s\n", test.shortName())
+				fmt.Printf("%s was left hanging, but actually passed\n", args[0])
+				testOutput.AddResult(test.longName(), "PASS")
+			}
 		} else if testResult.Error != nil {
 			fmt.Printf("%s\n", test.longName())
 			fmt.Printf("%s failed to complete: %s\n", args[0], testResult.Error)