Use CentOS 7 for building release binaries

.drone.yml

@@ -90,7 +90,7 @@ steps:
   - apk add --no-cache make
   - chown -R $UID:$GID /go
   - cd /go/src/github.com/gravitational/teleport
-  - make -C build.assets release-amd64
+  - make -C build.assets release-amd64-centos7
   environment:
     ARCH: amd64
     GID: "1000"
@@ -299,7 +299,7 @@ steps:
   - chown -R $UID:$GID /go
   - cd /go/src/github.com/gravitational/teleport
   - export VERSION=$(cat /go/.version.txt)
-  - make -C build.assets release-amd64-fips
+  - make -C build.assets release-amd64-centos7-fips
   environment:
     ARCH: amd64
     FIPS: "yes"
@@ -1464,7 +1464,7 @@ steps:
   - apk add --no-cache make
   - chown -R $UID:$GID /go
   - cd /go/src/github.com/gravitational/teleport
-  - make -C build.assets release-amd64
+  - make -C build.assets release-amd64-centos7
   environment:
     ARCH: amd64
     GID: "1000"
@@ -1619,7 +1619,7 @@ steps:
   - chown -R $UID:$GID /go
   - cd /go/src/github.com/gravitational/teleport
   - export VERSION=$(cat /go/.version.txt)
-  - make -C build.assets release-amd64-fips
+  - make -C build.assets release-amd64-centos7-fips
   environment:
     ARCH: amd64
     FIPS: "yes"
@@ -5614,6 +5614,6 @@ volumes:
       name: drone-s3-debrepo-pvc
 ---
 kind: signature
-hmac: 36cb56689a9f2dc75b3adeb626cb64db2d728dd2c1060bc86d34401ad28e6e8f
+hmac: b662295f3a64d55f5b446997e0a822dc840da2aa35ede16d24282c9951168228
 
 ...

Makefile

@@ -45,6 +45,9 @@ CGOFLAG_TSH = $(CGOFLAG)
 endif
 
 ifeq ("$(OS)","linux")
+# Link static version of libgcc to reduce system dependencies.
+CGOFLAG ?= CGO_ENABLED=1 CGO_LDFLAGS="-Wl,--as-needed"
+CGOFLAG_TSH ?= CGO_ENABLED=1 CGO_LDFLAGS="-Wl,--as-needed"
 # ARM builds need to specify the correct C compiler
 ifeq ("$(ARCH)","arm")
 CGOFLAG = CGO_ENABLED=1 CC=arm-linux-gnueabihf-gcc
@@ -115,8 +118,8 @@ RS_BPF_BUILDDIR := lib/restrictedsession/bytecode
 CLANG_BPF_SYS_INCLUDES = $(shell $(CLANG) -v -E - </dev/null 2>&1 \
 	| sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }')
 
-CGOFLAG = CGO_ENABLED=1 CGO_LDFLAGS="-Wl,-Bstatic -lbpf -lelf -lz -Wl,-Bdynamic"
-CGOFLAG_TSH = CGO_ENABLED=1 CGO_LDFLAGS="-Wl,-Bstatic -lelf -lz -Wl,-Bdynamic"
+CGOFLAG = CGO_ENABLED=1 CGO_LDFLAGS="-Wl,-Bstatic -lbpf -lelf -lz -Wl,-Bdynamic -Wl,--as-needed"
+CGOFLAG_TSH = CGO_ENABLED=1
 endif
 endif
 endif

build.assets/Dockerfile-centos7

@@ -53,6 +53,35 @@ RUN git clone --depth=1 https://github.com/Yubico/libfido2.git -b 1.11.0 && \
     echo /usr/local/lib64 > /etc/ld.so.conf.d/libfido2.conf && \
     ldconfig
 
+FROM centos:7 AS libbpf
+
+# Install required dependencies.
+RUN yum groupinstall -y 'Development Tools' && \
+    yum install -y epel-release && \
+    yum update -y && \
+    yum -y install centos-release-scl-rh && \
+    yum install -y \
+    # required by libbpf
+    centos-release-scl \
+    # required by libbpf
+    devtoolset-11-gcc* \
+    # required by libbpf
+    devtoolset-11-make \
+    # required by libbpf
+    elfutils-libelf-devel-static \
+    git \
+    # required by libbpf
+    scl-utils \
+    yum clean all
+
+# Install libbpf - compile with a newer GCC. The one installed by default is not able to compile it.
+# BUILD_STATIC_ONLY disables libbpf.so build as we don't need it.
+ARG LIBBPF_VERSION
+RUN mkdir -p /opt && cd /opt && \
+    curl -L https://github.com/gravitational/libbpf/archive/refs/tags/v${LIBBPF_VERSION}.tar.gz | tar xz && \
+    cd /opt/libbpf-${LIBBPF_VERSION}/src && \
+    scl enable devtoolset-11 "make && BUILD_STATIC_ONLY=y DESTDIR=/opt/libbpf make install"
+
 FROM centos:7 AS buildbox
 
 ENV LANGUAGE=en_US.UTF-8 \
@@ -69,16 +98,27 @@ RUN (groupadd ci --gid=$GID -o && useradd ci --uid=$UID --gid=$GID --create-home
      mkdir -p -m0700 /var/lib/teleport && chown -R ci /var/lib/teleport)
 
 RUN yum groupinstall -y 'Development Tools' && \
+    yum install -y epel-release && \
+    yum update -y && \
+    yum -y install centos-release-scl-rh && \
     yum install -y \
-        git \
-        net-tools \
-        # required by Teleport PAM support
-        pam-devel \
-        perl-IPC-Cmd \
-        tree \
-        # used by our Makefile
-        which \
-        zip \
+    #required by libbpf
+    centos-release-scl \
+    # required by libbpf
+    devtoolset-11-* \
+    # required by libbpf
+    elfutils-libelf-devel-static \
+    git \
+    net-tools \
+    # required by Teleport PAM support
+    pam-devel \
+    perl-IPC-Cmd \
+    tree \
+    # used by our Makefile
+    which \
+    zip \
+    # required by libbpf
+    zlib-static && \
     yum clean all
 
 # Install etcd.
@@ -93,7 +133,7 @@ RUN mkdir -p /opt && cd /opt && curl https://storage.googleapis.com/golang/$GOLA
     /opt/go/bin/go version
 ENV GOPATH="/go" \
     GOROOT="/opt/go" \
-    PATH="/opt/bin:$PATH:/opt/go/bin:/go/bin:/go/src/github.com/gravitational/teleport/build"
+    PATH="/opt/llvm/bin:$PATH:/opt/go/bin:/go/bin:/go/src/github.com/gravitational/teleport/build"
 
 ARG BUILDARCH
 
@@ -147,6 +187,12 @@ RUN cd /usr/local/lib64 && \
 COPY pkgconfig/centos7/ /
 ENV PKG_CONFIG_PATH="/usr/local/lib64/pkgconfig"
 
+# Download pre-built CentOS 7 assets with clang needed to build BPF tools.
+RUN cd / && curl -L https://s3.amazonaws.com/clientbuilds.gravitational.io/go/centos7-assets.tar.gz | tar -xz
+
+# Copy libbpf into the final image.
+COPY --from=libbpf /opt/libbpf/usr /usr
+
 USER ci
 VOLUME ["/go/src/github.com/gravitational/teleport"]
 EXPOSE 6600 2379 2380

build.assets/Dockerfile-centos7-assets

@@ -0,0 +1,69 @@
+FROM centos:7 AS centos-devtoolset
+
+# Install required dependencies.
+RUN yum groupinstall -y 'Development Tools' && \
+    yum install -y epel-release && \
+    yum update -y && \
+    yum -y install centos-release-scl-rh && \
+    yum install -y \
+        # required by libbpf, Clang
+        centos-release-scl \
+        # required by Clang/LLVM
+        cmake3 \
+        # required by libbpf and Clang
+        devtoolset-11-gcc* \
+        # required by libbpf
+        devtoolset-11-make \
+        # required by libbpf
+        elfutils-libelf-devel \
+        # required by libbpf
+        elfutils-libelf-devel-static \
+        git \
+        # required by libbpf, Clang
+        scl-utils \
+        # required by libbpf
+        zlib-devel \
+        # required by libbpf
+        zlib-static && \
+    yum clean all
+
+# Use just created devtool image with newer GCC and Cmake
+FROM centos-devtoolset as clang10
+
+# Compile Clang 10.0.1 from source. It is needed to create BPF files.
+# Centos 7 doesn't provide it as a package unfortunately.
+# LLVM_INCLUDE_BENCHMARKS must be off, otherwise compilation fails,
+# CLANG_BUILD_TOOLS must be on, it builds clang binary,
+# LLVM_BUILD_TOOLS must be on, it builds llvm-strip binary.
+# the rest is disabled to speedup the compilation.
+RUN git clone --branch llvmorg-10.0.1 --depth=1 https://github.com/llvm/llvm-project.git && \
+    cd llvm-project/ && \
+    mkdir build && cd build/ && \
+    scl enable devtoolset-11 'bash -c "cmake3 \
+    -DCLANG_BUILD_TOOLS=ON \
+    -DCLANG_ENABLE_ARCMT=OFF \
+    -DCLANG_ENABLE_STATIC_ANALYZER=OFF \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_INSTALL_PREFIX=/opt/llvm \
+    -DLLVM_BUILD_TOOLS=ON \
+    -DLLVM_BUILD_UTILS=OFF \
+    -DLLVM_ENABLE_BINDINGS=OFF \
+    -DLLVM_ENABLE_PROJECTS=clang \
+    -DLLVM_INCLUDE_BENCHMARKS=OFF \
+    -DLLVM_INCLUDE_GO_TESTS=OFF \
+    -DLLVM_INCLUDE_TESTS=OFF \
+    -DLLVM_TOOL_LLI_BUILD=OFF \
+    -G \"Unix Makefiles\" ../llvm && \
+    make -j6 &&  \
+    make install"' && \
+    cd ../.. && \
+    rm -rf llvm-project
+
+FROM centos:7 AS assetbox
+
+# Copy Clang into the final image.
+COPY --from=clang10 /opt/llvm /opt/llvm/
+ENV PATH=/opt/llvm/bin:${PATH}
+
+# Create the archive and copy it to the host.
+RUN tar -czvf centos7-assets.tar.gz /opt

build.assets/Dockerfile-centos7-fips

@@ -1,3 +1,32 @@
+FROM centos:7 AS libbpf
+
+# Install required dependencies.
+RUN yum groupinstall -y 'Development Tools' && \
+    yum install -y epel-release && \
+    yum update -y && \
+    yum -y install centos-release-scl-rh && \
+    yum install -y \
+    # required by libbpf
+    centos-release-scl \
+    # required by libbpf
+    devtoolset-11-gcc* \
+    # required by libbpf
+    devtoolset-11-make \
+    # required by libbpf
+    elfutils-libelf-devel-static \
+    git \
+    # required by libbpf
+    scl-utils \
+    yum clean all
+
+# Install libbpf - compile with a newer GCC. The one installed by default is not able to compile it.
+# BUILD_STATIC_ONLY disables libbpf.so build as we don't need it.
+ARG LIBBPF_VERSION
+RUN mkdir -p /opt && cd /opt && \
+    curl -L https://github.com/gravitational/libbpf/archive/refs/tags/v${LIBBPF_VERSION}.tar.gz | tar xz && \
+    cd /opt/libbpf-${LIBBPF_VERSION}/src && \
+    scl enable devtoolset-11 "make && BUILD_STATIC_ONLY=y DESTDIR=/opt/libbpf make install"
+
 FROM centos:7
 
 ENV LANGUAGE=en_US.UTF-8 \
@@ -13,9 +42,28 @@ ARG GID
 RUN (groupadd ci --gid=$GID -o && useradd ci --uid=$UID --gid=$GID --create-home --shell=/bin/sh && \
      mkdir -p -m0700 /var/lib/teleport && chown -R ci /var/lib/teleport)
 
-# Install dev tools (make, etc) and a Perl package needed to build OpenSSL.
-RUN yum groupinstall -y "Development Tools"
-RUN yum install -y pam-devel net-tools tree git zip libatomic perl-IPC-Cmd && \
+RUN yum groupinstall -y 'Development Tools' && \
+    yum install -y epel-release && \
+    yum update -y && \
+    yum -y install centos-release-scl-rh && \
+    yum install -y \
+    #required by libbpf
+    centos-release-scl \
+    # required by libbpf
+    devtoolset-11-* \
+    # required by libbpf
+    elfutils-libelf-devel-static \
+    git \
+    net-tools \
+    # required by Teleport PAM support
+    pam-devel \
+    perl-IPC-Cmd \
+    tree \
+    # used by our Makefile
+    which \
+    zip \
+    # required by libbpf
+    zlib-static && \
     yum clean all
 
 # Install etcd.
@@ -39,14 +87,20 @@ RUN mkdir -p /go-bootstrap && cd /go-bootstrap && curl https://dl.google.com/go/
 
 ENV GOPATH="/go" \
     GOROOT="/opt/go" \
-    PATH="/opt/bin:$PATH:/opt/go/bin:/go/bin:/go/src/github.com/gravitational/teleport/build"
+    PATH="/opt/llvm/bin:$PATH:/opt/go/bin:/go/bin:/go/src/github.com/gravitational/teleport/build"
 
 # Install PAM module and policies for testing.
 COPY pam/ /opt/pam_teleport/
 RUN make -C /opt/pam_teleport install
 
 RUN chmod a-w /
 
+# Download pre-built CentOS 7 assets with clang needed to build BPF tools.
+RUN cd / && curl -L https://s3.amazonaws.com/clientbuilds.gravitational.io/go/centos7-assets.tar.gz | tar -xz
+
+# Copy libbpf into the final image.
+COPY --from=libbpf /opt/libbpf/usr /usr
+
 USER ci
 VOLUME ["/go/src/github.com/gravitational/teleport"]
 EXPOSE 6600 2379 2380

build.assets/Makefile

@@ -177,6 +177,7 @@ buildbox-centos7-fips:
 		--build-arg GID=$(GID) \
 		--build-arg BORINGCRYPTO_RUNTIME=$(BORINGCRYPTO_RUNTIME) \
 		--build-arg RUST_VERSION=$(RUST_VERSION) \
+		--build-arg LIBBPF_VERSION=$(LIBBPF_VERSION) \
 		--cache-from $(BUILDBOX_CENTOS7_FIPS) \
 		--tag $(BUILDBOX_CENTOS7_FIPS) -f Dockerfile-centos7-fips .
 
@@ -396,7 +397,7 @@ release-fips: buildbox-fips
 .PHONY:release-centos7
 release-centos7: buildbox-centos7
 	docker run $(DOCKERFLAGS) -i $(NOROOT) $(BUILDBOX_CENTOS7) \
-		/usr/bin/make release -e ADDFLAGS="$(ADDFLAGS)" OS=$(OS) ARCH=$(ARCH) RUNTIME=$(GOLANG_VERSION) FIDO2=$(FIDO2) REPRODUCIBLE=no
+		/usr/bin/scl enable devtoolset-11 'make release -e ADDFLAGS="$(ADDFLAGS)" OS=$(OS) ARCH=$(ARCH) RUNTIME=$(GOLANG_VERSION) FIDO2=$(FIDO2) REPRODUCIBLE=no'
 
 #
 # Create a Teleport FIPS package for CentOS 7 using the build container.
@@ -405,7 +406,7 @@ release-centos7: buildbox-centos7
 .PHONY:release-centos7-fips
 release-centos7-fips:
 	docker run $(DOCKERFLAGS) -i $(NOROOT) $(BUILDBOX_CENTOS7_FIPS) \
-		/usr/bin/make -C e release -e ADDFLAGS="$(ADDFLAGS)" OS=$(OS) ARCH=$(ARCH) RUNTIME=$(GOLANG_VERSION) FIPS=yes VERSION=$(VERSION) GITTAG=v$(VERSION) REPRODUCIBLE=no
+		/usr/bin/scl enable devtoolset-11 '/usr/bin/make -C e release -e ADDFLAGS="$(ADDFLAGS)" OS=$(OS) ARCH=$(ARCH) RUNTIME=$(GOLANG_VERSION) FIPS=yes VERSION=$(VERSION) GITTAG=v$(VERSION) REPRODUCIBLE=no'
 
 #
 # Create a Windows Teleport package using the build container.
@@ -462,3 +463,11 @@ print-node-version:
 .PHONY:print-buildbox-version
 print-buildbox-version:
 	@echo $(BUILDBOX_VERSION)
+
+#
+# Build CentOS 7 assets such as clang.
+#
+.PHONY:build-centos7-assets
+build-centos7-assets:
+	docker build --build-arg LIBBPF_VERSION=$(LIBBPF_VERSION) -t buildbox-centos7-assets -f Dockerfile-centos7-assets .
+	docker run -v $$(pwd):/centos7.assets -it buildbox-centos7-assets cp /centos7-assets.tar.gz /centos7.assets

dronegen/common.go

@@ -197,7 +197,8 @@ func dockerVolumeRefs(v ...volumeRef) []volumeRef {
 // releaseMakefileTarget gets the correct Makefile target for a given arch/fips/centos combo
 func releaseMakefileTarget(b buildType) string {
 	makefileTarget := fmt.Sprintf("release-%s", b.arch)
-	if b.centos7 {
+	// All x86_64 binaries are built on CentOS 7 now for better glibc compatibility.
+	if b.centos7 || b.arch == "amd64" {
 		makefileTarget += "-centos7"
 	}
 	if b.fips {