New GHA workflow for simpler dotnet build

This workflow uses `dotnet test` and `dotnet build` steps directly in the GHA runner, rather than building an LfMerge builder image. That way it's a lot easier to understand the build process, and to replicate it on dev machines (just do `dotnet build` and `dotnet test`).
sillsdev · Mar 25, 2024 · 33d0dd8 · 33d0dd8
1 parent 4f55942
commit 33d0dd8
Show file tree

Hide file tree

Showing 8 changed files with 156 additions and 48 deletions.
diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
@@ -21,7 +21,7 @@ jobs:
 
     steps:
       - name: Check out current branch
-        uses: actions/checkout@v3
+        uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0
         with:
           fetch-depth: 0  # All history for all tags and branches, since version number script needs that
 
@@ -43,7 +43,7 @@ jobs:
           VERSION: ${{ steps.version.outputs.MsBuildVersion }}
 
       - name: Set up buildx for Docker
-        uses: docker/setup-buildx-action@v2.4.1
+        uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0
 
       - name: Find current UID
         id: uid
@@ -52,7 +52,7 @@ jobs:
       - name: Output diagnostics
         run: |
           echo "the developer is = ${{steps.uid.outputs.developer}}"
-          
+
       - name: Build DBVersion-specific Docker image
         uses: docker/build-push-action@v4.0.0
         with:
@@ -86,13 +86,15 @@ jobs:
         with:
           nunit_files: "**/TestResults/TestResults.xml"
 
+      # TODO: Test if this is still needed with upload-artifact@v4. Does GitHub now create tarballs that preserve Unix file permissions? If so, we can eliminate this step.
       - name: Compress tarball images for faster uploads
         run: time (tar cf - tarball | gzip -c9 > tarball.tar.gz)
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
-          name: lfmerge-tarball
+          name: lfmerge-tarball-${{ matrix.dbversion }}
           path: tarball.tar.gz
+          compression-level: 0 # Already compressed in previous step. If we ever remove the previous step, we should remove this line also
     outputs:
       MsBuildVersion: ${{ steps.output_version_number.outputs.VersionFor7000072 }}
       TagFor7000072: ${{ steps.output_version_number.outputs.TagFor7000072 }}

diff --git a/.github/workflows/dotnet-build.yml b/.github/workflows/dotnet-build.yml
@@ -0,0 +1,109 @@
+name: dotnet-build
+
+on:
+  push:
+    branches: [ develop, master, chore/dotnet-build-in-ci ]
+    # TODO: Remove chore/dotnet-build-in-ci once testing is complete
+  pull_request:
+    branches: [ develop ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  build:
+    # We won't use ubuntu-latest so that we can control when to switch to the ubuntu-24.04 runner once it comes out
+    runs-on: ubuntu-22.04
+
+    # As of 2022-08-16, we build LfMerge for LCM DB version 72 only (and will expand this to include any future DbVersions)
+    strategy:
+      matrix:
+        dbversion: [7000072]
+
+    steps:
+      - name: Check out current branch
+        uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0
+        with:
+          fetch-depth: 0  # All history for all tags and branches, since version number script needs that
+
+      - name: Calculate version number
+        id: version
+        env:
+          BUILD_NUMBER: ${{ github.run_number }}
+          DbVersion: ${{ matrix.dbversion }}
+        run: docker/scripts/get-version-number.sh
+
+      - name: Save current version number to an output
+        id: output_version_number
+        run: |
+          echo Will tag ${{matrix.dbversion}} with ${TAG}
+          echo "TagFor${{matrix.dbversion}}=${TAG}" >> $GITHUB_OUTPUT
+          echo "VersionFor${{matrix.dbversion}}=${VERSION}" >> $GITHUB_OUTPUT
+        env:
+          TAG: v${{ steps.version.outputs.MsBuildVersion }}
+          VERSION: ${{ steps.version.outputs.MsBuildVersion }}
+
+      - name: Set up buildx for Docker
+        uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0
+
+      - name: Find current UID
+        id: uid
+        run: echo "uid=$(id -u)" >> $GITHUB_OUTPUT
+
+      - name: dotnet build
+        run: dotnet build /property:Configuration=Release /property:DatabaseVersion=${DbVersion} LfMerge.sln
+        env:
+          BUILD_NUMBER: ${{ github.run_number }}
+          DbVersion: ${{ matrix.dbversion }}
+          DebPackageVersion: ${{ steps.version.outputs.DebPackageVersion }}
+          MsBuildVersion: ${{ steps.version.outputs.MsBuildVersion }}
+          MajorMinorPatch: ${{ steps.version.outputs.MajorMinorPatch }}
+          AssemblySemVer: ${{ steps.version.outputs.AssemblySemVer }}
+          AssemblySemFileVer: ${{ steps.version.outputs.AssemblySemFileVer }}
+          InformationalVersion: ${{ steps.version.outputs.InformationalVersion }}
+
+      # TODO: Will need to add step to mkdir /usr/lib/lfmerge/${DbVersion} and any other directories that the tests assume are already there
+
+      - name: dotnet test
+        run: dotnet test -l:"console;verbosity=normal" -l:nunit -c Release
+        env:
+          BUILD_NUMBER: ${{ github.run_number }}
+          DbVersion: ${{ matrix.dbversion }}
+          VSTEST_TESTHOST_SHUTDOWN_TIMEOUT: "30000"
+          DebPackageVersion: ${{ steps.version.outputs.DebPackageVersion }}
+          MsBuildVersion: ${{ steps.version.outputs.MsBuildVersion }}
+          MajorMinorPatch: ${{ steps.version.outputs.MajorMinorPatch }}
+          AssemblySemVer: ${{ steps.version.outputs.AssemblySemVer }}
+          AssemblySemFileVer: ${{ steps.version.outputs.AssemblySemFileVer }}
+          InformationalVersion: ${{ steps.version.outputs.InformationalVersion }}
+
+      - name: Report test results
+        uses: EnricoMi/publish-unit-test-result-action@v2.3.0
+        if: always()
+        with:
+          nunit_files: "**/TestResults/TestResults.xml"
+
+      - name: Prepare build output for installation tarball
+        run: docker/scripts/create-installation-tarball.sh ${DbVersion}
+        env:
+          DbVersion: ${{ matrix.dbversion }}
+
+      # TODO: Test if this is still needed with upload-artifact@v4. Does GitHub now create tarballs that preserve Unix file permissions? If so, we can eliminate this step.
+      - name: Compress tarball images for faster uploads
+        run: time (tar cf - tarball | gzip -c > tarball.tar.gz)
+
+      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        with:
+          name: lfmerge-tarball-${{ matrix.dbversion }}
+          path: tarball.tar.gz
+          compression-level: 0 # Already compressed in previous step. If we ever remove the previous step, we should remove this line also
+    outputs:
+      MsBuildVersion: ${{ steps.output_version_number.outputs.VersionFor7000072 }}
+      TagFor7000072: ${{ steps.output_version_number.outputs.TagFor7000072 }}
+
+  release:
+    needs: build
+    uses: ./.github/workflows/release.yml
+    with:
+      MsBuildVersion: ${{ needs.build.outputs.MsBuildVersion }}
+      TagFor7000072: ${{ needs.build.outputs.TagFor7000072 }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0
       with:
         fetch-depth: 0
 
@@ -50,7 +50,7 @@ jobs:
         echo "DockerTags=${TAGS}" >> $GITHUB_OUTPUT
 
     - name: Download build artifacts
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
       with:
         # No name specified, so will download all artifacts
         path: all-tarballs
@@ -89,3 +89,18 @@ jobs:
     - name: List Docker images to verify build
       run: docker image ls
 
+    # During development, we'll want the Docker image as a build artifact
+    - name: Save Docker image to tarball
+      if: (!(github.event_name == 'push' && (github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master')))
+      run: docker image save "${TAG}" -o "lfmerge-${MS_BUILD_VERSION}.tar"
+      env:
+        MS_BUILD_VERSION: ${{ inputs.MsBuildVersion }}
+        TAG: ghcr.io/sillsdev/lfmerge:${{ inputs.MsBuildVersion }}
+
+    - name: Upload Docker image as build artifact
+      if: (!(github.event_name == 'push' && (github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master')))
+      uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+      with:
+        name: lfmerge-docker-${{ inputs.MsBuildVersion }}
+        path: lfmerge-${{ inputs.MsBuildVersion }}.tar
+        compression-level: 6
diff --git a/Dockerfile b/Dockerfile
@@ -30,7 +30,6 @@ RUN mkdir -p /usr/lib/lfmerge/${DbVersion} /var/lib/languageforge/lexicon/sendre
 	&& cd /var/lib/languageforge/lexicon/sendreceive/ && mkdir Templates state editqueue syncqueue webwork && cd - \
 	&& chown -R builder:users /var/lib/languageforge/lexicon/sendreceive
 
-USER builder
 WORKDIR /home/builder/repo
 # Git repo should be mounted under /home/builder/repo when run
 # E.g., `docker run --mount type=bind,source="$(pwd)",target=/home/builder/repo

diff --git a/Dockerfile.finalresult b/Dockerfile.finalresult
@@ -1,7 +1,13 @@
 # syntax=docker/dockerfile:experimental
 ARG DbVersion=7000072
 
-FROM ghcr.io/sillsdev/lfmerge-base:runtime
+FROM mcr.microsoft.com/dotnet/runtime:6.0 AS lfmerge-builder-base
+
+# Ensure fieldworks group exists in case lfmerge-fdo package didn't install it, and that www-data is part of that group
+# Also ensure that www-data has a .local dir in its home directory (/var/www) since some of lfmerge's dependencies assume that $HOME/.local exists
+RUN addgroup --system --quiet fieldworks ; \
+	adduser --quiet www-data fieldworks ; \
+	install -d -o www-data -g www-data -m 02775 /var/www/.local/share
 
 # install LFMerge prerequisites
 # tini - PID 1 handler
@@ -10,7 +16,7 @@ FROM ghcr.io/sillsdev/lfmerge-base:runtime
 # less - so we can read syslog during manual debugging of issues
 # vim-tiny - so we can edit state files (to change HOLD to IDLE) during manual debugging of issues
 RUN apt-get update \
-	&& apt-get install --yes --no-install-recommends tini python iputils-ping inotify-tools less vim-tiny \
+	&& apt-get install --yes --no-install-recommends tini python-is-python2 sudo iputils-ping inotify-tools less vim-tiny \
 	&& rm -rf /var/lib/apt/lists/*
 
 ADD tarball/lfmerge* /

diff --git a/docker/scripts/build-and-test.sh b/docker/scripts/build-and-test.sh
@@ -8,7 +8,7 @@ SCRIPT_DIR=$(dirname $(readlink -f "$0"))
 
 export DbVersion="${1-7000072}"
 echo "Building for ${DbVersion}"
-sudo mkdir -p /usr/lib/lfmerge/${DbVersion}
+[ -d "/usr/lib/lfmerge/${DbVersion}" ] || sudo mkdir -p /usr/lib/lfmerge/${DbVersion}
 
 echo Running as $(id)
 # Assuming script is being run from inside the repo, find the repo root and use that as the working directory from now on
@@ -25,7 +25,7 @@ dotnet restore -v:m
 echo "Compiling LfMerge"
 dotnet build --no-restore /v:m /property:Configuration=Release /property:DatabaseVersion=${DbVersion} LfMerge.sln
 
-if [ -n "$RUN_UNIT_TESTS" -a "$RUN_UNIT_TESTS" -ne 0 ]; then
+if [ -n "$RUN_UNIT_TESTS" ] && [ "$RUN_UNIT_TESTS" -ne 0 ]; then
 	echo "Running unit tests"
 	# dotnet test defaults to killing test processes after 100ms, which is way too short
 	export VSTEST_TESTHOST_SHUTDOWN_TIMEOUT=30000  # 30 seconds, please, since some of our tests can run very long

diff --git a/docker/scripts/get-version-number.sh b/docker/scripts/get-version-number.sh
@@ -73,6 +73,10 @@ fi
 GIT_SHA=${GITHUB_SHA:-$(git rev-parse ${REV})}
 TAG_SUFFIX="$(date +%Y%m%d)-${GIT_SHA}"
 export InformationalVersion="${MsBuildVersion}${INFO_SUFFIX}:${TAG_SUFFIX}"
+
+# If not running on GitHub Actions, GITHUB_OUTPUT is empty which would cause "ambiguous redirect" errors below
+GITHUB_OUTPUT=${GITHUB_OUTPUT:-/dev/stdout}
+
 echo "Calculated version number ${MsBuildVersion} for ${DbVersion}"
 echo "DebPackageVersion=${DebPackageVersion}" >> $GITHUB_OUTPUT
 echo "MsBuildVersion=${MsBuildVersion}" >> $GITHUB_OUTPUT

diff --git a/pbuild.sh b/pbuild.sh
@@ -37,47 +37,20 @@ FW9_BUILD_BRANCH="$(git name-rev --name-only HEAD)"
 echo Will build ONLY the FW9 build, from "${FW9_BUILD_BRANCH}"
 
 # Clean up any previous builds
-# This for loop includes all historical DbVersions even if BUILD_FW8 is 0
-for DbVersion in ${HISTORICAL_VERSIONS[@]} ${DBMODEL_VERSIONS[@]}; do
-	# Can safely ignore "container doesn't exist" as that's not an error
-	docker container kill tmp-lfmerge-build-${DbVersion} >/dev/null 2>/dev/null || true
-	docker container rm tmp-lfmerge-build-${DbVersion} >/dev/null 2>/dev/null || true
-done
-
-CURRENT_UID=$(id -u)
+dotnet clean LfMerge.sln
+[ -d tarball ] && rm -rf tarball
 
-# First create the base build container ONCE (it will be reused as a base by each DbVersion build), which should help with caching
-docker build -t ghcr.io/sillsdev/lfmerge-base:sdk -f Dockerfile.builder-base .
-docker build -t ghcr.io/sillsdev/lfmerge-base:runtime -f Dockerfile.runtime-base .
-docker build --build-arg "BUILDER_UID=${CURRENT_UID}" -t lfmerge-builder-base --target lfmerge-builder-base .
-
-# Create the build images for each DbVersion
-for DbVersion in ${DBMODEL_VERSIONS[@]}; do
-	docker build --build-arg DbVersion=${DbVersion} --build-arg "BUILDER_UID=${CURRENT_UID}" -t lfmerge-build-${DbVersion} .
-done
+# Create prerequisite directories that LfMerge expects in the unit tests
+# TODO: Probably not needed now?
+# mkdir -p ${HOME}/.nuget/packages
+# mkdir -p output/{Debug,Release}/net8.0
+# mkdir -p src/{FixFwData,LfMergeAuxTool,LfMerge.Core,LfMerge.Core.Tests,LfMerge,LfMergeQueueManager,LfMerge.Tests}/obj/{Debug,Release}/net8.0
 
 . docker/scripts/get-version-number.sh
 
-# Clean out previous installation files if they exist
-[ -d tarball ] && rm -rf tarball
-
-mkdir -p ${HOME}/.nuget/packages 
-
-# Run the build
+# Run build once for each DbVersion
 for DbVersion in ${DBMODEL_VERSIONS[@]}; do
-	docker run -it \
-		--mount type=bind,source="$(pwd)",target=/home/builder/repo \
-		--mount type=bind,src="${HOME}/.nuget/packages",dst=/home/builder/.nuget/packages \
-		--mount type=tmpfs,dst=/tmp \
-		--env "BUILD_NUMBER=999" \
-		--env "DebPackageVersion=${DebPackageVersion}" \
-		--env "Version=${MsBuildVersion}" \
-		--env "MajorMinorPatch=${MajorMinorPatch}" \
-		--env "AssemblyVersion=${AssemblySemVer}" \
-		--env "FileVersion=${AssemblySemFileVer}" \
-		--env "InformationalVersion=${InformationalVersion}" \
-		--name tmp-lfmerge-build-${DbVersion} \
-		lfmerge-build-${DbVersion}
+	docker/scripts/build-and-test.sh ${DbVersion}
 done
 
 time docker build -t ghcr.io/sillsdev/lfmerge -f Dockerfile.finalresult .