CI: Use only free runners

Instead of running on big agents, split up the work into multiple pipelines:
- Basic tests
- Other tests
- Long tests

This allows fanning out the work to a swarm of agents. Also, it should lead to less failures due to less parallelism

.github/workflows/acc-tests.yml

@@ -43,22 +43,25 @@ jobs:
         # OSS tests, run on all versions
         version: ['10.2.0', '10.1.5', '9.5.13', '8.5.27']
         type: ['oss']
-        runner: ['ubuntu-latest-16-cores']
+        subset: ['basic', 'other', 'long']
         include:
           # TLS proxy tests, run only on latest version
           - version: '10.2.0'
             type: 'tls'
-            runner: 'ubuntu-latest' # Smaller instance
+            subset: 'basic'
           # Sub-path tests. Runs tests on localhost:3000/grafana/
           - version: '10.2.0'
             type: 'subpath'
-            runner: 'ubuntu-latest-16-cores'
+            subset: 'basic'
+          - version: '10.2.0'
+            type: 'subpath'
+            subset: 'other'
           # Enterprise tests, run only on latest version
           - version: '10.2.0'
             type: 'enterprise'
-            runner: 'ubuntu-latest' # Smaller instance
-    name: ${{ matrix.version }} - ${{ matrix.type }}
-    runs-on: ${{ matrix.runner }}
+            subset: 'all'
+    name: ${{ matrix.version }} - ${{ matrix.type }} - ${{ matrix.subset }}
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v4
@@ -74,7 +77,15 @@ jobs:
         with:
           repo_secrets: |
             GF_ENTERPRISE_LICENSE_TEXT=enterprise:license
+      - name: Cache Docker image
+        uses: ScribeMD/docker-cache@0.3.6
+        with:
+          key: docker-${{ runner.os }}-${{ matrix.type == 'enterprise' && 'enterprise' || 'oss' }}
       - run: make testacc-${{ matrix.type }}-docker
         env:
           GRAFANA_VERSION: ${{ matrix.version }}
-          TESTARGS: ${{ matrix.type == 'tls' && '-run ".*_basic"' || '' }} # Run subset of tests for TLS proxy, it's slower
+          TESTARGS: >- 
+            ${{ matrix.subset == 'all' && '-parallel 2' || '' }}
+            ${{ matrix.subset == 'basic' && '-run=".*_basic" -short -parallel 2' || '' }}
+            ${{ matrix.subset == 'other' && '-skip=".*_basic" -short -parallel 2' || '' }}
+            ${{ matrix.subset == 'long' && '-run=".*longtest" -parallel 1' || '' }}

GNUmakefile

@@ -1,7 +1,7 @@
 GRAFANA_VERSION ?= 10.1.5
 
 testacc:
-	TF_ACC=1 go test ./... -v $(TESTARGS) -timeout 120m -parallel 4
+	TF_ACC=1 go test ./... -v $(TESTARGS) -timeout 120m
 
 # Test OSS features
 testacc-oss:

internal/resources/grafana/resource_organization_test.go

@@ -198,7 +198,10 @@ func TestAccOrganization_roleNoneUser(t *testing.T) {
 	})
 }
 
-func TestAccOrganization_createManyUsers(t *testing.T) {
+func TestAccOrganization_createManyUsers_longtest(t *testing.T) {
+	if testing.Short() { // Also named "longtest" to allow targeting with -run=.*longtest
+		t.Skip("skipping test in short mode")
+	}
 	testutils.CheckOSSTestsEnabled(t)
 
 	var org gapi.Org

internal/resources/grafana/resource_service_account_test.go

@@ -86,7 +86,10 @@ func TestAccServiceAccount_NoneRole(t *testing.T) {
 	})
 }
 
-func TestAccServiceAccount_many(t *testing.T) {
+func TestAccServiceAccount_many_longtest(t *testing.T) {
+	if testing.Short() { // Also named "longtest" to allow targeting with -run=.*longtest
+		t.Skip("skipping test in short mode")
+	}
 	testutils.CheckOSSTestsEnabled(t, ">=9.1.0")
 
 	name := acctest.RandString(10)