Add acme clients internal data structures and adjust tests

builtin/logical/pki/acme_billing_test.go

@@ -131,10 +131,10 @@ func validateClientCount(t *testing.T, client *api.Client, mount string, expecte
 
 	require.NotNil(t, resp)
 	require.NotNil(t, resp.Data)
-	require.Contains(t, resp.Data, "non_entity_clients")
+	require.Contains(t, resp.Data, "acme_clients")
 	require.Contains(t, resp.Data, "months")
 
-	rawCount := resp.Data["non_entity_clients"].(json.Number)
+	rawCount := resp.Data["acme_clients"].(json.Number)
 	count, err := rawCount.Int64()
 	require.NoError(t, err, "failed to parse number as int64: "+rawCount.String())
 
@@ -158,8 +158,8 @@ func validateClientCount(t *testing.T, client *api.Client, mount string, expecte
 	// Validate this month's aggregate counts match the overall value.
 	require.Contains(t, monthlyInfo, "counts", "expected monthly info to contain a count key")
 	monthlyCounts := monthlyInfo["counts"].(map[string]interface{})
-	require.Contains(t, monthlyCounts, "non_entity_clients", "expected month[0].counts to contain a non_entity_clients key")
-	monthlyCountNonEntityRaw := monthlyCounts["non_entity_clients"].(json.Number)
+	require.Contains(t, monthlyCounts, "acme_clients", "expected month[0].counts to contain a non_entity_clients key")
+	monthlyCountNonEntityRaw := monthlyCounts["acme_clients"].(json.Number)
 	monthlyCountNonEntity, err := monthlyCountNonEntityRaw.Int64()
 	require.NoError(t, err, "failed to parse number as int64: "+monthlyCountNonEntityRaw.String())
 	require.Equal(t, count, monthlyCountNonEntity, "expected equal values for non entity client counts")
@@ -194,8 +194,8 @@ func validateClientCount(t *testing.T, client *api.Client, mount string, expecte
 		// This namespace must have a non-empty aggregate non-entity count.
 		require.Contains(t, namespace, "counts", "expected monthly.namespaces[%v] to contain a counts key", index)
 		namespaceCounts := namespace["counts"].(map[string]interface{})
-		require.Contains(t, namespaceCounts, "non_entity_clients", "expected namespace counts to contain a non_entity_clients key")
-		namespaceCountNonEntityRaw := namespaceCounts["non_entity_clients"].(json.Number)
+		require.Contains(t, namespaceCounts, "acme_clients", "expected namespace counts to contain a non_entity_clients key")
+		namespaceCountNonEntityRaw := namespaceCounts["acme_clients"].(json.Number)
 		namespaceCountNonEntity, err := namespaceCountNonEntityRaw.Int64()
 		require.NoError(t, err, "failed to parse number as int64: "+namespaceCountNonEntityRaw.String())
 		require.Greater(t, namespaceCountNonEntity, int64(0), "expected at least one non-entity client count value in the namespace")
@@ -217,8 +217,8 @@ func validateClientCount(t *testing.T, client *api.Client, mount string, expecte
 			// This mount must also have a non-empty non-entity client count.
 			require.Contains(t, mountInfo, "counts", "expected monthly.namespaces[%v].mounts[%v] to contain a counts key", index, mountIndex)
 			mountCounts := mountInfo["counts"].(map[string]interface{})
-			require.Contains(t, mountCounts, "non_entity_clients", "expected mount counts to contain a non_entity_clients key")
-			mountCountNonEntityRaw := mountCounts["non_entity_clients"].(json.Number)
+			require.Contains(t, mountCounts, "acme_clients", "expected mount counts to contain a non_entity_clients key")
+			mountCountNonEntityRaw := mountCounts["acme_clients"].(json.Number)
 			mountCountNonEntity, err := mountCountNonEntityRaw.Int64()
 			require.NoError(t, err, "failed to parse number as int64: "+mountCountNonEntityRaw.String())
 			require.Greater(t, mountCountNonEntity, int64(0), "expected at least one non-entity client count value in the mount")

changelog/26020.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+core/activity: Include ACME clients in activity log responses
+```

vault/activity/query.go

@@ -24,17 +24,19 @@ type NamespaceRecord struct {
 	NonEntityTokens uint64         `json:"non_entity_tokens"`
 	SecretSyncs     uint64         `json:"secret_syncs"`
 	Mounts          []*MountRecord `json:"mounts"`
+	ACMEClients     uint64         `json:"acme_clients"`
 }
 
 type CountsRecord struct {
 	EntityClients    int `json:"entity_clients"`
 	NonEntityClients int `json:"non_entity_clients"`
 	SecretSyncs      int `json:"secret_syncs"`
+	ACMEClients      int `json:"acme_clients"`
 }
 
 // HasCounts returns true when any of the record's fields have a non-zero value
 func (c *CountsRecord) HasCounts() bool {
-	return c.EntityClients+c.NonEntityClients+c.SecretSyncs != 0
+	return c.EntityClients+c.NonEntityClients+c.SecretSyncs+c.ACMEClients != 0
 }
 
 type NewClientRecord struct {

vault/activity_log.go

@@ -87,6 +87,8 @@ const (
 	secretSyncActivityType     = "secret-sync"
 )
 
+var ActivityClientTypes = []string{nonEntityTokenActivityType, entityActivityType, secretSyncActivityType, ACMEActivityType}
+
 type segmentInfo struct {
 	startTimestamp       int64
 	currentClients       *activity.EntityActivityLog
@@ -1573,6 +1575,7 @@ type ResponseCounts struct {
 	NonEntityClients int `json:"non_entity_clients" mapstructure:"non_entity_clients"`
 	Clients          int `json:"clients"`
 	SecretSyncs      int `json:"secret_syncs" mapstructure:"secret_syncs"`
+	ACMEClients      int `json:"acme_clients" mapstructure:"acme_clients"`
 }
 
 // Add adds the new record's counts to the existing record
@@ -1585,6 +1588,7 @@ func (r *ResponseCounts) Add(newRecord *ResponseCounts) {
 	r.DistinctEntities += newRecord.DistinctEntities
 	r.NonEntityClients += newRecord.NonEntityClients
 	r.NonEntityTokens += newRecord.NonEntityTokens
+	r.ACMEClients += newRecord.ACMEClients
 	r.SecretSyncs += newRecord.SecretSyncs
 }
 
@@ -2035,6 +2039,7 @@ func (p *processCounts) toCountsRecord() *activity.CountsRecord {
 	return &activity.CountsRecord{
 		EntityClients:    p.countByType(entityActivityType),
 		NonEntityClients: p.countByType(nonEntityTokenActivityType),
+		ACMEClients:      p.countByType(ACMEActivityType),
 		SecretSyncs:      p.countByType(secretSyncActivityType),
 	}
 }
@@ -2045,25 +2050,14 @@ func (p *processCounts) toCountsRecord() *activity.CountsRecord {
 func (p *processCounts) countByType(typ string) int {
 	switch typ {
 	case nonEntityTokenActivityType:
-		return len(p.ClientsByType[nonEntityTokenActivityType]) + int(p.Tokens) + len(p.ClientsByType[ACMEActivityType])
+		return len(p.ClientsByType[nonEntityTokenActivityType]) + int(p.Tokens)
 	}
 	return len(p.ClientsByType[typ])
 }
 
 // clientsByType returns the set of client IDs with the given type.
 // ACME clients are included in the non entity results
 func (p *processCounts) clientsByType(typ string) clientIDSet {
-	switch typ {
-	case nonEntityTokenActivityType:
-		clients := make(clientIDSet)
-		for c := range p.ClientsByType[nonEntityTokenActivityType] {
-			clients[c] = struct{}{}
-		}
-		for c := range p.ClientsByType[ACMEActivityType] {
-			clients[c] = struct{}{}
-		}
-		return clients
-	}
 	return p.ClientsByType[typ]
 }
 
@@ -2792,6 +2786,7 @@ func (a *ActivityLog) partialMonthClientCount(ctx context.Context) (map[string]i
 	responseData["non_entity_clients"] = totalCounts.NonEntityClients
 	responseData["clients"] = totalCounts.Clients
 	responseData["secret_syncs"] = totalCounts.SecretSyncs
+	responseData["acme_clients"] = totalCounts.ACMEClients
 
 	// The partialMonthClientCount should not have more than one month worth of data.
 	// If it does, something has gone wrong and we should warn that the activity log data

vault/activity_log_test.go

@@ -4219,14 +4219,14 @@ func TestActivityLog_partialMonthClientCountWithMultipleMountPaths(t *testing.T)
 // TestActivityLog_processNewClients_delete ensures that the correct clients are deleted from a processNewClients struct
 func TestActivityLog_processNewClients_delete(t *testing.T) {
 	mount := "mount"
-	namespace := "namespace"
+	ns := "namespace"
 	clientID := "client-id"
 	run := func(t *testing.T, clientType string) {
 		t.Helper()
 		isNonEntity := clientType == nonEntityTokenActivityType || clientType == ACMEActivityType
 		record := &activity.EntityRecord{
 			MountAccessor: mount,
-			NamespaceID:   namespace,
+			NamespaceID:   ns,
 			ClientID:      clientID,
 			NonEntity:     isNonEntity,
 			ClientType:    clientType,
@@ -4235,17 +4235,17 @@ func TestActivityLog_processNewClients_delete(t *testing.T) {
 		newClients.add(record)
 
 		require.True(t, newClients.Counts.contains(record))
-		require.True(t, newClients.Namespaces[namespace].Counts.contains(record))
-		require.True(t, newClients.Namespaces[namespace].Mounts[mount].Counts.contains(record))
+		require.True(t, newClients.Namespaces[ns].Counts.contains(record))
+		require.True(t, newClients.Namespaces[ns].Mounts[mount].Counts.contains(record))
 
 		newClients.delete(record)
 
 		byNS := newClients.Namespaces
 		counts := newClients.Counts
 		for _, typ := range []string{nonEntityTokenActivityType, secretSyncActivityType, entityActivityType, ACMEActivityType} {
 			require.NotContains(t, counts.clientsByType(typ), clientID)
-			require.NotContains(t, byNS[namespace].Mounts[mount].Counts.clientsByType(typ), clientID)
-			require.NotContains(t, byNS[namespace].Counts.clientsByType(typ), clientID)
+			require.NotContains(t, byNS[ns].Mounts[mount].Counts.clientsByType(typ), clientID)
+			require.NotContains(t, byNS[ns].Counts.clientsByType(typ), clientID)
 		}
 	}
 	t.Run("entity", func(t *testing.T) {
@@ -4267,43 +4267,44 @@ func TestActivityLog_processNewClients_delete(t *testing.T) {
 func TestActivityLog_processClientRecord(t *testing.T) {
 	startTime := time.Now()
 	mount := "mount"
-	namespace := "namespace"
+	ns := "namespace"
 	clientID := "client-id"
+
 	run := func(t *testing.T, clientType string) {
 		t.Helper()
 		isNonEntity := clientType == nonEntityTokenActivityType || clientType == ACMEActivityType
 		record := &activity.EntityRecord{
 			MountAccessor: mount,
-			NamespaceID:   namespace,
+			NamespaceID:   ns,
 			ClientID:      clientID,
 			NonEntity:     isNonEntity,
 			ClientType:    clientType,
 		}
 		byNS := make(summaryByNamespace)
 		byMonth := make(summaryByMonth)
 		processClientRecord(record, byNS, byMonth, startTime)
-		require.Contains(t, byNS, namespace)
-		require.Contains(t, byNS[namespace].Mounts, mount)
+		require.Contains(t, byNS, ns)
+		require.Contains(t, byNS[ns].Mounts, mount)
 		monthIndex := timeutil.StartOfMonth(startTime).UTC().Unix()
 		require.Contains(t, byMonth, monthIndex)
 		require.Equal(t, byMonth[monthIndex].Namespaces, byNS)
 		require.Equal(t, byMonth[monthIndex].NewClients.Namespaces, byNS)
 
-		for _, typ := range []string{nonEntityTokenActivityType, secretSyncActivityType, entityActivityType} {
-			if clientType == typ || (clientType == ACMEActivityType && typ == nonEntityTokenActivityType) {
+		for _, typ := range ActivityClientTypes {
+			if clientType == typ {
 				require.Contains(t, byMonth[monthIndex].Counts.clientsByType(typ), clientID)
 				require.Contains(t, byMonth[monthIndex].NewClients.Counts.clientsByType(typ), clientID)
-				require.Contains(t, byNS[namespace].Mounts[mount].Counts.clientsByType(typ), clientID)
-				require.Contains(t, byNS[namespace].Counts.clientsByType(typ), clientID)
+				require.Contains(t, byNS[ns].Mounts[mount].Counts.clientsByType(typ), clientID)
+				require.Contains(t, byNS[ns].Counts.clientsByType(typ), clientID)
 			} else {
 				require.NotContains(t, byMonth[monthIndex].Counts.clientsByType(typ), clientID)
 				require.NotContains(t, byMonth[monthIndex].NewClients.Counts.clientsByType(typ), clientID)
-				require.NotContains(t, byNS[namespace].Mounts[mount].Counts.clientsByType(typ), clientID)
-				require.NotContains(t, byNS[namespace].Counts.clientsByType(typ), clientID)
-
+				require.NotContains(t, byNS[ns].Mounts[mount].Counts.clientsByType(typ), clientID)
+				require.NotContains(t, byNS[ns].Counts.clientsByType(typ), clientID)
 			}
 		}
 	}
+
 	t.Run("non entity", func(t *testing.T) {
 		run(t, nonEntityTokenActivityType)
 	})
@@ -4600,13 +4601,20 @@ func TestActivityLog_writePrecomputedQuery(t *testing.T) {
 		MountAccessor: "mnt-3",
 		ClientType:    secretSyncActivityType,
 	}
+	acmeClient := &activity.EntityRecord{
+		ClientID:      "id-4",
+		NamespaceID:   "ns-4",
+		MountAccessor: "mnt-4",
+		ClientType:    ACMEActivityType,
+	}
 
 	now := time.Now()
 
 	// add the 2 clients to the namespace and month summaries
 	processClientRecord(clientEntity, byNS, byMonth, now)
 	processClientRecord(clientNonEntity, byNS, byMonth, now)
 	processClientRecord(secretSync, byNS, byMonth, now)
+	processClientRecord(acmeClient, byNS, byMonth, now)
 
 	endTime := timeutil.EndOfMonth(now)
 	opts := pqOptions{
@@ -4624,8 +4632,8 @@ func TestActivityLog_writePrecomputedQuery(t *testing.T) {
 	require.Equal(t, now.UTC().Unix(), val.StartTime.UTC().Unix())
 	require.Equal(t, endTime.UTC().Unix(), val.EndTime.UTC().Unix())
 
-	// ns-1, ns-2, and ns-3 should both be present in the results
-	require.Len(t, val.Namespaces, 3)
+	// ns-1, ns-2, ns-3, ns-4 should all be present in the results
+	require.Len(t, val.Namespaces, 4)
 	require.Len(t, val.Months, 1)
 	resultByNS := make(map[string]*activity.NamespaceRecord)
 	for _, ns := range val.Namespaces {
@@ -4634,53 +4642,76 @@ func TestActivityLog_writePrecomputedQuery(t *testing.T) {
 	ns1 := resultByNS["ns-1"]
 	ns2 := resultByNS["ns-2"]
 	ns3 := resultByNS["ns-3"]
+	ns4 := resultByNS["ns-4"]
 
 	require.Equal(t, ns1.Entities, uint64(1))
 	require.Equal(t, ns1.NonEntityTokens, uint64(0))
 	require.Equal(t, ns1.SecretSyncs, uint64(0))
+	require.Equal(t, ns1.ACMEClients, uint64(0))
 	require.Equal(t, ns2.Entities, uint64(0))
 	require.Equal(t, ns2.NonEntityTokens, uint64(1))
 	require.Equal(t, ns2.SecretSyncs, uint64(0))
+	require.Equal(t, ns2.ACMEClients, uint64(0))
 	require.Equal(t, ns3.Entities, uint64(0))
 	require.Equal(t, ns3.NonEntityTokens, uint64(0))
 	require.Equal(t, ns3.SecretSyncs, uint64(1))
+	require.Equal(t, ns3.ACMEClients, uint64(0))
+	require.Equal(t, ns4.Entities, uint64(0))
+	require.Equal(t, ns4.NonEntityTokens, uint64(0))
+	require.Equal(t, ns4.SecretSyncs, uint64(0))
+	require.Equal(t, ns4.ACMEClients, uint64(1))
 
 	require.Len(t, ns1.Mounts, 1)
 	require.Len(t, ns2.Mounts, 1)
 	require.Len(t, ns3.Mounts, 1)
+	require.Len(t, ns4.Mounts, 1)
+
 	// ns-1 needs to have mnt-1
 	require.Contains(t, ns1.Mounts[0].MountPath, "mnt-1")
 	// ns-2 needs to have mnt-2
 	require.Contains(t, ns2.Mounts[0].MountPath, "mnt-2")
 	// ns-3 needs to have mnt-3
 	require.Contains(t, ns3.Mounts[0].MountPath, "mnt-3")
+	// ns-4 needs to have mnt-4
+	require.Contains(t, ns4.Mounts[0].MountPath, "mnt-4")
 
 	// ns1 only has an entity client
 	require.Equal(t, 1, ns1.Mounts[0].Counts.EntityClients)
 	require.Equal(t, 0, ns1.Mounts[0].Counts.NonEntityClients)
 	require.Equal(t, 0, ns1.Mounts[0].Counts.SecretSyncs)
+	require.Equal(t, 0, ns1.Mounts[0].Counts.ACMEClients)
 
 	// ns2 only has a non entity client
 	require.Equal(t, 0, ns2.Mounts[0].Counts.EntityClients)
 	require.Equal(t, 1, ns2.Mounts[0].Counts.NonEntityClients)
 	require.Equal(t, 0, ns2.Mounts[0].Counts.SecretSyncs)
+	require.Equal(t, 0, ns2.Mounts[0].Counts.ACMEClients)
 
 	// ns3 only has a secret sync association
 	require.Equal(t, 0, ns3.Mounts[0].Counts.EntityClients)
 	require.Equal(t, 0, ns3.Mounts[0].Counts.NonEntityClients)
 	require.Equal(t, 1, ns3.Mounts[0].Counts.SecretSyncs)
+	require.Equal(t, 0, ns3.Mounts[0].Counts.ACMEClients)
+
+	// ns4 only has an ACME client
+	require.Equal(t, 0, ns4.Mounts[0].Counts.EntityClients)
+	require.Equal(t, 0, ns4.Mounts[0].Counts.NonEntityClients)
+	require.Equal(t, 0, ns4.Mounts[0].Counts.SecretSyncs)
+	require.Equal(t, 1, ns4.Mounts[0].Counts.ACMEClients)
 
 	monthRecord := val.Months[0]
 	// there should only be one month present, since the clients were added with the same timestamp
 	require.Equal(t, monthRecord.Timestamp, timeutil.StartOfMonth(now).UTC().Unix())
 	require.Equal(t, 1, monthRecord.Counts.NonEntityClients)
 	require.Equal(t, 1, monthRecord.Counts.EntityClients)
 	require.Equal(t, 1, monthRecord.Counts.SecretSyncs)
-	require.Len(t, monthRecord.Namespaces, 3)
-	require.Len(t, monthRecord.NewClients.Namespaces, 3)
+	require.Equal(t, 1, monthRecord.Counts.ACMEClients)
+	require.Len(t, monthRecord.Namespaces, 4)
+	require.Len(t, monthRecord.NewClients.Namespaces, 4)
 	require.Equal(t, 1, monthRecord.NewClients.Counts.EntityClients)
 	require.Equal(t, 1, monthRecord.NewClients.Counts.NonEntityClients)
 	require.Equal(t, 1, monthRecord.NewClients.Counts.SecretSyncs)
+	require.Equal(t, 1, monthRecord.NewClients.Counts.ACMEClients)
 }
 
 type mockTimeNowClock struct {
@@ -4759,9 +4790,9 @@ func TestAddActivityToFragment(t *testing.T) {
 	a.SetEnable(true)
 
 	mount := "mount"
-	namespace := "root"
+	ns := "root"
 	id := "id1"
-	a.AddActivityToFragment(id, namespace, 0, entityActivityType, mount)
+	a.AddActivityToFragment(id, ns, 0, entityActivityType, mount)
 
 	testCases := []struct {
 		name         string
@@ -4810,13 +4841,14 @@ func TestAddActivityToFragment(t *testing.T) {
 			isNonEntity:  true,
 		},
 	}
+
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			a.fragmentLock.RLock()
 			numClientsBefore := len(a.fragment.Clients)
 			a.fragmentLock.RUnlock()
 
-			a.AddActivityToFragment(tc.id, namespace, 0, tc.activityType, mount)
+			a.AddActivityToFragment(tc.id, ns, 0, tc.activityType, mount)
 			a.fragmentLock.RLock()
 			defer a.fragmentLock.RUnlock()
 			numClientsAfter := len(a.fragment.Clients)
@@ -4830,7 +4862,7 @@ func TestAddActivityToFragment(t *testing.T) {
 			require.Contains(t, a.partialMonthClientTracker, tc.expectedID)
 			require.True(t, proto.Equal(&activity.EntityRecord{
 				ClientID:      tc.expectedID,
-				NamespaceID:   namespace,
+				NamespaceID:   ns,
 				Timestamp:     0,
 				NonEntity:     tc.isNonEntity,
 				MountAccessor: mount,