Unified Namespace

.github/workflows/publish.yml

@@ -165,7 +165,9 @@ jobs:
           - acs-service-setup
           - acs-visualiser
           - edge-helm-charts
-          - influxdb-sparkplug-ingester
+          - historian-sparkplug
+          - historian-uns
+          - uns-ingester-sparkplug
     permissions:
       contents: read
       packages: write

ReleaseNotes.md

@@ -8,6 +8,24 @@ chronological order.
 These changes have not been released yet, but are likely to appear in
 the next release.
 
+## v3.4.0
+
+### Unified Namespace & Historian
+This release of ACS enables a true Unified Namespace (UNS). The UNS is a
+single point of truth for all data collected by ACS in human-readable
+format. The UNS is "fed" by ingesters, which take channels of data (in
+this case, Sparkplug), and publishes the human-readable content to
+`UNS/v1`. In the future additional ingesters may be added to ACS.
+
+In addition to the Sparkplug ingester, this release features a UNS 
+historian, which persists the UNS data to the same InfluxDB 
+database used by the legacy Sparkplug historian. **By default, the 
+UNS historian is disabled** in an effort to minimise the impact of
+this change on existing installations. To enable the UNS historian,
+set the `historians.uns.enabled` environment variable to `true`. If 
+you only want to exclusively persist UNS data (and not legacy 
+Sparkplug data) then set `historians.sparkplug.enabled` to `false`.
+
 ## v3.1.0
 
 ### Administration interface

acs-manager/get-keytab.sh

@@ -4,4 +4,4 @@
 #
 
 # bin/bash
-kubectl --kubeconfig ./k3s.yaml get -n fpd-v3 secret manager-keytab -o jsonpath="{.data.client-keytab}" | base64 -d >"./keytab"
+kubectl --kubeconfig /Users/me1ago/.kube/ago.yaml get -n factory-plus secret manager-keytab -o jsonpath="{.data.client-keytab}" | base64 -d >"./keytab"

acs-service-setup/dumps/admin.yaml

@@ -49,3 +49,6 @@ aces:
   - principal: !u ACS.Group.GlobalDebuggers
     permission: !u UUIDs.Permission.CmdEsc.Rebirth
     target: !u UUIDs.Special.Null
+  - principal: !u ACS.Group.GlobalDebuggers
+    permission: !u ACS.Perm.MQTT.ReadEntireUNS
+    target: !u UUIDs.Special.Null

acs-service-setup/dumps/uns.yaml

@@ -0,0 +1,39 @@
+# This dump creates a service account for the Sparkplug Ingester service
+# and grants it the necessary permissions to read and write to the UNS.
+# It also adds the ReadEntireUNS permission to the Warehouse role so
+# that the UNS historian can use it and has permission to read the UNS.
+---
+service: !u UUIDs.Service.ConfigDB
+version: 1
+overwrite: true
+objects:
+  !u ACS.Class.Permission:
+    - !u ACS.Perm.MQTT.WriteToEntireUNS
+    - !u ACS.Perm.MQTT.ReadEntireUNS
+  !u ACS.Class.UserGroup:
+    - !u ACS.Group.SparkplugIngesters
+    - !u ACS.Group.HistorianUNS
+configs:
+  !u UUIDs.App.Info:
+    !u ACS.Perm.MQTT.WriteToEntireUNS: { name: "MQTT: Write to entire UNS" }
+    !u ACS.Perm.MQTT.ReadEntireUNS: { name: "MQTT: Read entire UNS" }
+    !u ACS.Group.SparkplugIngesters: { name: "Sparkplug Ingesters" }
+    !u ACS.Group.HistorianUNS: { name: "Historian UNS" }
+  !u UUIDs.App.MQTTPermissionTemplate:
+    !u ACS.Perm.MQTT.WriteToEntireUNS:
+      UNS/v1/#: w
+    !u ACS.Perm.MQTT.ReadEntireUNS:
+      UNS/v1/#: rs
+---
+service: !u UUIDs.Service.Authentication
+version: 1
+groups:
+  !u ACS.Group.GlobalDebuggers:
+    - !u ACS.Group.SparkplugIngesters
+aces:
+  - principal: !u ACS.Group.HistorianUNS
+    permission: !u ACS.Perm.MQTT.ReadEntireUNS
+    target: !u UUIDs.Special.Null
+  - principal: !u ACS.Group.SparkplugIngesters
+    permission: !u ACS.Perm.MQTT.WriteToEntireUNS
+    target: !u UUIDs.Special.Self

acs-service-setup/lib/uuids.js

@@ -8,6 +8,7 @@ export const ACS = {
         ClientRole:         "1c567e3c-5519-4418-8682-6086f22fbc13",
         EdgeAccount:        "97756c9a-38e6-4238-b78c-3df6f227a6c9",
         ServiceAccount:     "e463b4ae-a322-46cc-8976-4ba76838e908",
+        Permission:         "8ae784bb-c4b5-4995-9bf6-799b3c7f21ad",
         UserAccount:        "8b3e8f35-78e5-4f93-bf21-7238bcb2ba9d",
         UserGroup:          "f1fabdd1-de90-4399-b3da-ccf6c2b2c08b",
     },
@@ -22,6 +23,8 @@ export const ACS = {
         EdgeGroups:         "9ba0de4b-056f-4b5e-b966-2d5d85d07767",
         EdgePermissions:    "7594cd71-e5b9-4467-88c0-b11a66d47fec",
         CentralMonitor:     "1bc3dbca-68fe-48d2-9590-3a528c111827",
+        SparkplugIngesters: "e414d355-b991-429b-8f5d-97e823ff71f5",
+        HistorianUNS:       "03f5f08a-f61e-4134-8f66-b2951e3bbb69",
     },
     Perm: {
         MQTT: {
@@ -30,6 +33,8 @@ export const ACS = {
             ReadAllStates:      "8790cf3d-b793-423c-b373-8cfcf9f63529",
             ReadNode:           "046d6603-fa62-4208-9400-65d61f8b1ec4",
             ReadWholeNamespace: "81833dbb-1150-4078-b1db-978c646ba73e",
+            WriteToEntireUNS:   "9fa6ff20-9d2a-4444-960c-40ebcf56f5b4",
+            ReadEntireUNS:      "ffa40b36-3a61-4545-832a-2d1e8b860d63",
         },
     },
     PermGroup: {
@@ -52,9 +57,11 @@ export const ACS = {
     Service: {
         Manager:                "619eecab-742d-4824-8b97-bcae472e5c04",
     },
+    ServiceAccount: {},
     Role: {
         EdgeNodeConsumer:       "17a64293-b82d-4db4-af4d-63359bb62934",
         GlobalDebugger:         "4473fe9c-05b0-42cc-ad8c-8e05f6d0ca86",
+        Warehouse:              "6958c812-fbe2-4e6c-b997-6f850b89f679",
     },
     /* XXX This should not be fixed. Currently this matches the fixed
      * UUID deployed by the dumps in the ACS Helm chart. This needs

acs-service-setup/package.json

@@ -12,7 +12,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@amrc-factoryplus/utilities": "^1.3.1",
+    "@amrc-factoryplus/utilities": "^1.3.4",
     "eslint": "^8.55.0",
     "yaml": "^2.3.4"
   }

deploy/templates/auth/principals/service-clients.yaml

@@ -1,18 +1,18 @@
 ## Service client principals.
-  ##
-  ## These are the principals that services use when they are acting as a client of another
-  ## service. These are the principals that should be granted rights.
-  ##
-  ## Random type principals get a random key and a Kerberos keytab. They are the most
-  ## secure option but can only be used for services that can act as a Kerberos client.
-  ##
-  ## Password type principals get a random password generated by the system. This is next
-  ## most secure.
-  ##
-  ## PresetPassword type principals use a password set by the administrator in the
-  ## krb5-presets secret. This option should only be used for services that can't pull
-  ## their password from a k8s secret, like Canary.
-  #
+##
+## These are the principals that services use when they are acting as a client of another
+## service. These are the principals that should be granted rights.
+##
+## Random type principals get a random key and a Kerberos keytab. They are the most
+## secure option but can only be used for services that can act as a Kerberos client.
+##
+## Password type principals get a random password generated by the system. This is next
+## most secure.
+##
+## PresetPassword type principals use a password set by the administrator in the
+## krb5-presets secret. This option should only be used for services that can't pull
+## their password from a k8s secret, like Canary.
+#
 {{- if .Values.auth.enabled }}
 apiVersion: factoryplus.app.amrc.co.uk/v1
 kind: KerberosKey
@@ -80,7 +80,7 @@ spec:
   secret: manager-keytab/client-keytab
 {{- end }}
 ---
-{{- if .Values.warehouse.ingester.enabled }}
+{{- if .Values.historians.sparkplug.enabled }}
 apiVersion: factoryplus.app.amrc.co.uk/v1
 kind: KerberosKey
 metadata:
@@ -122,3 +122,41 @@ spec:
       group: {{ .Values.acs.organisation }}-Service-Core
       node: Monitor
 {{- end }}
+---
+{{- if .Values.unsIngesters.sparkplug.enabled }}
+apiVersion: factoryplus.app.amrc.co.uk/v1
+kind: KerberosKey
+metadata:
+  name: sv1sparkplugingester
+  namespace: {{ .Release.Namespace }}
+spec:
+  type: Random
+  principal: sv1sparkplugingester@{{ .Values.identity.realm | required "values.identity.realm is required!" }}
+  secret: uns-ingester-sparkplug-keytabs/client
+  account:
+    class: e463b4ae-a322-46cc-8976-4ba76838e908
+    name: Sparkplug Ingester
+    groups:
+      - e414d355-b991-429b-8f5d-97e823ff71f5
+
+{{- end }}
+---
+# In the future if we have any more ingesters then this should be
+# enabled if any of the ingesters are enabled.
+{{- if .Values.historians.uns.enabled }}
+apiVersion: factoryplus.app.amrc.co.uk/v1
+kind: KerberosKey
+metadata:
+  name: sv1historianuns
+  namespace: {{ .Release.Namespace }}
+spec:
+  type: Random
+  principal: sv1historianuns@{{ .Values.identity.realm | required "values.identity.realm is required!" }}
+  secret: historian-uns-keytabs/client
+  account:
+    class: e463b4ae-a322-46cc-8976-4ba76838e908
+    name: Historian UNS
+    groups:
+      - 03f5f08a-f61e-4134-8f66-b2951e3bbb69
+
+{{- end }}

deploy/templates/auth/principals/services.yaml

@@ -100,19 +100,6 @@ spec:
   principal: postgres/postgres.{{ .Release.Namespace }}.svc.cluster.local@{{ .Values.identity.realm | required "values.identity.realm is required!" }}
 {{- end }}
 ---
-{{- if .Values.warehouse.enabled }}
-apiVersion: factoryplus.app.amrc.co.uk/v1
-kind: KerberosKey
-metadata:
-  name: http.warehouse
-  namespace: {{ .Release.Namespace }}
-spec:
-  type: Random
-  principal: HTTP/warehouse.{{ .Release.Namespace }}.svc.cluster.local@{{ .Values.identity.realm | required "values.identity.realm is required!" }}
-  additionalPrincipals:
-    - HTTP/warehouse.{{.Values.acs.baseUrl | required "values.acs.baseUrl is required"}}@{{ .Values.identity.realm | required "values.identity.realm is required!" }}
-{{- end }}
----
 {{- if .Values.git.enabled }}
 apiVersion: factoryplus.app.amrc.co.uk/v1
 kind: KerberosKey

deploy/templates/warehouse/influxdb-ingester.yaml → deploy/templates/historians/sparkplug.yaml

@@ -1,21 +1,21 @@
-{{ if .Values.warehouse.ingester.enabled }}
+{{ if .Values.historians.sparkplug.enabled }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: influxdb-ingester
+  name: historian-sparkplug
   namespace: {{ .Release.Namespace }}
   labels:
-    component: influxdb-ingester
+    component: historian-sparkplug
 spec:
   replicas: 1
   selector:
     matchLabels:
-      component: influxdb-ingester
+      component: historian-sparkplug
   template:
     metadata:
       labels:
-        component: influxdb-ingester
-        factory-plus.service: influxdb-ingester
+        component: historian-sparkplug
+        factory-plus.service: historian-sparkplug
     spec:
       {{- with .Values.acs.imagePullSecrets }}
       imagePullSecrets:
@@ -33,8 +33,8 @@ spec:
                 path: client
 
       containers:
-        - name: influxdb-ingester
-          image: "{{ include "amrc-connectivity-stack.image-name" (list . .Values.warehouse.ingester) }}"
+        - name: historian-sparkplug
+          image: "{{ include "amrc-connectivity-stack.image-name" (list . .Values.historians.sparkplug) }}"
           command: [ "/usr/bin/k5start", "-Uf", "/keytabs/client" ]
           args: [ "node", "--es-module-specifier-resolution=node", "bin/ingester.js" ]
           imagePullPolicy: Always
@@ -44,11 +44,11 @@ spec:
             - name: CLIENT_KEYTAB
               value: /keytabs/client
             - name: LOG_LEVEL
-              value: {{ .Values.warehouse.ingester.logLevel | required "values.warehouse.ingester.logLevel" }}
+              value: {{ .Values.historians.sparkplug.logLevel | required "values.historians.sparkplug.logLevel" }}
             - name: BATCH_SIZE
-              value: {{ .Values.warehouse.ingester.batchSize | quote | required "values.warehouse.ingester.batchSize" }}
+              value: {{ .Values.historians.sparkplug.batchSize | quote | required "values.historians.sparkplug.batchSize" }}
             - name: FLUSH_INTERVAL
-              value: {{ .Values.warehouse.ingester.flushInterval | quote | required "values.warehouse.ingester.flushInterval" }}
+              value: {{ .Values.historians.sparkplug.flushInterval | quote | required "values.historians.sparkplug.flushInterval" }}
             - name: DIRECTORY_URL
               value: http://directory.{{ .Release.Namespace }}.svc.cluster.local
             - name: INFLUX_URL

deploy/templates/historians/uns.yaml

@@ -0,0 +1,69 @@
+# In the future if we have any more ingesters then this should be
+# enabled if any of the ingesters are enabled.
+{{- if .Values.unsIngesters.sparkplug.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: historian-uns
+  namespace: {{ .Release.Namespace }}
+  labels:
+    component: historian-uns
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      component: historian-uns
+  template:
+    metadata:
+      labels:
+        component: historian-uns
+        factory-plus.service: historian-uns
+    spec:
+      {{- with .Values.acs.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      volumes:
+        - name: krb5-conf
+          configMap:
+            name: krb5-conf
+        - name: keytabs
+          secret:
+            secretName: historian-uns-keytabs
+
+      containers:
+        - name: historian-uns
+          image: "{{ include "amrc-connectivity-stack.image-name" (list . .Values.historians.uns) }}"
+          command: [ "/usr/bin/k5start", "-Uf", "/keytabs/client" ]
+          args: [ "node", "--es-module-specifier-resolution=node", "app.js" ]
+          imagePullPolicy: Always
+          env:
+            - name: KRB5_CONFIG
+              value: /config/krb5-conf/krb5.conf
+            - name: CLIENT_KEYTAB
+              value: /keytabs/client
+            - name: LOG_LEVEL
+              value: {{ .Values.historians.uns.logLevel | required "values.historians.uns.logLevel" }}
+            - name: BATCH_SIZE
+              value: {{ .Values.historians.uns.batchSize | quote | required "values.historians.uns.batchSize" }}
+            - name: FLUSH_INTERVAL
+              value: {{ .Values.historians.uns.flushInterval | quote | required "values.historians.uns.flushInterval" }}
+            - name: DIRECTORY_URL
+              value: http://directory.{{ .Release.Namespace }}.svc.cluster.local
+            - name: INFLUX_URL
+              value: http://acs-influxdb2.{{ .Release.Namespace }}.svc.cluster.local
+            - name: INFLUX_ORG
+              value: default
+            - name: INFLUX_BUCKET
+              value: 'uns'
+            - name: INFLUX_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: influxdb-auth
+                  key: admin-token
+          volumeMounts:
+            - mountPath: /config/krb5-conf
+              name: krb5-conf
+            - mountPath: /keytabs
+              name: keytabs
+{{- end -}}

deploy/templates/warehouse/influx-ingress.yaml → deploy/templates/influxdb/influx-ingress.yaml

@@ -18,6 +18,6 @@ spec:
   tls:
     secretName: {{ coalesce .Values.influxdb2.tlsSecretName .Values.acs.tlsSecretName }}
     domains:
-      - main: influxdb-ingester.{{.Values.acs.baseUrl | required "values.acs.baseUrl is required"}}
+      - main: influxdb.{{.Values.acs.baseUrl | required "values.acs.baseUrl is required"}}
   {{- end -}}
 {{- end -}}