feat: control loop for model state monitoring (#1)

Because

- monitor the states of model resources continuously 

This commit

- add production/development dockerfile
- implement control loop
- add service methods between model-backend according to protobufs

.github/workflows/images.yml

@@ -0,0 +1,69 @@
+name: Build and Push Images
+
+on:
+  push:
+    branches:
+      - main
+  release:
+    types: [published]
+
+jobs:
+  docker-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.botGitHubToken }}
+
+      - name: Load .env file
+        uses: cardinalby/export-env-action@v2
+        with:
+          envFile: .env
+          expand: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: dropletbot
+          password: ${{ secrets.botDockerHubPassword }}
+
+      - name: Build and push (latest)
+        uses: docker/build-push-action@v3
+        with:
+          platforms: linux/amd64,linux/arm64
+          context: .
+          push: true
+          build-args: |
+            SERVICE_NAME=controller
+            GOLANG_VERSION=${{ env.GOLANG_VERSION }}
+          tags: instill/controller:latest
+          cache-from: type=registry,ref=instill/controller:buildcache
+          cache-to: type=registry,ref=instill/controller:buildcache,mode=max
+
+      - name: Set Versions
+        if: github.event_name == 'release'
+        uses: actions/github-script@v6
+        id: set_version
+        with:
+          script: |
+            const tag = '${{ github.ref_name }}'
+            const no_v_tag = tag.replace('v', '')
+            core.setOutput('tag', tag)
+            core.setOutput('no_v_tag', no_v_tag)
+
+      - name: Build and push (release)
+        if: github.event_name == 'release'
+        uses: docker/build-push-action@v3
+        with:
+          platforms: linux/amd64,linux/arm64
+          context: .
+          push: true
+          build-args: |
+            SERVICE_NAME=controller
+            GOLANG_VERSION=${{ env.GOLANG_VERSION }}
+          tags: instill/controller:${{steps.set_version.outputs.no_v_tag}}
+          cache-from: type=registry,ref=instill/controller:buildcache
+          cache-to: type=registry,ref=instill/controller:buildcache,mode=max

.github/workflows/releases.yml

@@ -0,0 +1,45 @@
+name: Release Please
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  release-please:
+    runs-on: ubuntu-latest
+    outputs:
+      RELEASE_CREATED: ${{ steps.release.outputs.release_created }}
+      TAG_NAME: ${{ steps.release.outputs.tag_name }}
+      SHA: ${{ steps.release.outputs.sha }}
+    steps:
+      - uses: google-github-actions/release-please-action@v3
+        id: release
+        with:
+          token: ${{ secrets.botGitHubToken }}
+          release-type: go
+          command: manifest
+          config-file: release-please/config.json
+          manifest-file: release-please/manifest.json
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.botGitHubToken }}
+      - name: Import GPG Key
+        uses: crazy-max/ghaction-import-gpg@v4
+        with:
+          gpg_private_key: ${{ secrets.botGPGPrivateKey }}
+          passphrase: ${{ secrets.botGPGPassphrase }}
+          git_user_signingkey: true
+          git_commit_gpgsign: true
+          git_tag_gpgsign: true
+      - name: Tag major and minor versions
+        if: ${{ steps.release.outputs.release_created }}
+        run: |
+          git tag -d v${{ steps.release.outputs.major }} || true
+          git tag -d v${{ steps.release.outputs.major }}.${{ steps.release.outputs.minor }} || true
+          git push origin :v${{ steps.release.outputs.major }} || true
+          git push origin :v${{ steps.release.outputs.major }}.${{ steps.release.outputs.minor }} || true
+          git tag -a v${{ steps.release.outputs.major }} -m "Release v${{ steps.release.outputs.major }} pointing to tag ${{ steps.release.outputs.tag_name }}"
+          git tag -a v${{ steps.release.outputs.major }}.${{ steps.release.outputs.minor }} -m "Release v${{ steps.release.outputs.major }}.${{ steps.release.outputs.minor }} pointing to tag ${{ steps.release.outputs.tag_name }}"
+          git push origin v${{ steps.release.outputs.major }}
+          git push origin v${{ steps.release.outputs.major }}.${{ steps.release.outputs.minor }}

.pre-commit-config.yaml

@@ -5,91 +5,6 @@ repos:
       - id: check-yaml
       - id: end-of-file-fixer
       - id: trailing-whitespace
-  - repo: https://github.com/tekwizely/pre-commit-golang
-    rev: master
-    hooks:
-    #
-    # Go Build
-    #
-    -   id: go-build-mod
-    -   id: go-build-pkg
-    -   id: go-build-repo-mod
-    -   id: go-build-repo-pkg
-    #
-    # Go Mod Tidy
-    #
-    -   id: go-mod-tidy
-    -   id: go-mod-tidy-repo
-    #
-    # Go Test
-    #
-    -   id: go-test-mod
-    -   id: go-test-pkg
-    -   id: go-test-repo-mod
-    -   id: go-test-repo-pkg
-    #
-    # Go Vet
-    #
-    -   id: go-vet
-    -   id: go-vet-mod
-    -   id: go-vet-pkg
-    -   id: go-vet-repo-mod
-    -   id: go-vet-repo-pkg
-    #
-    # Revive
-    #
-    -   id: go-revive
-    -   id: go-revive-mod
-    -   id: go-revive-repo-mod
-    #
-    # GoSec
-    #
-    -   id: go-sec-mod
-    -   id: go-sec-pkg
-    -   id: go-sec-repo-mod
-    -   id: go-sec-repo-pkg
-    #
-    # StaticCheck
-    #
-    -   id: go-staticcheck-mod
-    -   id: go-staticcheck-pkg
-    -   id: go-staticcheck-repo-mod
-    -   id: go-staticcheck-repo-pkg
-    #
-    # StructSlop
-    #
-    -   id: go-structslop-mod
-    -   id: go-structslop-pkg
-    -   id: go-structslop-repo-mod
-    -   id: go-structslop-repo-pkg
-    #
-    # Formatters
-    #
-    -   id: go-fmt
-    -   id: go-fmt-repo
-    -   id: go-fumpt        # replaces go-fmt
-    -   id: go-fumpt-repo   # replaces go-fmt-repo
-    -   id: go-imports      # replaces go-fmt
-    -   id: go-imports-repo # replaces go-fmt-repo
-    -   id: go-returns      # replaces go-imports & go-fmt
-    -   id: go-returns-repo # replaces go-imports-repo & go-fmt-repo
-    #
-    # Style Checkers
-    #
-    -   id: go-lint
-    -   id: go-critic
-    #
-    # GolangCI-Lint
-    # - Fast Multi-Linter
-    # - Can be configured to replace MOST other hooks
-    # - Supports repo config file for configuration
-    # - https://github.com/golangci/golangci-lint
-    #
-    -   id: golangci-lint
-    -   id: golangci-lint-mod
-    -   id: golangci-lint-pkg
-    -   id: golangci-lint-repo-mod
-    -   id: golangci-lint-repo-pkg
   - repo: https://github.com/pinglin/conventional-pre-commit
     rev: v1.1.0
     hooks:

Dockerfile

@@ -0,0 +1,43 @@
+ARG GOLANG_VERSION
+ARG UBUNTU_VERSION
+
+FROM --platform=$BUILDPLATFORM golang:${GOLANG_VERSION} AS build
+
+ARG SERVICE_NAME
+
+WORKDIR /src
+
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+
+ARG TARGETOS TARGETARCH
+RUN --mount=target=. --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /${SERVICE_NAME} ./cmd/main
+
+# Mounting points
+RUN mkdir /etc/vdp
+RUN mkdir /vdp
+
+FROM --platform=$BUILDPLATFORM ubuntu:${UBUNTU_VERSION}
+
+# Need permission of /tmp folder for internal process such as store temporary files.
+RUN chown -R nobody:nogroup /tmp
+# Need permission of /nonexistent folder for HuggingFace internal process.
+RUN mkdir /nonexistent > /dev/null && chown -R nobody:nogroup /nonexistent
+
+USER nobody:nogroup
+
+ARG SERVICE_NAME
+
+WORKDIR /${SERVICE_NAME}
+
+COPY --from=docker:dind-rootless --chown=nobody:nogroup /usr/local/bin/docker /usr/local/bin
+
+COPY --from=build --chown=nobody:nogroup /src/config ./config
+COPY --from=build --chown=nobody:nogroup /src/assets ./assets
+COPY --from=build --chown=nobody:nogroup /src/release-please ./release-please
+
+COPY --from=build --chown=nobody:nogroup /${SERVICE_NAME} ./
+
+COPY --from=build --chown=nobody:nogroup /etc/vdp /etc/vdp
+COPY --from=build --chown=nobody:nogroup /vdp /vdp

Dockerfile.dev

@@ -24,6 +24,7 @@ RUN tar -xf k6-v${K6_VERSION}-linux-$TARGETARCH.tar.gz --strip-components 1 -C /
 
 COPY . .
 RUN go mod download
+RUN chown -R nobody:nogroup /${SERVICE_NAME}
 
 RUN chown -R nobody:nogroup /go
 ENV GOCACHE /go/.cache/go-build

cmd/main/main.go

@@ -10,6 +10,7 @@ import (
 	"regexp"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"github.com/rs/cors"
@@ -66,6 +67,7 @@ func main() {
 	}()
 	grpc_zap.ReplaceGrpcLoggerV2(logger)
 
+	// TODO: remove if unnecessary
 	db := database.GetConnection()
 	defer database.Close(db)
 
@@ -117,6 +119,9 @@ func main() {
 	pipelinePublicServiceClient, pipelinePublicServiceClientConn := external.InitPipelinePublicServiceClient()
 	defer pipelinePublicServiceClientConn.Close()
 
+	modelPublicServiceClient, modelPublicServiceClientConn := external.InitModelPublicServiceClient()
+	defer modelPublicServiceClientConn.Close()
+
 	modelPrivateServiceClient, modelPrivateServiceClientConn := external.InitModelPrivateServiceClient()
 	defer modelPrivateServiceClientConn.Close()
 
@@ -131,6 +136,7 @@ func main() {
 			service.NewService(
 				*etcdClient,
 				modelPrivateServiceClient,
+				modelPublicServiceClient,
 				mgmtPublicServiceClient,
 				pipelinePublicServiceClient,
 				connectorPublicServiceClient,
@@ -185,6 +191,35 @@ func main() {
 	}
 	logger.Info("gRPC server is running.")
 
+	go func() {
+		service := service.NewService(
+			*etcdClient,
+			modelPrivateServiceClient,
+			modelPublicServiceClient,
+			mgmtPublicServiceClient,
+			pipelinePublicServiceClient,
+			connectorPublicServiceClient,
+		)
+		logger.Info("[controller] control loop started")
+		for {
+			logger.Info("[Controller] --------------Start probing------------")
+			// Backend services
+			err := service.ProbeBackend(config.Config.ModelBackend.Host)
+			if err != nil {
+				logger.Error(err.Error())
+			}
+
+			// Models
+			err = service.ProbeModels()
+
+			if err != nil {
+				logger.Error(err.Error())
+			}
+
+			time.Sleep(config.Config.Server.LoopInterval * time.Second)
+		}
+	}()
+
 	// kill (no param) default send syscall.SIGTERM
 	// kill -2 is syscall.SIGINT
 	// kill -9 is syscall.SIGKILL but can't be catch, so don't need add it

config/config.go

@@ -37,10 +37,12 @@ type ServerConfig struct {
 		Cert string `koanf:"cert"`
 		Key  string `koanf:"key"`
 	}
-	CORSOrigins  []string `koanf:"corsorigins"`
-	Edition      string   `koanf:"edition"`
-	DisableUsage bool     `koanf:"disableusage"`
-	Debug        bool     `koanf:"debug"`
+	CORSOrigins  []string      `koanf:"corsorigins"`
+	Edition      string        `koanf:"edition"`
+	DisableUsage bool          `koanf:"disableusage"`
+	Debug        bool          `koanf:"debug"`
+	LoopInterval time.Duration `koanf:"loopinterval"`
+	Timeout      time.Duration `koanf:"timeout"`
 }
 
 // EtcdConfig
@@ -91,9 +93,10 @@ type ConnectorBackendConfig struct {
 
 // ModelBackendConfig related to model-backend
 type ModelBackendConfig struct {
-	Host  string `koanf:"host"`
-	Port  int    `koanf:"port"`
-	HTTPS struct {
+	Host        string `koanf:"host"`
+	PublicPort  int    `koanf:"publicport"`
+	PrivatePort int    `koanf:"privateport"`
+	HTTPS       struct {
 		Cert string `koanf:"cert"`
 		Key  string `koanf:"key"`
 	}

config/config.yaml

@@ -6,11 +6,13 @@ server:
   corsorigins:
   edition: local-ce:dev
   disableusage: false
+  loopinterval: 5
+  timeout: 120
   debug: true
 etcd:
   host: etcd
   port: 2379
-  timeout: 30
+  timeout: 10
 database:
   username: postgres
   password: password
@@ -41,7 +43,8 @@ connectorbackend:
     key:
 modelbackend:
   host: model-backend
-  port: 8083
+  publicport: 8083
+  privateport: 3083
   https:
     cert:
     key: