[Jasoet] Enable Support for Yaml Config

.env.test

@@ -1,4 +1,3 @@
-export ENVIRONMENT=test
 export PROCTOR_KUBE_CONFIG=out-of-cluster
 export PROCTOR_KUBE_CONTEXT=minikube
 export PROCTOR_LOG_LEVEL=debug
@@ -12,11 +11,7 @@ export PROCTOR_KUBE_SERVICE_ACCOUNT_NAME=default
 export PROCTOR_LOGS_STREAM_READ_BUFFER_SIZE=140
 export PROCTOR_LOGS_STREAM_WRITE_BUFFER_SIZE=4096
 export PROCTOR_KUBE_WAIT_FOR_RESOURCE_POLL_COUNT=5
-export PROCTOR_KUBE_CLUSTER_HOST_NAME=192.168.99.100:8443
-export PROCTOR_KUBE_POD_LIST_WAIT_TIME=60
 export PROCTOR_KUBE_LOG_PROCESS_WAIT_TIME=60
-export PROCTOR_KUBE_CA_CERT_ENCODED=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQVRBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwdGFXNXAKYTNWaVpVTkJNQjRYRFRFNU1EY3dPVEF6TkRZeE1Wb1hEVEk1TURjd056QXpORFl4TVZvd0ZURVRNQkVHQTFVRQpBeE1LYldsdWFXdDFZbVZEUVRDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTkNkCmpXRUhaTi8wanFRbmE5RVorVUFiT2ZoYzAyTUdQWW1Nd0VFUEZMSlNJQnhONEMzWTNEOFBENkVsZ3NjL1NjY04KYUNyZHBZbkhqdjZnQ2lCNmFtMVRjdkVacy81MHl2QlNjcjVldFdwakNLVmd6NnJBU2d1YUdoaU5obUU3a2xtWgpMQVc5c3R0c3F0N1ZrRjJ4VnBNOTA2Y2FiZ2RFdzBrcTJITWcydzJGRHJyaG1VZU9NUWpxSjBBUFQvc252bGMrCkRYRG9QTDh2aFVUakpsdzZ2OE5WMTlCSjZBYnJqMmxzSXJYbk5saEIwRnBldENZeFRmRXNQWnp3bmgwdEVjZk0KTGNNTFBqV3dLejg5cHJBTCt6Qjd5cWErUXprMUVWWEJlMVl3VnlpRXFiRHAvY2V4WGhPeHUyd25HemxxOXN4NApneWR4b3l2eEdnLzYxUVhjWm9FQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXCk1CUUdDQ3NHQVFVRkJ3TUNCZ2dyQmdFRkJRY0RBVEFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFDMVlZYWsrY2xHZVlPT21QOUpwWUcrQ3R1TTJXaFZQK0Rad2ZDR0tnNWQ1eWhONThTRwp3Q09seDVZMzAxQ3A1dG9NVmU1a3NISityMFQrb0pzMjQvVFlzcVBneTA2MS9Ja3ViYnRjZWova3VBVUdjbkRkCmFmaWg4L1N0d0N4Y1ZzR1ljR3NBRGZGclZYLzhaZm1GZWdwS1dHMWcxcGNpNVByUWxWR2dMdG0rd0lvcVJPWEQKaktuc1VhNWVGQzN3L0h5cmxRZzd6d29qcGMvaURWYzN6UzlxWXlqUUd5MFpEbWZOYloxNVJvS1M0YytHR2lrSQo4OVRKZHU0SDBkckZBU1RsMTlDQ29zcE1KRFdoRGhaWTU4OENPYjVNeUt1RjBDRVVuZ1hBSmxkaGdncXJocFR3CmkwT1phajBteE0xNmtrbFAwR0tyNW1jVmNnYkgzdTBqOGE4cgotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
-export PROCTOR_KUBE_BASIC_AUTH_ENCODED=YWRtaW46cGFzc3dvcmQK
 export PROCTOR_POSTGRES_USER=postgres
 export PROCTOR_POSTGRES_PASSWORD=
 export PROCTOR_POSTGRES_HOST=localhost
@@ -33,10 +28,10 @@ export PROCTOR_MAIL_PASSWORD=password
 export PROCTOR_MAIL_SERVER_HOST=smtp.mail.com
 export PROCTOR_MAIL_SERVER_PORT=123
 export PROCTOR_JOB_POD_ANNOTATIONS={\"key.one\":\"true\"}
-export PROCTOR_SENTRY_DSN=foo
 export PROCTOR_DOCS_PATH=/path/to/docs/dir
 export PROCTOR_AUTH_PLUGIN_BINARY=
 export PROCTOR_AUTH_PLUGIN_EXPORTED=GateAuth
 export PROCTOR_AUTH_ENABLED=false
 export PROCTOR_NOTIFICATION_PLUGIN_BINARY=
 export PROCTOR_NOTIFICATION_PLUGIN_EXPORTED=SlackNotification
+export PROCTOR_REQUIRED_ADMIN_GROUP=proctor_admin

Dockerfile

@@ -2,6 +2,7 @@ FROM golang:1.12 AS builder
 WORKDIR /go/src/app
 COPY . .
 RUN make plugin.auth
+RUN make plugin.slack
 RUN make server
 
 FROM ubuntu:latest
@@ -10,8 +11,8 @@ RUN apt-get install -y ca-certificates
 WORKDIR /app/
 COPY --from=builder /go/src/app/_output/bin/server .
 COPY --from=builder /go/src/app/_output/bin/plugin/auth.so .
+COPY --from=builder /go/src/app/_output/bin/plugin/slack.so .
 COPY --from=builder /go/src/app/migrations ./migrations
-ENV PROCTOR_AUTH_PLUGIN_BINARY=/app/auth.so
 
 ENTRYPOINT ["./server"]
 CMD ["s"]

Makefile

@@ -14,6 +14,7 @@ FTEST_DIR := test/procs
 CONFIG_DIR := test/config
 GOPROXY ?= https://proxy.golang.org
 GO111MODULE := on
+CONFIG_LOCATION := $(SRC_DIR)
 
 $(@info $(shell mkdir -p $(OUT_DIR) $(BIN_DIR) $(PLUGIN_DIR))
 

config.yaml

@@ -0,0 +1,51 @@
+kube:
+  config: out-of-cluster
+  context: minikube
+  job:
+    active.deadline.seconds: 60
+    retries: 0
+  service.account.name: default
+  wait.for.resource.poll.count: 5
+  log.process.wait.time: 60
+log.level: debug
+app.port: 5000
+default.namespace: default
+redis:
+  address: localhost:6379
+  max.active.connections: 10
+logs.stream:
+  read.buffer.size: 140
+  write.buffer.size: 4096
+postgres:
+  user: postgres
+  password:
+  host: localhost
+  port: 5432
+  database: proctord_test
+  max.connections: 50
+  connections.max.lifetime: 30
+new.relic:
+  app.name: proctor-service
+  licence.key:
+min.client.version: v2.0.0
+scheduled.jobs.fetch.interval.in.mins: 5
+mail:
+  username: user@mail.com
+  password: password
+  server:
+    host: smtp.mail.com
+    port: 123
+job.pod.annotations: "{\"key.one\":\"true\"}"
+docs.path: /path/to/docs/dir
+auth:
+  enabled: false
+  plugin:
+    binary:
+    exported: GateAuth
+  required.admin.group: proctor_admin
+notification.plugin:
+  binary:
+  exported: SlackNotification
+
+
+

internal/app/service/execution/service/execution_integration_test.go

@@ -98,7 +98,7 @@ func (suite *TestExecutionIntegrationSuite) TestStreamLogsSuccess() {
 	executedJobname, err := suite.kubernetesClient.ExecuteJobWithCommand(sampleImageName, envVarsForContainer, []string{"echo", "Bimo Horizon"})
 	assert.NoError(t, err)
 
-	waitTime := config.Config().KubePodsListWaitTime * time.Second
+	waitTime := config.Config().KubeLogProcessWaitTime * time.Second
 	logStream, err := suite.service.StreamJobLogs(executedJobname, waitTime)
 	assert.NoError(t, err)
 

internal/app/service/infra/config/config.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/spf13/viper"
+	"os"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -66,7 +67,6 @@ type ProctorConfig struct {
 	RedisMaxActiveConnections        int
 	LogsStreamWriteBufferSize        int
 	KubeWaitForResourcePollCount     int
-	KubePodsListWaitTime             time.Duration
 	KubeLogProcessWaitTime           time.Duration
 	KubeJobActiveDeadlineSeconds     *int64
 	KubeJobRetries                   *int32
@@ -80,15 +80,14 @@ type ProctorConfig struct {
 	PostgresMaxConnections           int
 	PostgresConnectionMaxLifetime    int
 	NewRelicAppName                  string
-	NewRelicLicenceKey               string
+	NewRelicLicenseKey               string
 	MinClientVersion                 string
 	ScheduledJobsFetchIntervalInMins int
 	MailUsername                     string
 	MailServerHost                   string
 	MailPassword                     string
 	MailServerPort                   string
 	JobPodAnnotations                map[string]string
-	SentryDSN                        string
 	DocsPath                         string
 	AuthPluginBinary                 string
 	AuthEnabled                      bool
@@ -97,51 +96,61 @@ type ProctorConfig struct {
 	AuthRequiredAdminGroup           []string
 }
 
-func Load() ProctorConfig {
+func load() ProctorConfig {
 	fang := viper.New()
-	fang.AutomaticEnv()
+
 	fang.SetEnvPrefix("PROCTOR")
+	fang.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
+	fang.AutomaticEnv()
+
+	fang.SetConfigName("config")
+	fang.AddConfigPath(".")
+	fang.AddConfigPath("$HOME/.proctor")
+	value, available := os.LookupEnv("CONFIG_LOCATION")
+	if available {
+		fang.AddConfigPath(value)
+	}
+	_ = fang.ReadInConfig()
+
 	proctorConfig := ProctorConfig{
 		viper:                            fang,
-		KubeConfig:                       fang.GetString("KUBE_CONFIG"),
-		KubeContext:                      GetStringDefault(fang, "KUBE_CONTEXT", "default"),
-		LogLevel:                         GetStringDefault(fang, "LOG_LEVEL", "DEBUG"),
-		AppPort:                          GetStringDefault(fang, "APP_PORT", "5001"),
-		DefaultNamespace:                 fang.GetString("DEFAULT_NAMESPACE"),
-		RedisAddress:                     fang.GetString("REDIS_ADDRESS"),
-		RedisMaxActiveConnections:        fang.GetInt("REDIS_MAX_ACTIVE_CONNECTIONS"),
-		LogsStreamReadBufferSize:         fang.GetInt("LOGS_STREAM_READ_BUFFER_SIZE"),
-		LogsStreamWriteBufferSize:        fang.GetInt("LOGS_STREAM_WRITE_BUFFER_SIZE"),
-		KubeWaitForResourcePollCount:     fang.GetInt("KUBE_WAIT_FOR_RESOURCE_POLL_COUNT"),
-		KubePodsListWaitTime:             time.Duration(fang.GetInt("KUBE_POD_LIST_WAIT_TIME")),
-		KubeLogProcessWaitTime:           time.Duration(fang.GetInt("KUBE_LOG_PROCESS_WAIT_TIME")),
-		KubeJobActiveDeadlineSeconds:     GetInt64Ref(fang, "KUBE_JOB_ACTIVE_DEADLINE_SECONDS"),
-		KubeJobRetries:                   GetInt32Ref(fang, "KUBE_JOB_RETRIES"),
-		KubeServiceAccountName:           fang.GetString("KUBE_SERVICE_ACCOUNT_NAME"),
-		PostgresUser:                     fang.GetString("POSTGRES_USER"),
-		PostgresPassword:                 fang.GetString("POSTGRES_PASSWORD"),
-		PostgresHost:                     fang.GetString("POSTGRES_HOST"),
-		PostgresPort:                     fang.GetInt("POSTGRES_PORT"),
-		PostgresDatabase:                 fang.GetString("POSTGRES_DATABASE"),
-		PostgresMaxConnections:           fang.GetInt("POSTGRES_MAX_CONNECTIONS"),
-		PostgresConnectionMaxLifetime:    fang.GetInt("POSTGRES_CONNECTIONS_MAX_LIFETIME"),
-		NewRelicAppName:                  fang.GetString("NEW_RELIC_APP_NAME"),
-		NewRelicLicenceKey:               fang.GetString("NEW_RELIC_LICENCE_KEY"),
-		MinClientVersion:                 fang.GetString("MIN_CLIENT_VERSION"),
-		ScheduledJobsFetchIntervalInMins: fang.GetInt("SCHEDULED_JOBS_FETCH_INTERVAL_IN_MINS"),
-		MailUsername:                     fang.GetString("MAIL_USERNAME"),
-		MailServerHost:                   fang.GetString("MAIL_SERVER_HOST"),
-		MailPassword:                     fang.GetString("MAIL_PASSWORD"),
-		MailServerPort:                   fang.GetString("MAIL_SERVER_PORT"),
-		JobPodAnnotations:                GetMapFromJson(fang, "JOB_POD_ANNOTATIONS"),
-		SentryDSN:                        fang.GetString("SENTRY_DSN"),
-		DocsPath:                         fang.GetString("DOCS_PATH"),
-		AuthPluginBinary:                 fang.GetString("AUTH_PLUGIN_BINARY"),
-		AuthPluginExported:               GetStringDefault(fang, "AUTH_PLUGIN_EXPORTED", "Auth"),
-		AuthEnabled:                      GetBoolDefault(fang, "AUTH_ENABLED", false),
-		NotificationPluginBinary:         GetArrayString(fang, "NOTIFICATION_PLUGIN_BINARY"),
-		NotificationPluginExported:       GetArrayString(fang, "NOTIFICATION_PLUGIN_EXPORTED"),
-		AuthRequiredAdminGroup:           GetArrayStringDefault(fang, "AUTH_REQUIRED_ADMIN_GROUP", []string{"proctor_admin"}),
+		KubeConfig:                       fang.GetString("kube.config"),
+		KubeContext:                      GetStringDefault(fang, "kube.context", "default"),
+		LogLevel:                         GetStringDefault(fang, "log.level", "DEBUG"),
+		AppPort:                          GetStringDefault(fang, "app.port", "5001"),
+		DefaultNamespace:                 fang.GetString("default.namespace"),
+		RedisAddress:                     fang.GetString("redis.address"),
+		RedisMaxActiveConnections:        fang.GetInt("redis.max.active.connections"),
+		LogsStreamReadBufferSize:         fang.GetInt("logs.stream.read.buffer.size"),
+		LogsStreamWriteBufferSize:        fang.GetInt("logs.stream.write.buffer.size"),
+		KubeWaitForResourcePollCount:     fang.GetInt("kube.wait.for.resource.poll.count"),
+		KubeLogProcessWaitTime:           time.Duration(fang.GetInt("kube.log.process.wait.time")),
+		KubeJobActiveDeadlineSeconds:     GetInt64Ref(fang, "kube.job.active.deadline.seconds"),
+		KubeJobRetries:                   GetInt32Ref(fang, "kube.job.retries"),
+		KubeServiceAccountName:           fang.GetString("kube.service.account.name"),
+		PostgresUser:                     fang.GetString("postgres.user"),
+		PostgresPassword:                 fang.GetString("postgres.password"),
+		PostgresHost:                     fang.GetString("postgres.host"),
+		PostgresPort:                     fang.GetInt("postgres.port"),
+		PostgresDatabase:                 fang.GetString("postgres.database"),
+		PostgresMaxConnections:           fang.GetInt("postgres.max.connections"),
+		PostgresConnectionMaxLifetime:    fang.GetInt("postgres.connections.max.lifetime"),
+		NewRelicAppName:                  fang.GetString("new.relic.app.name"),
+		NewRelicLicenseKey:               fang.GetString("new.relic.licence.key"),
+		MinClientVersion:                 fang.GetString("min.client.version"),
+		ScheduledJobsFetchIntervalInMins: fang.GetInt("scheduled.jobs.fetch.interval.in.mins"),
+		MailUsername:                     fang.GetString("mail.username"),
+		MailServerHost:                   fang.GetString("mail.server.host"),
+		MailPassword:                     fang.GetString("mail.password"),
+		MailServerPort:                   fang.GetString("mail.server.port"),
+		JobPodAnnotations:                GetMapFromJson(fang, "job.pod.annotations"),
+		DocsPath:                         fang.GetString("docs.path"),
+		AuthPluginBinary:                 fang.GetString("auth.plugin.binary"),
+		AuthPluginExported:               GetStringDefault(fang, "auth.plugin.exported", "Auth"),
+		AuthEnabled:                      GetBoolDefault(fang, "auth.enabled", false),
+		NotificationPluginBinary:         GetArrayString(fang, "notification.plugin.binary"),
+		NotificationPluginExported:       GetArrayString(fang, "notification.plugin.exported"),
+		AuthRequiredAdminGroup:           GetArrayStringDefault(fang, "auth.required.admin.group", []string{"proctor.admin"}),
 	}
 
 	return proctorConfig
@@ -176,11 +185,11 @@ func Reset() {
 
 func Config() ProctorConfig {
 	once.Do(func() {
-		config = Load()
+		config = load()
 	})
 
 	if reset.Get() {
-		config = Load()
+		config = load()
 		reset.Set(false)
 	}