organize Environment

cmd/zinc/main.go

@@ -27,15 +27,15 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/rs/zerolog/log"
 
+	"github.com/zinclabs/zinc/pkg/config"
 	"github.com/zinclabs/zinc/pkg/core"
 	"github.com/zinclabs/zinc/pkg/meta"
 	"github.com/zinclabs/zinc/pkg/routes"
-	"github.com/zinclabs/zinc/pkg/zutils"
 )
 
 func main() {
 
-	if zutils.GetEnvToBool("ZINC_SENTRY", "true") {
+	if config.Global.SentryEnable {
 		/******** initialize sentry **********/
 		err := sentry.Init(sentry.ClientOptions{
 			Dsn:     "https://15b6d9b8be824b44896f32b0234c32b7@o1218932.ingest.sentry.io/6360942",
@@ -55,7 +55,7 @@ func main() {
 	routes.SetRoutes(r)     // Set up all API routes.
 
 	// Run the server
-	PORT := zutils.GetEnv("PORT", "4080")
+	PORT := config.Global.ServerPort
 	server := &http.Server{
 		Addr:    ":" + PORT,
 		Handler: r,

pkg/bluge/analysis/lang/chs/gse.go

@@ -16,12 +16,15 @@
 package chs
 
 import (
+	"strings"
+
 	"github.com/blugelabs/bluge/analysis"
 	"github.com/go-ego/gse"
 
 	"github.com/zinclabs/zinc/pkg/bluge/analysis/lang/chs/analyzer"
 	"github.com/zinclabs/zinc/pkg/bluge/analysis/lang/chs/token"
 	"github.com/zinclabs/zinc/pkg/bluge/analysis/lang/chs/tokenizer"
+	"github.com/zinclabs/zinc/pkg/config"
 	"github.com/zinclabs/zinc/pkg/zutils"
 )
 
@@ -48,8 +51,9 @@ var seg *gse.Segmenter
 
 func init() {
 	seg = new(gse.Segmenter)
-	enable := zutils.GetEnvToBool("ZINC_PLUGIN_GSE_ENABLE", "FALSE")     // false / true
-	embed := zutils.GetEnvToUpper("ZINC_PLUGIN_GSE_DICT_EMBED", "SMALL") // small / big
+	enable := config.Global.Plugin.GSE.Enable   // true / false
+	embed := config.Global.Plugin.GSE.DictEmbed // small / big
+	embed = strings.ToUpper(embed)
 	loadDict(enable, embed)
 }
 
@@ -70,7 +74,7 @@ func loadDict(enable bool, embed string) {
 	seg.SkipLog = true
 
 	// load user dict
-	dataPath := zutils.GetEnv("ZINC_PLUGIN_GSE_DICT_PATH", "./plugins/gse/dict")
+	dataPath := config.Global.Plugin.GSE.DictPath
 	userDict := dataPath + "/user.txt"
 	if ok, _ := zutils.IsExist(userDict); ok {
 		_ = seg.LoadDict(userDict)

pkg/bluge/directory/minio.go

@@ -32,7 +32,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/rs/zerolog/log"
 
-	"github.com/zinclabs/zinc/pkg/zutils"
+	"github.com/zinclabs/zinc/pkg/config"
 )
 
 // GetMinIOConfig returns a bluge config that will store index data in MinIO
@@ -53,9 +53,9 @@ type MinIODirectory struct {
 // NewMinIODirectory creates a new MinIODirectory instance which can be used to create MinIO backed indexes
 func NewMinIODirectory(bucket, prefix string) index.Directory {
 
-	endpoint := zutils.GetEnv("ZINC_MINIO_ENDPOINT", "")
-	accessKeyID := zutils.GetEnv("ZINC_MINIO_ACCESS_KEY_ID", "")
-	secretAccessKey := zutils.GetEnv("ZINC_MINIO_SECRET_ACCESS_KEY", "")
+	endpoint := config.Global.MinIO.Endpoint
+	accessKeyID := config.Global.MinIO.AccessKeyID
+	secretAccessKey := config.Global.MinIO.SecretAccessKey
 
 	opts := minio.Options{
 		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),

pkg/config/config.go

@@ -1,3 +1,18 @@
+/* Copyright 2022 Zinc Labs Inc. and Contributors
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+ */
+
 package config
 
 import (
@@ -12,20 +27,19 @@ import (
 )
 
 type config struct {
-	NodeID           string `env:"ZINC_NODE_ID,default=1"`
-	DataPath         string `env:"ZINC_DATA_PATH,default=./data"`
-	GinMode          string `env:"GIN_MODE"`
-	SentryEnable     bool   `env:"ZINC_SENTRY,default=true"`
-	TelemetryEnable  bool   `env:"ZINC_TELEMETRY,default=true"`
-	PrometheusEnable bool   `env:"ZINC_PROMETHEUS_ENABLE,default=false"`
-
-	BatchSize            int `env:"ZINC_BATCH_SIZE,default=1024"`
-	MaxResults           int `env:"ZINC_MAX_RESULTS,default=10000"`
-	AggregationTermsSize int `env:"ZINC_AGGREGATION_TERMS_SIZE,default=1000"`
-
-	S3     s3
-	MinIO  minIO
-	Plugin plugin
+	GinMode              string `env:"GIN_MODE"`
+	ServerPort           string `env:"ZINC_PORT,default=4080"`
+	NodeID               string `env:"ZINC_NODE_ID,default=1"`
+	DataPath             string `env:"ZINC_DATA_PATH,default=./data"`
+	SentryEnable         bool   `env:"ZINC_SENTRY,default=true"`
+	TelemetryEnable      bool   `env:"ZINC_TELEMETRY,default=true"`
+	PrometheusEnable     bool   `env:"ZINC_PROMETHEUS_ENABLE,default=false"`
+	BatchSize            int    `env:"ZINC_BATCH_SIZE,default=1024"`
+	MaxResults           int    `env:"ZINC_MAX_RESULTS,default=10000"`
+	AggregationTermsSize int    `env:"ZINC_AGGREGATION_TERMS_SIZE,default=1000"`
+	S3                   s3
+	MinIO                minIO
+	Plugin               plugin
 }
 
 type s3 struct {
@@ -54,12 +68,12 @@ type gse struct {
 	DictPath  string `env:"ZINC_PLUGIN_GSE_DICT_PATH,default=./plugins/gse/dict"`
 }
 
-var Config = new(config)
+var Global = new(config)
 
 func init() {
 	err := godotenv.Load()
 	fmt.Println(err)
-	rv := reflect.ValueOf(Config).Elem()
+	rv := reflect.ValueOf(Global).Elem()
 	loadConfig(rv)
 }
 

pkg/config/config_test.go

@@ -1,3 +1,18 @@
+/* Copyright 2022 Zinc Labs Inc. and Contributors
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+ */
+
 package config
 
 import (
@@ -7,21 +22,21 @@ import (
 )
 
 func TestConfig(t *testing.T) {
-	assert.Equal(t, "1", Config.NodeID)
-	assert.Equal(t, "./data", Config.DataPath)
-	assert.Equal(t, "", Config.GinMode)
-	assert.Equal(t, true, Config.SentryEnable)
-	assert.Equal(t, true, Config.TelemetryEnable)
-	assert.Equal(t, false, Config.PrometheusEnable)
+	assert.Equal(t, "1", Global.NodeID)
+	assert.Equal(t, "./data", Global.DataPath)
+	assert.Equal(t, "", Global.GinMode)
+	assert.Equal(t, true, Global.SentryEnable)
+	assert.Equal(t, true, Global.TelemetryEnable)
+	assert.Equal(t, false, Global.PrometheusEnable)
 
-	assert.Equal(t, 1024, Config.BatchSize)
-	assert.Equal(t, 10000, Config.MaxResults)
-	assert.Equal(t, 1000, Config.AggregationTermsSize)
+	assert.Equal(t, 1024, Global.BatchSize)
+	assert.Equal(t, 10000, Global.MaxResults)
+	assert.Equal(t, 1000, Global.AggregationTermsSize)
 
-	assert.Equal(t, "", Config.S3.Bucket)
-	assert.Equal(t, "", Config.MinIO.Endpoint)
+	assert.Equal(t, "", Global.S3.Bucket)
+	assert.Equal(t, "", Global.MinIO.Endpoint)
 
-	assert.Equal(t, false, Config.Plugin.GSE.Enable)
-	assert.Equal(t, "small", Config.Plugin.GSE.DictEmbed)
-	assert.Equal(t, "./plugins/gse/dict", Config.Plugin.GSE.DictPath)
+	assert.Equal(t, false, Global.Plugin.GSE.Enable)
+	assert.Equal(t, "small", Global.Plugin.GSE.DictEmbed)
+	assert.Equal(t, "./plugins/gse/dict", Global.Plugin.GSE.DictPath)
 }

pkg/core/deleteindex.go

@@ -20,15 +20,15 @@ import (
 	"errors"
 	"os"
 
-	"github.com/aws/aws-sdk-go-v2/config"
+	awsconfig "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/blugelabs/bluge"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/rs/zerolog/log"
 
-	"github.com/zinclabs/zinc/pkg/zutils"
+	"github.com/zinclabs/zinc/pkg/config"
 )
 
 func DeleteIndex(name string) error {
@@ -46,7 +46,7 @@ func DeleteIndex(name string) error {
 
 	// 3. Physically delete the index
 	if index.StorageType == "disk" {
-		dataPath := zutils.GetEnv("ZINC_DATA_PATH", "./data")
+		dataPath := config.Global.DataPath
 		err := os.RemoveAll(dataPath + "/" + index.Name)
 		if err != nil {
 			log.Error().Msgf("failed to delete index: %s", err.Error())
@@ -73,10 +73,10 @@ func DeleteIndex(name string) error {
 }
 
 func deleteFilesForIndexFromMinIO(indexName string) error {
-	endpoint := zutils.GetEnv("ZINC_MINIO_ENDPOINT", "")
-	accessKeyID := zutils.GetEnv("ZINC_MINIO_ACCESS_KEY_ID", "")
-	secretAccessKey := zutils.GetEnv("ZINC_MINIO_SECRET_ACCESS_KEY", "")
-	minioBucket := zutils.GetEnv("ZINC_MINIO_BUCKET", "")
+	endpoint := config.Global.MinIO.Endpoint
+	accessKeyID := config.Global.MinIO.AccessKeyID
+	secretAccessKey := config.Global.MinIO.SecretAccessKey
+	minioBucket := config.Global.MinIO.Bucket
 
 	opts := minio.Options{
 		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
@@ -109,14 +109,14 @@ func deleteFilesForIndexFromMinIO(indexName string) error {
 
 func deleteFilesForIndexFromS3(indexName string) error {
 	// Load the Shared AWS Configuration (~/.aws/config)
-	cfg, err := config.LoadDefaultConfig(context.TODO())
+	cfg, err := awsconfig.LoadDefaultConfig(context.TODO())
 	if err != nil {
 		log.Print("Error loading AWS config: ", err)
 		return err
 	}
 	client := s3.NewFromConfig(cfg)
 
-	s3bucket := zutils.GetEnv("ZINC_S3_BUCKET", "")
+	s3bucket := config.Global.S3.Bucket
 	ctx := context.Background()
 
 	// List Objects in the bucket at prefix

pkg/core/index.go

@@ -28,6 +28,7 @@ import (
 	"github.com/blugelabs/bluge/analysis"
 	"github.com/goccy/go-json"
 
+	"github.com/zinclabs/zinc/pkg/config"
 	"github.com/zinclabs/zinc/pkg/meta"
 	zincanalysis "github.com/zinclabs/zinc/pkg/uquery/analysis"
 	"github.com/zinclabs/zinc/pkg/zutils"
@@ -301,7 +302,7 @@ func (index *Index) LoadStorageSize() float64 {
 	case "minio":
 		return size // TODO: implement later
 	default:
-		path := zutils.GetEnv("ZINC_DATA_PATH", "./data")
+		path := config.Global.DataPath
 		indexLocation := filepath.Join(path, index.Name)
 		size, _ = zutils.DirSize(indexLocation)
 		return math.Round(size)

pkg/core/newindex.go

@@ -25,8 +25,8 @@ import (
 	"github.com/goccy/go-json"
 
 	"github.com/zinclabs/zinc/pkg/bluge/directory"
+	"github.com/zinclabs/zinc/pkg/config"
 	"github.com/zinclabs/zinc/pkg/meta"
-	"github.com/zinclabs/zinc/pkg/zutils"
 )
 
 // NewIndex creates an instance of a physical zinc index that can be used to store and retrieve data.
@@ -39,25 +39,25 @@ func NewIndex(name, storageType string, defaultSearchAnalyzer *analysis.Analyzer
 	}
 
 	var dataPath string
-	var config bluge.Config
+	var cfg bluge.Config
 	switch storageType {
 	case "s3":
-		dataPath = zutils.GetEnv("ZINC_S3_BUCKET", "")
-		config = directory.GetS3Config(dataPath, name)
+		dataPath = config.Global.S3.Bucket
+		cfg = directory.GetS3Config(dataPath, name)
 	case "minio":
-		dataPath = zutils.GetEnv("ZINC_MINIO_BUCKET", "")
-		config = directory.GetMinIOConfig(dataPath, name)
+		dataPath = config.Global.MinIO.Bucket
+		cfg = directory.GetMinIOConfig(dataPath, name)
 	default:
 		storageType = "disk"
-		dataPath = zutils.GetEnv("ZINC_DATA_PATH", "./data")
-		config = bluge.DefaultConfig(dataPath + "/" + name)
+		dataPath = config.Global.DataPath
+		cfg = bluge.DefaultConfig(dataPath + "/" + name)
 	}
 
 	if defaultSearchAnalyzer != nil {
-		config.DefaultSearchAnalyzer = defaultSearchAnalyzer
+		cfg.DefaultSearchAnalyzer = defaultSearchAnalyzer
 	}
 
-	writer, err := bluge.OpenWriter(config)
+	writer, err := bluge.OpenWriter(cfg)
 	if err != nil {
 		return nil, err
 	}
@@ -79,24 +79,24 @@ func NewIndex(name, storageType string, defaultSearchAnalyzer *analysis.Analyzer
 // LoadIndexWriter load the index writer from the storage
 func LoadIndexWriter(name string, storageType string, defaultSearchAnalyzer *analysis.Analyzer) (*bluge.Writer, error) {
 	var dataPath string
-	var config bluge.Config
+	var cfg bluge.Config
 	switch storageType {
 	case "s3":
-		dataPath = zutils.GetEnv("ZINC_S3_BUCKET", "")
-		config = directory.GetS3Config(dataPath, name)
+		dataPath = config.Global.S3.Bucket
+		cfg = directory.GetS3Config(dataPath, name)
 	case "minio":
-		dataPath = zutils.GetEnv("ZINC_MINIO_BUCKET", "")
-		config = directory.GetMinIOConfig(dataPath, name)
+		dataPath = config.Global.MinIO.Bucket
+		cfg = directory.GetMinIOConfig(dataPath, name)
 	default:
-		dataPath = zutils.GetEnv("ZINC_DATA_PATH", "./data")
-		config = bluge.DefaultConfig(dataPath + "/" + name)
+		dataPath = config.Global.DataPath
+		cfg = bluge.DefaultConfig(dataPath + "/" + name)
 	}
 
 	if defaultSearchAnalyzer != nil {
-		config.DefaultSearchAnalyzer = defaultSearchAnalyzer
+		cfg.DefaultSearchAnalyzer = defaultSearchAnalyzer
 	}
 
-	return bluge.OpenWriter(config)
+	return bluge.OpenWriter(cfg)
 }
 
 // storeIndex stores the index to metadata

pkg/core/telemetry.go

@@ -29,9 +29,9 @@ import (
 	"github.com/shirou/gopsutil/mem"
 	"gopkg.in/segmentio/analytics-go.v3"
 
+	"github.com/zinclabs/zinc/pkg/config"
 	"github.com/zinclabs/zinc/pkg/ider"
 	"github.com/zinclabs/zinc/pkg/meta"
-	"github.com/zinclabs/zinc/pkg/zutils"
 )
 
 // Telemetry instance
@@ -119,7 +119,7 @@ func (t *telemetry) initBaseInfo() {
 }
 
 func (t *telemetry) Instance() {
-	if !zutils.GetEnvToBool("ZINC_TELEMETRY", "true") {
+	if !config.Global.TelemetryEnable {
 		return
 	}
 
@@ -138,7 +138,7 @@ func (t *telemetry) Instance() {
 }
 
 func (t *telemetry) Event(event string, data map[string]interface{}) {
-	if !zutils.GetEnvToBool("ZINC_TELEMETRY", "true") {
+	if !config.Global.TelemetryEnable {
 		return
 	}