use match-scanner library

config.yaml

@@ -1,10 +1,7 @@
 # Secret Scanner Configuration File
-
-blacklisted_strings: [ ] # skip matches containing any of these strings (case sensitive)
-blacklisted_extensions: [ ".exe", ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".psd", ".xcf", ".zip", ".tar", ".tar.gz", ".ttf", ".lock", ".pem", ".so", ".jar", ".gz" ]
-blacklisted_paths: [ "{sep}var{sep}lib{sep}docker", "{sep}var{sep}lib{sep}containerd", "{sep}var{sep}lib{sep}containers", "{sep}var{sep}lib{sep}crio", "{sep}var{sep}run{sep}containers", "{sep}bin", "{sep}boot", "{sep}dev", "{sep}lib", "{sep}lib64", "{sep}media", "{sep}proc", "{sep}run", "{sep}sbin", "{sep}usr{sep}lib", "{sep}sys", "{sep}home{sep}kubernetes" ]
-exclude_paths: [ "{sep}var{sep}lib{sep}docker", "{sep}var{name_sep}lib{name_sep}docker","{sep}var{sep}lib{sep}containerd", "{sep}var{name_sep}lib{name_sep}containerd", "lost+found", "{sep}bin", "{sep}boot", "{sep}dev", "{sep}lib", "{sep}lib64", "{sep}media", "{sep}proc", "{sep}run", "{sep}sbin", "{sep}usr{sep}lib", "{sep}sys", "{sep}home{sep}kubernetes" ] # use {sep} for the OS' path seperator and {name_sep} for -  (i.e. / or \)
-
+exclude_extensions: [ ".exe", ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".psd", ".xcf", ".zip", ".tar", ".tar.gz", ".ttf", ".lock", ".pem", ".so", ".jar", ".gz" ]
+exclude_paths: ["/var/lib/docker", "/var/lib/containerd", "/dev", "/proc", "/usr/lib", "/sys", "/boot", "/run", ".home/kubernetes"]
+max_file_size: 1073741824
 
 signatures:
 - part: 'extension'

core/config.go

@@ -1,12 +1,14 @@
 package core
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path"
 	"path/filepath"
 	"regexp"
 
+	"github.com/deepfence/match-scanner/pkg/config"
 	"gopkg.in/yaml.v3"
 )
 
@@ -146,6 +148,23 @@ func loadConfigFile(configPath string) (*Config, error) {
 	return config, nil
 }
 
+func loadExtractorConfigFile(options *Options) (config.Config, error) {
+	configs := options.ConfigPath.Values()
+	if len(configs) != 1 {
+		return config.Config{}, errors.New("too many config files")
+	}
+	configPath := configs[0]
+	fstat, err := os.Stat(configPath)
+	if err != nil {
+		return config.Config{}, err
+	}
+
+	if fstat.IsDir() {
+		return config.ParseConfig(filepath.Join(configPath, "config,yaml"))
+	}
+	return config.ParseConfig(configPath)
+}
+
 func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	*c = Config{}
 	type plain Config

core/session.go

@@ -7,15 +7,17 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/deepfence/match-scanner/pkg/config"
 	log "github.com/sirupsen/logrus"
 )
 
 type Session struct {
 	sync.Mutex
-	Version string
-	Options *Options
-	Config  *Config
-	Context context.Context
+	Version         string
+	Options         *Options
+	Config          *Config
+	Context         context.Context
+	ExtractorConfig config.Config
 }
 
 var (
@@ -53,6 +55,11 @@ func GetSession() *Session {
 			os.Exit(1)
 		}
 
+		if session.ExtractorConfig, err = loadExtractorConfigFile(session.Options); err != nil {
+			log.Error(err)
+			os.Exit(1)
+		}
+
 		pathSeparator := string(os.PathSeparator)
 		nameSeperator := "-"
 		var blacklistedPaths []string

go.mod

@@ -1,14 +1,16 @@
 module github.com/deepfence/SecretScanner
 
-go 1.21
+go 1.21.0
+
+toolchain go1.22.0
 
 replace github.com/deepfence/agent-plugins-grpc => ./agent-plugins-grpc
 
 require (
 	github.com/deepfence/agent-plugins-grpc v0.0.0-00010101000000-000000000000
 	github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20231201173641-092afefd00a2
 	github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20231201173641-092afefd00a2
-	github.com/deepfence/vessel v0.12.3
+	github.com/deepfence/match-scanner v0.0.0-20240627065846-d2405fb72cfb
 	github.com/fatih/color v1.16.0
 	github.com/flier/gohs v1.2.2
 	github.com/olekukonko/tablewriter v0.0.5
@@ -20,7 +22,6 @@ require (
 require (
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
 	github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 // indirect
-	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/Microsoft/hcsshim v0.12.3 // indirect
 	github.com/containerd/cgroups/v3 v3.0.2 // indirect
@@ -31,6 +32,7 @@ require (
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/containerd/ttrpc v1.2.3 // indirect
 	github.com/containerd/typeurl/v2 v2.1.1 // indirect
+	github.com/deepfence/vessel v0.12.3 // indirect
 	github.com/distribution/reference v0.6.0 // indirect
 	github.com/docker/docker v26.1.1+incompatible // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
@@ -55,6 +57,7 @@ require (
 	github.com/moby/sys/sequential v0.5.0 // indirect
 	github.com/moby/sys/signal v0.7.0 // indirect
 	github.com/moby/sys/user v0.1.0 // indirect
+	github.com/nlepage/go-tarfs v1.2.1 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/opencontainers/runtime-spec v1.2.0 // indirect

go.sum

@@ -38,6 +38,8 @@ github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20231201173641-092afefd0
 github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20231201173641-092afefd00a2/go.mod h1:+rchMc4YNjCoHo0YAwKsT+DRBNr1hdDG0WrvAOOCc5k=
 github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20231201173641-092afefd00a2 h1:b7PmvEUzF2b+XJ5XxZJNt+gkInw85cxryfoOfCkLL3c=
 github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20231201173641-092afefd00a2/go.mod h1:jHS6Adf3VrxnKZZ3RY10BirtFlwWj99Zd4JBAhP9SqM=
+github.com/deepfence/match-scanner v0.0.0-20240627065846-d2405fb72cfb h1:E3ffVItZVnhj1CD6UO/FPKyPz7Osinc4M770Jmm4JKc=
+github.com/deepfence/match-scanner v0.0.0-20240627065846-d2405fb72cfb/go.mod h1:eSZaZ9yVo4FoA3hJzTVWJL8HKvm3YPQzw3Nx/NKuq9A=
 github.com/deepfence/vessel v0.12.3 h1:C34t+sV+JoFdfYhg+uyS+YOEDAFIYjBKHShD3emDISA=
 github.com/deepfence/vessel v0.12.3/go.mod h1:bY97YUKMm0Oxasz/9o7Te60FjWCQWUYpgiWNC1E00xo=
 github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
@@ -133,6 +135,8 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
 github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/nlepage/go-tarfs v1.2.1 h1:o37+JPA+ajllGKSPfy5+YpsNHDjZnAoyfvf5GsUa+Ks=
+github.com/nlepage/go-tarfs v1.2.1/go.mod h1:rno18mpMy9aEH1IiJVftFsqPyIpwqSUiAOpJYjlV2NA=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
@@ -218,7 +222,6 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

jobs/scan.go

@@ -7,11 +7,13 @@ import (
 	"sync"
 	"time"
 
+	"github.com/deepfence/SecretScanner/core"
 	"github.com/deepfence/SecretScanner/output"
 	"github.com/deepfence/SecretScanner/scan"
 	"github.com/deepfence/golang_deepfence_sdk/utils/tasks"
 
 	pb "github.com/deepfence/agent-plugins-grpc/srcgo"
+	cfg "github.com/deepfence/match-scanner/pkg/config"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -45,34 +47,29 @@ func DispatchScan(r *pb.FindRequest) {
 			close(res)
 		}()
 
-		var secrets chan output.SecretFound
+		var (
+			scanType scan.ScanType
+			nodeID   string
+		)
 
 		if r.GetPath() != "" {
-			var isFirstSecret bool = true
-			secrets, err = scan.ScanSecretsInDirStream("", r.GetPath(), r.GetPath(),
-				&isFirstSecret, scanCtx)
-			if err != nil {
-				return
-			}
+			scanType = scan.DirScan
+			nodeID = r.GetPath()
 		} else if r.GetImage() != nil && r.GetImage().Name != "" {
-			secrets, err = scan.ExtractAndScanImageStream(r.GetImage().Name, scanCtx)
-			if err != nil {
-				return
-			}
+			scanType = scan.ImageScan
+			nodeID = r.GetImage().Name
 		} else if r.GetContainer() != nil && r.GetContainer().Id != "" {
-			secrets, err = scan.ExtractAndScanContainerStream(r.GetContainer().Id,
-				r.GetContainer().Namespace, scanCtx)
-			if err != nil {
-				return
-			}
+			scanType = scan.ContainerScan
+			nodeID = r.GetContainer().Id
 		} else {
 			err = fmt.Errorf("Invalid request")
 			return
 		}
 
-		for secret := range secrets {
-			writeSingleScanData(output.SecretToSecretInfo(secret), r.ScanId)
-		}
+		filters := cfg.Config2Filter(core.GetSession().ExtractorConfig)
+		err = scan.Scan(scanCtx, scanType, filters, "", nodeID, r.GetScanId(), func(sf output.SecretFound, s string) {
+			writeSingleScanData(output.SecretToSecretInfo(sf), r.ScanId)
+		})
 	}()
 }
 

main.go

@@ -38,6 +38,7 @@ import (
 	"github.com/deepfence/SecretScanner/scan"
 	"github.com/deepfence/SecretScanner/server"
 	"github.com/deepfence/SecretScanner/signature"
+	"github.com/deepfence/match-scanner/pkg/config"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -53,109 +54,51 @@ var (
 // and setup the session to start scanning for secrets
 var session = core.GetSession()
 
-// Scan a container image for secrets layer by layer
-// @parameters
-// image - Name of the container image to scan (e.g. "alpine:3.5")
-// @returns
-// Error, if any. Otherwise, returns nil
-func findSecretsInImage(image string) (*output.JSONImageSecretsOutput, error) {
-
-	res, err := scan.ExtractAndScanImage(image)
-	if err != nil {
-		return nil, err
-	}
-	jsonImageSecretsOutput := output.JSONImageSecretsOutput{ImageName: image}
-	jsonImageSecretsOutput.SetTime()
-	jsonImageSecretsOutput.SetImageID(res.ImageId)
-	jsonImageSecretsOutput.SetSecrets(res.Secrets)
-
-	return &jsonImageSecretsOutput, nil
-}
-
-// Scan a directory
-// @parameters
-// dir - Complete path of the directory to be scanned
-// @returns
-// Error, if any. Otherwise, returns nil
-func findSecretsInDir(dir string) (*output.JSONDirSecretsOutput, error) {
-	var isFirstSecret bool = true
-
-	secrets, err := scan.ScanSecretsInDir("", "", dir, &isFirstSecret, nil)
-	if err != nil {
-		log.Error("findSecretsInDir: %s", err)
-		return nil, err
-	}
-
-	jsonDirSecretsOutput := output.JSONDirSecretsOutput{DirName: *session.Options.Local}
-	jsonDirSecretsOutput.SetTime()
-	jsonDirSecretsOutput.SetSecrets(secrets)
-
-	return &jsonDirSecretsOutput, nil
-}
-
-// Scan a container for secrets
-// @parameters
-// containerId - Id of the container to scan (e.g. "0fdasf989i0")
-// @returns
-// Error, if any. Otherwise, returns nil
-func findSecretsInContainer(containerId string, containerNS string) (*output.JSONImageSecretsOutput, error) {
-
-	res, err := scan.ExtractAndScanContainer(containerId, containerNS, nil)
-	if err != nil {
-		return nil, err
-	}
-	jsonImageSecretsOutput := output.JSONImageSecretsOutput{ContainerID: containerId}
-	jsonImageSecretsOutput.SetTime()
-	jsonImageSecretsOutput.SetImageID(res.ContainerId)
-	jsonImageSecretsOutput.SetSecrets(res.Secrets)
-
-	return &jsonImageSecretsOutput, nil
-}
-
 type SecretsWriter interface {
 	WriteJSON() error
 	WriteTable() error
 	GetSecrets() []output.SecretFound
+	AddSecret(output.SecretFound)
 }
 
-func runOnce(format string) {
+func runOnce(filters config.Filters, format string) {
 	var result SecretsWriter
 	var err error
 	node_type := ""
 	node_id := ""
+	var nodeType scan.ScanType
 
 	// Scan container image for secrets
 	if len(*session.Options.ImageName) > 0 {
 		node_type = "image"
 		node_id = *session.Options.ImageName
+		nodeType = scan.ImageScan
 		log.Infof("Scanning image %s for secrets...", *session.Options.ImageName)
-		result, err = findSecretsInImage(*session.Options.ImageName)
-		if err != nil {
-			log.Fatal("main: error while scanning image: %s", err)
+		result = &output.JSONImageSecretsOutput{
+			ImageName: *session.Options.ImageName,
+			Secrets:   []output.SecretFound{},
 		}
-	}
-
-	// Scan local directory for secrets
-	if len(*session.Options.Local) > 0 {
-		node_id = output.GetHostname()
-		log.Debugf("Scanning local directory: %s", *session.Options.Local)
-		result, err = findSecretsInDir(*session.Options.Local)
-		if err != nil {
-			log.Fatal("main: error while scanning dir: %s", err)
+	} else if len(*session.Options.Local) > 0 { // Scan local directory for secrets
+		node_id = *session.Options.Local
+		nodeType = scan.DirScan
+		result = &output.JSONDirSecretsOutput{
+			DirName: *session.Options.Local,
+			Secrets: []output.SecretFound{},
 		}
-	}
-
-	// Scan existing container for secrets
-	if len(*session.Options.ContainerID) > 0 {
+	} else if len(*session.Options.ContainerID) > 0 { // Scan existing container for secrets
 		node_type = "container_image"
 		node_id = *session.Options.ContainerID
-		log.Debugf("Scanning container %s for secrets...", *session.Options.ContainerID)
-		result, err = findSecretsInContainer(*session.Options.ContainerID, *session.Options.ContainerNS)
-		if err != nil {
-			log.Fatal("main: error while scanning container: %s", err)
+		nodeType = scan.ContainerScan
+		result = &output.JSONImageSecretsOutput{
+			ContainerID: *session.Options.ContainerID,
+			Secrets:     []output.SecretFound{},
 		}
 	}
 
+	scan.Scan(nil, nodeType, filters, "", node_id, "", func(sf output.SecretFound, s string) {
+		result.AddSecret(sf)
+	})
+
 	if result == nil {
 		log.Error("set either -local or -image-name flag")
 		return
@@ -237,6 +180,7 @@ func main() {
 			log.Fatal("main: failed to serve: %v", err)
 		}
 	} else {
-		runOnce(*core.GetSession().Options.OutFormat)
+		extCfg := config.Config2Filter(core.GetSession().ExtractorConfig)
+		runOnce(extCfg, *core.GetSession().Options.OutFormat)
 	}
 }

output/output.go

@@ -73,6 +73,10 @@ func (imageOutput *JSONImageSecretsOutput) GetSecrets() []SecretFound {
 	return imageOutput.Secrets
 }
 
+func (imageOutput *JSONImageSecretsOutput) AddSecret(secret SecretFound) {
+	imageOutput.Secrets = append(imageOutput.Secrets, secret)
+}
+
 func (imageOutput JSONImageSecretsOutput) WriteJSON() error {
 	return printSecretsToJSON(imageOutput)
 
@@ -105,6 +109,10 @@ func (dirOutput JSONDirSecretsOutput) WriteTable() error {
 	return WriteTableOutput(&dirOutput.Secrets)
 }
 
+func (imageOutput *JSONDirSecretsOutput) AddSecret(secret SecretFound) {
+	imageOutput.Secrets = append(imageOutput.Secrets, secret)
+}
+
 func printSecretsToJSON(secretsJSON interface{}) error {
 	file, err := json.MarshalIndent(secretsJSON, "", Indent)
 	if err != nil {