diff --git a/config.yaml b/config.yaml index 31b89e5..3a87209 100644 --- a/config.yaml +++ b/config.yaml @@ -1,10 +1,7 @@ # Secret Scanner Configuration File - -blacklisted_strings: [ ] # skip matches containing any of these strings (case sensitive) -blacklisted_extensions: [ ".exe", ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".psd", ".xcf", ".zip", ".tar", ".tar.gz", ".ttf", ".lock", ".pem", ".so", ".jar", ".gz" ] -blacklisted_paths: [ "{sep}var{sep}lib{sep}docker", "{sep}var{sep}lib{sep}containerd", "{sep}var{sep}lib{sep}containers", "{sep}var{sep}lib{sep}crio", "{sep}var{sep}run{sep}containers", "{sep}bin", "{sep}boot", "{sep}dev", "{sep}lib", "{sep}lib64", "{sep}media", "{sep}proc", "{sep}run", "{sep}sbin", "{sep}usr{sep}lib", "{sep}sys", "{sep}home{sep}kubernetes" ] -exclude_paths: [ "{sep}var{sep}lib{sep}docker", "{sep}var{name_sep}lib{name_sep}docker","{sep}var{sep}lib{sep}containerd", "{sep}var{name_sep}lib{name_sep}containerd", "lost+found", "{sep}bin", "{sep}boot", "{sep}dev", "{sep}lib", "{sep}lib64", "{sep}media", "{sep}proc", "{sep}run", "{sep}sbin", "{sep}usr{sep}lib", "{sep}sys", "{sep}home{sep}kubernetes" ] # use {sep} for the OS' path seperator and {name_sep} for - (i.e. / or \) - +exclude_extensions: [ ".exe", ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".psd", ".xcf", ".zip", ".tar", ".tar.gz", ".ttf", ".lock", ".pem", ".so", ".jar", ".gz" ] +exclude_paths: ["/var/lib/docker", "/var/lib/containerd", "/dev", "/proc", "/usr/lib", "/sys", "/boot", "/run", ".home/kubernetes"] +max_file_size: 1073741824 signatures: - part: 'extension' diff --git a/core/config.go b/core/config.go index ecd8de7..20aedf8 100644 --- a/core/config.go +++ b/core/config.go @@ -1,12 +1,14 @@ package core import ( + "errors" "fmt" "os" "path" "path/filepath" "regexp" + "github.com/deepfence/match-scanner/pkg/config" "gopkg.in/yaml.v3" ) @@ -146,6 +148,23 @@ func loadConfigFile(configPath string) (*Config, error) { return config, nil } +func loadExtractorConfigFile(options *Options) (config.Config, error) { + configs := options.ConfigPath.Values() + if len(configs) != 1 { + return config.Config{}, errors.New("too many config files") + } + configPath := configs[0] + fstat, err := os.Stat(configPath) + if err != nil { + return config.Config{}, err + } + + if fstat.IsDir() { + return config.ParseConfig(filepath.Join(configPath, "config,yaml")) + } + return config.ParseConfig(configPath) +} + func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { *c = Config{} type plain Config diff --git a/core/session.go b/core/session.go index 034a3ac..ae33650 100644 --- a/core/session.go +++ b/core/session.go @@ -7,15 +7,17 @@ import ( "strings" "sync" + "github.com/deepfence/match-scanner/pkg/config" log "github.com/sirupsen/logrus" ) type Session struct { sync.Mutex - Version string - Options *Options - Config *Config - Context context.Context + Version string + Options *Options + Config *Config + Context context.Context + ExtractorConfig config.Config } var ( @@ -53,6 +55,11 @@ func GetSession() *Session { os.Exit(1) } + if session.ExtractorConfig, err = loadExtractorConfigFile(session.Options); err != nil { + log.Error(err) + os.Exit(1) + } + pathSeparator := string(os.PathSeparator) nameSeperator := "-" var blacklistedPaths []string diff --git a/go.mod b/go.mod index 8d6718c..8c1f6a6 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/deepfence/SecretScanner -go 1.21 +go 1.21.0 + +toolchain go1.22.0 replace github.com/deepfence/agent-plugins-grpc => ./agent-plugins-grpc @@ -8,7 +10,7 @@ require ( github.com/deepfence/agent-plugins-grpc v0.0.0-00010101000000-000000000000 github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20231201173641-092afefd00a2 github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20231201173641-092afefd00a2 - github.com/deepfence/vessel v0.12.3 + github.com/deepfence/match-scanner v0.0.0-20240627065846-d2405fb72cfb github.com/fatih/color v1.16.0 github.com/flier/gohs v1.2.2 github.com/olekukonko/tablewriter v0.0.5 @@ -20,7 +22,6 @@ require ( require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 // indirect - github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Microsoft/hcsshim v0.12.3 // indirect github.com/containerd/cgroups/v3 v3.0.2 // indirect @@ -31,6 +32,7 @@ require ( github.com/containerd/log v0.1.0 // indirect github.com/containerd/ttrpc v1.2.3 // indirect github.com/containerd/typeurl/v2 v2.1.1 // indirect + github.com/deepfence/vessel v0.12.3 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/docker v26.1.1+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect @@ -55,6 +57,7 @@ require ( github.com/moby/sys/sequential v0.5.0 // indirect github.com/moby/sys/signal v0.7.0 // indirect github.com/moby/sys/user v0.1.0 // indirect + github.com/nlepage/go-tarfs v1.2.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect diff --git a/go.sum b/go.sum index c1c2a44..a5ef52d 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20231201173641-092afefd0 github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20231201173641-092afefd00a2/go.mod h1:+rchMc4YNjCoHo0YAwKsT+DRBNr1hdDG0WrvAOOCc5k= github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20231201173641-092afefd00a2 h1:b7PmvEUzF2b+XJ5XxZJNt+gkInw85cxryfoOfCkLL3c= github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20231201173641-092afefd00a2/go.mod h1:jHS6Adf3VrxnKZZ3RY10BirtFlwWj99Zd4JBAhP9SqM= +github.com/deepfence/match-scanner v0.0.0-20240627065846-d2405fb72cfb h1:E3ffVItZVnhj1CD6UO/FPKyPz7Osinc4M770Jmm4JKc= +github.com/deepfence/match-scanner v0.0.0-20240627065846-d2405fb72cfb/go.mod h1:eSZaZ9yVo4FoA3hJzTVWJL8HKvm3YPQzw3Nx/NKuq9A= github.com/deepfence/vessel v0.12.3 h1:C34t+sV+JoFdfYhg+uyS+YOEDAFIYjBKHShD3emDISA= github.com/deepfence/vessel v0.12.3/go.mod h1:bY97YUKMm0Oxasz/9o7Te60FjWCQWUYpgiWNC1E00xo= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= @@ -133,6 +135,8 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/nlepage/go-tarfs v1.2.1 h1:o37+JPA+ajllGKSPfy5+YpsNHDjZnAoyfvf5GsUa+Ks= +github.com/nlepage/go-tarfs v1.2.1/go.mod h1:rno18mpMy9aEH1IiJVftFsqPyIpwqSUiAOpJYjlV2NA= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -218,7 +222,6 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/jobs/scan.go b/jobs/scan.go index 1dad3e3..61606b4 100644 --- a/jobs/scan.go +++ b/jobs/scan.go @@ -7,11 +7,13 @@ import ( "sync" "time" + "github.com/deepfence/SecretScanner/core" "github.com/deepfence/SecretScanner/output" "github.com/deepfence/SecretScanner/scan" "github.com/deepfence/golang_deepfence_sdk/utils/tasks" pb "github.com/deepfence/agent-plugins-grpc/srcgo" + cfg "github.com/deepfence/match-scanner/pkg/config" log "github.com/sirupsen/logrus" ) @@ -45,34 +47,29 @@ func DispatchScan(r *pb.FindRequest) { close(res) }() - var secrets chan output.SecretFound + var ( + scanType scan.ScanType + nodeID string + ) if r.GetPath() != "" { - var isFirstSecret bool = true - secrets, err = scan.ScanSecretsInDirStream("", r.GetPath(), r.GetPath(), - &isFirstSecret, scanCtx) - if err != nil { - return - } + scanType = scan.DirScan + nodeID = r.GetPath() } else if r.GetImage() != nil && r.GetImage().Name != "" { - secrets, err = scan.ExtractAndScanImageStream(r.GetImage().Name, scanCtx) - if err != nil { - return - } + scanType = scan.ImageScan + nodeID = r.GetImage().Name } else if r.GetContainer() != nil && r.GetContainer().Id != "" { - secrets, err = scan.ExtractAndScanContainerStream(r.GetContainer().Id, - r.GetContainer().Namespace, scanCtx) - if err != nil { - return - } + scanType = scan.ContainerScan + nodeID = r.GetContainer().Id } else { err = fmt.Errorf("Invalid request") return } - for secret := range secrets { - writeSingleScanData(output.SecretToSecretInfo(secret), r.ScanId) - } + filters := cfg.Config2Filter(core.GetSession().ExtractorConfig) + err = scan.Scan(scanCtx, scanType, filters, "", nodeID, r.GetScanId(), func(sf output.SecretFound, s string) { + writeSingleScanData(output.SecretToSecretInfo(sf), r.ScanId) + }) }() } diff --git a/main.go b/main.go index e4d63e5..9f550f7 100644 --- a/main.go +++ b/main.go @@ -38,6 +38,7 @@ import ( "github.com/deepfence/SecretScanner/scan" "github.com/deepfence/SecretScanner/server" "github.com/deepfence/SecretScanner/signature" + "github.com/deepfence/match-scanner/pkg/config" log "github.com/sirupsen/logrus" ) @@ -53,109 +54,51 @@ var ( // and setup the session to start scanning for secrets var session = core.GetSession() -// Scan a container image for secrets layer by layer -// @parameters -// image - Name of the container image to scan (e.g. "alpine:3.5") -// @returns -// Error, if any. Otherwise, returns nil -func findSecretsInImage(image string) (*output.JSONImageSecretsOutput, error) { - - res, err := scan.ExtractAndScanImage(image) - if err != nil { - return nil, err - } - jsonImageSecretsOutput := output.JSONImageSecretsOutput{ImageName: image} - jsonImageSecretsOutput.SetTime() - jsonImageSecretsOutput.SetImageID(res.ImageId) - jsonImageSecretsOutput.SetSecrets(res.Secrets) - - return &jsonImageSecretsOutput, nil -} - -// Scan a directory -// @parameters -// dir - Complete path of the directory to be scanned -// @returns -// Error, if any. Otherwise, returns nil -func findSecretsInDir(dir string) (*output.JSONDirSecretsOutput, error) { - var isFirstSecret bool = true - - secrets, err := scan.ScanSecretsInDir("", "", dir, &isFirstSecret, nil) - if err != nil { - log.Error("findSecretsInDir: %s", err) - return nil, err - } - - jsonDirSecretsOutput := output.JSONDirSecretsOutput{DirName: *session.Options.Local} - jsonDirSecretsOutput.SetTime() - jsonDirSecretsOutput.SetSecrets(secrets) - - return &jsonDirSecretsOutput, nil -} - -// Scan a container for secrets -// @parameters -// containerId - Id of the container to scan (e.g. "0fdasf989i0") -// @returns -// Error, if any. Otherwise, returns nil -func findSecretsInContainer(containerId string, containerNS string) (*output.JSONImageSecretsOutput, error) { - - res, err := scan.ExtractAndScanContainer(containerId, containerNS, nil) - if err != nil { - return nil, err - } - jsonImageSecretsOutput := output.JSONImageSecretsOutput{ContainerID: containerId} - jsonImageSecretsOutput.SetTime() - jsonImageSecretsOutput.SetImageID(res.ContainerId) - jsonImageSecretsOutput.SetSecrets(res.Secrets) - - return &jsonImageSecretsOutput, nil -} - type SecretsWriter interface { WriteJSON() error WriteTable() error GetSecrets() []output.SecretFound + AddSecret(output.SecretFound) } -func runOnce(format string) { +func runOnce(filters config.Filters, format string) { var result SecretsWriter var err error node_type := "" node_id := "" + var nodeType scan.ScanType // Scan container image for secrets if len(*session.Options.ImageName) > 0 { node_type = "image" node_id = *session.Options.ImageName + nodeType = scan.ImageScan log.Infof("Scanning image %s for secrets...", *session.Options.ImageName) - result, err = findSecretsInImage(*session.Options.ImageName) - if err != nil { - log.Fatal("main: error while scanning image: %s", err) + result = &output.JSONImageSecretsOutput{ + ImageName: *session.Options.ImageName, + Secrets: []output.SecretFound{}, } - } - - // Scan local directory for secrets - if len(*session.Options.Local) > 0 { - node_id = output.GetHostname() - log.Debugf("Scanning local directory: %s", *session.Options.Local) - result, err = findSecretsInDir(*session.Options.Local) - if err != nil { - log.Fatal("main: error while scanning dir: %s", err) + } else if len(*session.Options.Local) > 0 { // Scan local directory for secrets + node_id = *session.Options.Local + nodeType = scan.DirScan + result = &output.JSONDirSecretsOutput{ + DirName: *session.Options.Local, + Secrets: []output.SecretFound{}, } - } - - // Scan existing container for secrets - if len(*session.Options.ContainerID) > 0 { + } else if len(*session.Options.ContainerID) > 0 { // Scan existing container for secrets node_type = "container_image" node_id = *session.Options.ContainerID - log.Debugf("Scanning container %s for secrets...", *session.Options.ContainerID) - result, err = findSecretsInContainer(*session.Options.ContainerID, *session.Options.ContainerNS) - if err != nil { - log.Fatal("main: error while scanning container: %s", err) + nodeType = scan.ContainerScan + result = &output.JSONImageSecretsOutput{ + ContainerID: *session.Options.ContainerID, + Secrets: []output.SecretFound{}, } } + scan.Scan(nil, nodeType, filters, "", node_id, "", func(sf output.SecretFound, s string) { + result.AddSecret(sf) + }) + if result == nil { log.Error("set either -local or -image-name flag") return @@ -237,6 +180,7 @@ func main() { log.Fatal("main: failed to serve: %v", err) } } else { - runOnce(*core.GetSession().Options.OutFormat) + extCfg := config.Config2Filter(core.GetSession().ExtractorConfig) + runOnce(extCfg, *core.GetSession().Options.OutFormat) } } diff --git a/output/output.go b/output/output.go index 2bd5f54..8db7250 100644 --- a/output/output.go +++ b/output/output.go @@ -73,6 +73,10 @@ func (imageOutput *JSONImageSecretsOutput) GetSecrets() []SecretFound { return imageOutput.Secrets } +func (imageOutput *JSONImageSecretsOutput) AddSecret(secret SecretFound) { + imageOutput.Secrets = append(imageOutput.Secrets, secret) +} + func (imageOutput JSONImageSecretsOutput) WriteJSON() error { return printSecretsToJSON(imageOutput) @@ -105,6 +109,10 @@ func (dirOutput JSONDirSecretsOutput) WriteTable() error { return WriteTableOutput(&dirOutput.Secrets) } +func (imageOutput *JSONDirSecretsOutput) AddSecret(secret SecretFound) { + imageOutput.Secrets = append(imageOutput.Secrets, secret) +} + func printSecretsToJSON(secretsJSON interface{}) error { file, err := json.MarshalIndent(secretsJSON, "", Indent) if err != nil { diff --git a/scan/process_container.go b/scan/process_container.go deleted file mode 100644 index e9fa2c6..0000000 --- a/scan/process_container.go +++ /dev/null @@ -1,174 +0,0 @@ -package scan - -import ( - "errors" - "os" - "strings" - - "github.com/deepfence/SecretScanner/core" - "github.com/deepfence/SecretScanner/output" - tasks "github.com/deepfence/golang_deepfence_sdk/utils/tasks" - "github.com/deepfence/vessel" - containerdRuntime "github.com/deepfence/vessel/containerd" - crioRuntime "github.com/deepfence/vessel/crio" - dockerRuntime "github.com/deepfence/vessel/docker" - podmanRuntime "github.com/deepfence/vessel/podman" - vesselConstants "github.com/deepfence/vessel/utils" - log "github.com/sirupsen/logrus" -) - -type ContainerScan struct { - containerId string - tempDir string - namespace string - numSecrets uint -} - -// Function to retrieve contents of container -// @parameters -// containerScan - Structure with details of the container to scan -// @returns -// Error - Errors, if any. Otherwise, returns nil -func (containerScan *ContainerScan) extractFileSystem() error { - // Auto-detect underlying container runtime - containerRuntime, endpoint, err := vessel.AutoDetectRuntime() - if err != nil { - return err - } - var containerRuntimeInterface vessel.Runtime - switch containerRuntime { - case vesselConstants.DOCKER: - containerRuntimeInterface = dockerRuntime.New(endpoint) - case vesselConstants.CONTAINERD: - containerRuntimeInterface = containerdRuntime.New(endpoint) - case vesselConstants.CRIO: - containerRuntimeInterface = crioRuntime.New(endpoint) - case vesselConstants.PODMAN: - containerRuntimeInterface = podmanRuntime.New(endpoint) - } - if containerRuntimeInterface == nil { - log.Error("Error: Could not detect container runtime") - os.Exit(1) - } - err = containerRuntimeInterface.ExtractFileSystemContainer( - containerScan.containerId, containerScan.namespace, - containerScan.tempDir+".tar") - - if err != nil { - return err - } - runCommand("mkdir", containerScan.tempDir) - _, stdErr, retVal := runCommand("tar", "-xf", containerScan.tempDir+".tar", "-C"+containerScan.tempDir) - if retVal != 0 { - return errors.New(stdErr) - } - runCommand("rm", containerScan.tempDir+".tar") - return nil -} - -// Function to scan extracted layers of container file system for secrets file by file -// @parameters -// containerScan - Structure with details of the container to scan -// @returns -// []output.SecretFound - List of all secrets found -// Error - Errors, if any. Otherwise, returns nil -func (containerScan *ContainerScan) scan(scanCtx *tasks.ScanContext) ([]output.SecretFound, error) { - var isFirstSecret bool = true - - secrets, err := ScanSecretsInDir("", containerScan.tempDir, containerScan.tempDir, - &isFirstSecret, scanCtx) - if err != nil { - log.Errorf("findSecretsInContainer: %s", err) - return nil, err - } - - for _, secret := range secrets { - secret.CompleteFilename = strings.Replace(secret.CompleteFilename, containerScan.tempDir, "", 1) - } - - return secrets, nil -} - -// Function to scan extracted layers of container file system for secrets file by file -// @parameters -// containerScan - Structure with details of the container to scan -// @returns -// []output.SecretFound - List of all secrets found -// Error - Errors, if any. Otherwise, returns nil -func (containerScan *ContainerScan) scanStream(scanCtx *tasks.ScanContext) (chan output.SecretFound, error) { - var isFirstSecret bool = true - - stream, err := ScanSecretsInDirStream("", containerScan.tempDir, - containerScan.tempDir, &isFirstSecret, scanCtx) - - if err != nil { - log.Errorf("findSecretsInContainer: %s", err) - return nil, err - } - - return stream, nil -} - -type ContainerExtractionResult struct { - Secrets []output.SecretFound - ContainerId string -} - -func ExtractAndScanContainer(containerId string, namespace string, - scanCtx *tasks.ScanContext) (*ContainerExtractionResult, error) { - - tempDir, err := core.GetTmpDir(containerId) - if err != nil { - return nil, err - } - defer core.DeleteTmpDir(tempDir) - - containerScan := ContainerScan{containerId: containerId, tempDir: tempDir, namespace: namespace} - err = containerScan.extractFileSystem() - - if err != nil { - return nil, err - } - - secrets, err := containerScan.scan(scanCtx) - - if err != nil { - return nil, err - } - return &ContainerExtractionResult{ContainerId: containerScan.containerId, Secrets: secrets}, nil -} - -func ExtractAndScanContainerStream(containerId string, namespace string, - scanCtx *tasks.ScanContext) (chan output.SecretFound, error) { - tempDir, err := core.GetTmpDir(containerId) - if err != nil { - return nil, err - } - - containerScan := ContainerScan{containerId: containerId, tempDir: tempDir, namespace: namespace} - err = containerScan.extractFileSystem() - - if err != nil { - core.DeleteTmpDir(tempDir) - return nil, err - } - - stream, err := containerScan.scanStream(scanCtx) - - if err != nil { - core.DeleteTmpDir(tempDir) - return nil, err - } - - res := make(chan output.SecretFound, secret_pipeline_size) - - go func() { - defer core.DeleteTmpDir(tempDir) - defer close(res) - for i := range stream { - res <- i - } - }() - - return res, nil -} diff --git a/scan/process_image.go b/scan/process_image.go deleted file mode 100644 index e4c96f7..0000000 --- a/scan/process_image.go +++ /dev/null @@ -1,809 +0,0 @@ -package scan - -import ( - "archive/tar" - "bufio" - "bytes" - "compress/gzip" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "os/exec" - "path" - "path/filepath" - "strings" - "syscall" - - "github.com/deepfence/SecretScanner/core" - "github.com/deepfence/SecretScanner/output" - "github.com/deepfence/SecretScanner/signature" - "github.com/deepfence/golang_deepfence_sdk/utils/tasks" - "github.com/deepfence/vessel" - log "github.com/sirupsen/logrus" -) - -// Data type to store details about the container image after parsing manifest -type manifestItem struct { - Config string - RepoTags []string - Layers []string - LayerIds []string `json:",omitempty"` -} - -var ( - imageTarFileName = "save-output.tar" - maxSecretsExceeded = errors.New("number of secrets exceeded max-secrets") -) - -const ( - secret_pipeline_size = 100 -) - -type ImageScan struct { - imageName string - imageId string - tempDir string - imageManifest manifestItem - numSecrets uint -} - -// Function to retrieve contents of container images layer by layer -// @parameters -// imageScan - Structure with details of the container image to scan -// @returns -// Error - Errors, if any. Otherwise, returns nil -func (imageScan *ImageScan) extractImage(saveImage bool) error { - imageName := imageScan.imageName - tempDir := imageScan.tempDir - imageScan.numSecrets = 0 - - if saveImage { - err := imageScan.saveImageData() - if err != nil { - log.Errorf("scanImage: Could not save container image: %s. Check if the image name is correct.", err) - return err - } - } - - _, err := extractTarFile(imageName, path.Join(tempDir, imageTarFileName), tempDir) - if err != nil { - log.Errorf("scanImage: Could not extract image tar file: %s", err) - return err - } - - imageManifest, err := extractDetailsFromManifest(tempDir) - if err != nil { - log.Errorf("ProcessImageLayers: Could not get image's history: %s,"+ - " please specify repo:tag and check disk space", err.Error()) - return err - } - - imageScan.imageManifest = imageManifest - // reading image id from imanifest file json path and tripping off extension - imageScan.imageId = strings.TrimSuffix(imageScan.imageManifest.Config, ".json") - - return nil -} - -// Function to scan extracted layers of container images for secrets file by file -// @parameters -// imageScan - Structure with details of the container image to scan -// @returns -// []output.SecretFound - List of all secrets found -// Error - Errors, if any. Otherwise, returns nil -func (imageScan *ImageScan) scan(scanCtx *tasks.ScanContext) ([]output.SecretFound, error) { - tempDir := imageScan.tempDir - defer core.DeleteTmpDir(tempDir) - - tempSecretsFound, err := imageScan.processImageLayers(tempDir, scanCtx) - if err != nil { - log.Error("scanImage: %s", err) - return tempSecretsFound, err - } - - return tempSecretsFound, nil -} - -// Function to scan extracted layers of container images for secrets file by file -// @parameters -// imageScan - Structure with details of the container image to scan -// @returns -// []output.SecretFound - List of all secrets found -// Error - Errors, if any. Otherwise, returns nil -func (imageScan *ImageScan) scanStream(scanCtx *tasks.ScanContext) (chan output.SecretFound, error) { - return imageScan.processImageLayersStream(imageScan.tempDir, scanCtx) -} - -func readFile(path string) ([]byte, error) { - var content string - file, err := os.OpenFile(path, os.O_RDONLY, os.ModePerm) - if err != nil { - return nil, err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - if len(line) == 0 { - continue - } - content += scanner.Text() + "\n" - } - return []byte(content), nil -} - -func scanFile(filePath, relPath, fileName, fileExtension, layer string, numSecrets *uint, matchedRuleSet map[uint]uint) ([]output.SecretFound, error) { - contents, err := readFile(filePath) - if err != nil { - return nil, err - } - // fmt.Println(relPath, file.Filename, file.Extension, layer) - secrets, err := signature.MatchPatternSignatures(contents, relPath, fileName, fileExtension, layer, numSecrets, matchedRuleSet) - if err != nil { - return nil, err - } - return secrets, nil -} - -// ScanSecretsInDir Scans a given directory recursively to find all secrets inside any file in the dir -// @parameters -// layer - layer ID, if we are scanning directory inside container image -// baseDir - Parent directory -// fullDir - Complete path of the directory to be scanned -// isFirstSecret - indicates if some secrets are already printed, used to properly format json -// @returns -// []output.SecretFound - List of all secrets found -// Error - Errors if any. Otherwise, returns nil -func ScanSecretsInDir(layer string, baseDir string, fullDir string, - isFirstSecret *bool, scanCtx *tasks.ScanContext) ([]output.SecretFound, error) { - var secretsFound []output.SecretFound - matchedRuleSet := map[uint]uint{} - - session := core.GetSession() - - if layer != "" { - core.UpdateDirsPermissionsRW(fullDir) - } - - maxFileSize := *session.Options.MaximumFileSize * 1024 - numSecrets := uint(0) - - walkErr := filepath.WalkDir(fullDir, func(path string, f os.DirEntry, err error) error { - if err != nil { - log.Debugf("Error in filepath.Walk: %s", err) - return err - } - - err = scanCtx.Checkpoint("walking in directories") - if err != nil { - return err - } - - var scanDirPath string - if layer != "" { - scanDirPath = strings.TrimPrefix(path, baseDir+"/"+layer) - if scanDirPath == "" { - scanDirPath = "/" - } - } else { - scanDirPath = path - } - - if f.IsDir() { - if core.IsSkippableDir(scanDirPath, baseDir) { - return filepath.SkipDir - } - return nil - } - - // No need to scan sym links. This avoids hangs when scanning stderr, stdour or special file descriptors - // Also, the pointed files will anyway be scanned directly - if !f.Type().IsRegular() { - return nil - } - - finfo, err := f.Info() - if err != nil { - log.Warnf("Skipping %v as info could not be retrieved: %v", path, err) - return nil - } - - if uint(finfo.Size()) > maxFileSize || core.IsSkippableFileExtension(path) { - return nil - } - - file := core.NewMatchFile(path) - - relPath, err := filepath.Rel(filepath.Join(baseDir, layer), file.Path) - if err != nil { - log.Warnf("scanSecretsInDir: Couldn't remove prefix of path: %s %s %s", - baseDir, layer, file.Path) - relPath = file.Path - } - - // Add RW permissions for reading and deleting contents of containers, not for regular file system - if layer != "" { - err = os.Chmod(file.Path, 0600) - if err != nil { - log.Errorf("scanSecretsInDir changine file permission: %s", err) - } - } - - log.Debugf("attempting scanFile on: %+v, relPath: %s", file, relPath) - - secrets, err := scanFile(file.Path, relPath, file.Filename, file.Extension, layer, &numSecrets, matchedRuleSet) - if err != nil { - log.Infof("relPath: %s, Filename: %s, Extension: %s, layer: %s", relPath, file.Filename, file.Extension, layer) - log.Errorf("scanSecretsInDir: %s", err) - } else { - if len(secrets) > 0 { - secretsFound = append(secretsFound, secrets...) - } - } - - secrets = signature.MatchSimpleSignatures(relPath, file.Filename, file.Extension, layer, &numSecrets) - secretsFound = append(secretsFound, secrets...) - - log.Debugf("scan completed for file: %+v, numSecrets: %d", file, numSecrets) - - // Don't report secrets if number of secrets exceeds MAX value - if numSecrets >= *session.Options.MaxSecrets { - return maxSecretsExceeded - } - return nil - }) - - if walkErr != nil { - if walkErr == maxSecretsExceeded { - log.Warnf("filepath.Walk: %s", walkErr) - } else { - log.Errorf("Error in filepath.Walk: %s", walkErr) - } - } - - return secretsFound, nil -} - -// ScanSecretsInDirStream Scans a given directory recursively to find all secrets inside any file in the dir -// @parameters -// layer - layer ID, if we are scanning directory inside container image -// baseDir - Parent directory -// fullDir - Complete path of the directory to be scanned -// isFirstSecret - indicates if some secrets are already printed, used to properly format json -// @returns -// chan output.SecretFound - Channel of all secrets found -// Error - Errors if any. Otherwise, returns nil -func ScanSecretsInDirStream(layer string, baseDir string, fullDir string, - isFirstSecret *bool, scanCtx *tasks.ScanContext) (chan output.SecretFound, error) { - - res := make(chan output.SecretFound, secret_pipeline_size) - - matchedRuleSet := map[uint]uint{} - numSecrets := uint(0) - - if layer != "" { - core.UpdateDirsPermissionsRW(fullDir) - } - - go func() { - - defer close(res) - session := core.GetSession() - maxFileSize := *session.Options.MaximumFileSize * 1024 - - walkErr := filepath.WalkDir(fullDir, func(path string, f os.DirEntry, err error) error { - if err != nil { - return err - } - - err = scanCtx.Checkpoint("walking in directories") - if err != nil { - return err - } - - var scanDirPath string - if layer != "" { - scanDirPath = strings.TrimPrefix(path, baseDir+"/"+layer) - if scanDirPath == "" { - scanDirPath = "/" - } - } else { - scanDirPath = path - } - - if f.IsDir() { - if core.IsSkippableDir(scanDirPath, baseDir) { - return filepath.SkipDir - } - return nil - } - - // No need to scan sym links. This avoids hangs when scanning stderr, stdour or special file descriptors - // Also, the pointed files will anyway be scanned directly - if !f.Type().IsRegular() { - return nil - } - - finfo, err := f.Info() - if err != nil { - log.Warnf("Skipping %v as info could not be retrieved: %v", path, err) - return nil - } - - if uint(finfo.Size()) > maxFileSize || core.IsSkippableFileExtension(path) { - return nil - } - - file := core.NewMatchFile(path) - - relPath, err := filepath.Rel(filepath.Join(baseDir, layer), file.Path) - if err != nil { - log.Warnf("scanSecretsInDir: Couldn't remove prefix of path: %s %s %s", - baseDir, layer, file.Path) - relPath = file.Path - } - - // Add RW permissions for reading and deleting contents of containers, not for regular file system - if layer != "" { - err = os.Chmod(file.Path, 0600) - if err != nil { - log.Errorf("scanSecretsInDir changine file permission: %s", err) - } - } - secrets, err := scanFile(file.Path, relPath, file.Filename, file.Extension, layer, &numSecrets, matchedRuleSet) - - if err != nil { - log.Infof("relPath: %s, Filename: %s, Extension: %s, layer: %s", relPath, file.Filename, file.Extension, layer) - log.Errorf("scanSecretsInDir: %s", err) - } else { - if len(secrets) > 0 { - for i := range secrets { - res <- secrets[i] - } - } - } - - secrets = signature.MatchSimpleSignatures(relPath, file.Filename, file.Extension, layer, &numSecrets) - for i := range secrets { - res <- secrets[i] - } - // Don't report secrets if number of secrets exceeds MAX value - if numSecrets >= *session.Options.MaxSecrets { - return maxSecretsExceeded - } - return nil - }) - if walkErr != nil { - if walkErr == maxSecretsExceeded { - log.Warnf("filepath.Walk: %s", walkErr) - } else { - log.Errorf("Error in filepath.Walk: %s", walkErr) - } - } - }() - return res, nil -} - -// Extract all the layers of the container image and then find secrets in each layer one by one -// @parameters -// imageScan - Structure with details of the container image to scan -// imageManifestPath - Complete path of directory where manifest of image has been extracted -// @returns -// []output.SecretFound - List of all secrets found -// Error - Errors if any. Otherwise, returns nil -func (imageScan *ImageScan) processImageLayers(imageManifestPath string, - scanCtx *tasks.ScanContext) ([]output.SecretFound, error) { - - var tempSecretsFound []output.SecretFound - var err error - var isFirstSecret bool = true - - // extractPath - Base directory where all the layers should be extracted to - extractPath := path.Join(imageManifestPath, core.ExtractedImageFilesDir) - layerIDs := imageScan.imageManifest.LayerIds - layerPaths := imageScan.imageManifest.Layers - - loopCntr := len(layerPaths) - var secrets []output.SecretFound - for i := 0; i < loopCntr; i++ { - log.Debugf("Analyzing layer path: %s", layerPaths[i]) - log.Debugf("Analyzing layer: %s", layerIDs[i]) - // savelayerID = layerIDs[i] - completeLayerPath := path.Join(imageManifestPath, layerPaths[i]) - targetDir := path.Join(extractPath, layerIDs[i]) - log.Debugf("Complete layer path: %s", completeLayerPath) - log.Debugf("Extracted to directory: %s", targetDir) - err = core.CreateRecursiveDir(targetDir) - if err != nil { - log.Errorf("ProcessImageLayers: Unable to create target directory to extract image layers... %s", err) - return tempSecretsFound, err - } - - _, error := extractTarFile("", completeLayerPath, targetDir) - if error != nil { - log.Errorf("ProcessImageLayers: Unable to extract image layer. Reason = %s", error.Error()) - // Don't stop. Print error and continue with remaning extracted files and other layers - // return tempSecretsFound, error - } - log.Debugf("Analyzing dir: %s", targetDir) - secrets, err = ScanSecretsInDir(layerIDs[i], extractPath, targetDir, - &isFirstSecret, scanCtx) - - imageScan.numSecrets += uint(len(secrets)) - tempSecretsFound = append(tempSecretsFound, secrets...) - if err != nil { - log.Errorf("ProcessImageLayers: %s", err) - // return tempSecretsFound, err - } - - // Don't report secrets if number of secrets exceeds MAX value - if imageScan.numSecrets >= *core.GetSession().Options.MaxSecrets { - return tempSecretsFound, nil - } - } - - return tempSecretsFound, nil -} - -// Extract all the layers of the container image and then stream secrets in each layer one by one -// @parameters -// imageScan - Structure with details of the container image to scan -// imageManifestPath - Complete path of directory where manifest of image has been extracted -// @returns -// []output.SecretFound - List of all secrets found -// Error - Errors if any. Otherwise, returns nil -func (imageScan *ImageScan) processImageLayersStream(imageManifestPath string, - scanCtx *tasks.ScanContext) (chan output.SecretFound, error) { - res := make(chan output.SecretFound, secret_pipeline_size) - - go func() { - var err error - var isFirstSecret bool = true - - defer close(res) - - // extractPath - Base directory where all the layers should be extracted to - extractPath := path.Join(imageManifestPath, core.ExtractedImageFilesDir) - layerIDs := imageScan.imageManifest.LayerIds - layerPaths := imageScan.imageManifest.Layers - - loopCntr := len(layerPaths) - var secrets []output.SecretFound - for i := 0; i < loopCntr; i++ { - log.Debugf("Analyzing layer path: %s", layerPaths[i]) - log.Debugf("Analyzing layer: %s", layerIDs[i]) - // savelayerID = layerIDs[i] - completeLayerPath := path.Join(imageManifestPath, layerPaths[i]) - targetDir := path.Join(extractPath, layerIDs[i]) - log.Infof("Complete layer path: %s", completeLayerPath) - log.Infof("Extracted to directory: %s", targetDir) - err = core.CreateRecursiveDir(targetDir) - if err != nil { - log.Error("ProcessImageLayers: Unable to create target directory extract image layers... %v", err) - continue - } - - _, error := extractTarFile("", completeLayerPath, targetDir) - if error != nil { - log.Errorf("ProcessImageLayers: Unable to extract image layer. Reason = %s", error.Error()) - // Don't stop. Print error and continue with remaning extracted files and other layers - continue - } - log.Debugf("Analyzing dir: %s", targetDir) - secrets, err = ScanSecretsInDir(layerIDs[i], extractPath, - targetDir, &isFirstSecret, scanCtx) - - imageScan.numSecrets += uint(len(secrets)) - for i := range secrets { - res <- secrets[i] - } - if err != nil { - log.Errorf("ProcessImageLayers: %s", err) - continue - } - - // Don't report secrets if number of secrets exceeds MAX value - if imageScan.numSecrets >= *core.GetSession().Options.MaxSecrets { - break - } - } - }() - - return res, nil -} - -// Save container image as tar file in specified directory -// @parameters -// imageScan - Structure with details of the container image to scan -// @returns -// Error - Errors if any. Otherwise, returns nil -func (imageScan *ImageScan) saveImageData() error { - imageName := imageScan.imageName - outputParam := path.Join(imageScan.tempDir, imageTarFileName) - drun, err := vessel.NewRuntime() - if err != nil { - return err - } - log.Infof("Scanning image %s for secrets...", outputParam) - _, err = drun.Save(imageName, outputParam) - - if err != nil { - return err - } - log.Infof("Image %s saved in %s", imageName, imageScan.tempDir) - return nil -} - -// Extract the contents of container image and save it in specified dir -// @parameters -// imageName - Name of the container image to save -// imageTarPath - Complete path where tarball of the image is stored -// extractPath - Complete path of directory where contents of image are to be extracted -// @returns -// string - directory where contents of image are extracted -// Error - Errors, if any. Otherwise, returns nil -func extractTarFile(imageName, imageTarPath string, extractPath string) (string, error) { - log.Debugf("Started extracting tar file %s", imageTarPath) - - path := extractPath - - // Extract the contents of image from tar file - if err := untar(imageTarPath, path); err != nil { - log.Error(err) - return "", err - } - - log.Debug("Finished extracting tar file %s", imageTarPath) - return path, nil -} - -// Extract all the details from image manifest -// @parameters -// path - Complete path where image contents are extracted -// @returns -// manifestItem - The manifestItem containing details about image layers -// Error - Errors, if any. Otherwise, returns nil -func untar(tarName string, xpath string) (err error) { - tarFile, err := os.Open(tarName) - if err != nil { - return err - } - defer func() { - err = tarFile.Close() - }() - - absPath, err := filepath.Abs(xpath) - if err != nil { - return err - } - - tr := tar.NewReader(tarFile) - if strings.HasSuffix(tarName, ".gz") || strings.HasSuffix(tarName, ".gzip") { - gz, err := gzip.NewReader(tarFile) - if err != nil { - return err - } - defer gz.Close() - tr = tar.NewReader(gz) - } - - // untar each segment - for { - hdr, err := tr.Next() - if err == io.EOF { - break - } - if err != nil { - return err - } - - // determine proper file path info - finfo := hdr.FileInfo() - fileName := hdr.Name - if filepath.IsAbs(fileName) { - fileName, err = filepath.Rel("/", fileName) - if err != nil { - return err - } - } - - absFileName := filepath.Join(absPath, fileName) - if strings.Contains(fileName, "/") { - relPath := strings.Split(fileName, "/") - var absDirPath string - if len(relPath) > 1 { - dirs := relPath[0 : len(relPath)-1] - absDirPath = filepath.Join(absPath, strings.Join(dirs, "/")) - } - if err := os.MkdirAll(absDirPath, 0755); err != nil { - log.Error(err) - } - } - - if finfo.Mode().IsDir() { - if err := os.MkdirAll(absFileName, 0755); err != nil { - return err - } - continue - } - - // create new file with original file mode - file, err := os.OpenFile(absFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, finfo.Mode().Perm()) - if err != nil { - log.Error(err) - return err - } - // fmt.Printf("x %s\n", absFileName) - n, cpErr := io.Copy(file, tr) - if closeErr := file.Close(); closeErr != nil { // close file immediately - log.Error("closeErr:" + closeErr.Error()) - return err - } - if cpErr != nil { - log.Error("copyErr:" + cpErr.Error()) - return cpErr - } - if n != finfo.Size() { - return fmt.Errorf("unexpected bytes written: wrote %d, want %d", n, finfo.Size()) - } - } - return nil -} - -// Extract all the details from image manifest -// @parameters -// path - Complete path where image contents are extracted -// @returns -// manifestItem - The manifestItem containing details about image layers -// Error - Errors, if any. Otherwise, returns nil -func extractDetailsFromManifest(path string) (manifestItem, error) { - mf, err := os.Open(path + "/manifest.json") - if err != nil { - return manifestItem{}, err - } - defer mf.Close() - - var manifest []manifestItem - if err = json.NewDecoder(mf).Decode(&manifest); err != nil { - return manifestItem{}, err - } else if len(manifest) != 1 { - return manifestItem{}, err - } - var layerIds []string - for _, layer := range manifest[0].Layers { - trimmedLayerId := strings.TrimSuffix(layer, "/layer.tar") - // manifests saved by some versions of skopeo has .tar extentions - trimmedLayerId = strings.TrimSuffix(trimmedLayerId, ".tar") - layerIds = append(layerIds, trimmedLayerId) - } - manifest[0].LayerIds = layerIds - // ImageScan.imageManifest = manifest[0] - return manifest[0], nil -} - -// Execute the specified command and return the output -// @parameters -// name - Command to be executed -// args - all the arguments to be passed to the command -// @returns -// string - contents of standard output -// string - contents of standard error -// int - exit code of the executed command -func runCommand(name string, args ...string) (stdout string, stderr string, exitCode int) { - var defaultFailedCode = 1 - var outbuf, errbuf bytes.Buffer - cmd := exec.Command(name, args...) - cmd.Stdout = &outbuf - cmd.Stderr = &errbuf - - err := cmd.Run() - stdout = outbuf.String() - stderr = errbuf.String() - - if err != nil { - // try to get the exit code - if exitError, ok := err.(*exec.ExitError); ok { - ws := exitError.Sys().(syscall.WaitStatus) - exitCode = ws.ExitStatus() - } else { - // This will happen (in OSX) if `name` is not available in $PATH, - // in this situation, exit code could not be get, and stderr will be - // empty string very likely, so we use the default fail code, and format err - // to string and set to stderr - log.Printf("Could not get exit code for failed program: %v, %v", name, args) - exitCode = defaultFailedCode - if stderr == "" { - stderr = err.Error() - } - } - } else { - // success, exitCode should be 0 if go is ok - ws := cmd.ProcessState.Sys().(syscall.WaitStatus) - exitCode = ws.ExitStatus() - } - return -} - -type ImageExtractionResult struct { - Secrets []output.SecretFound - ImageId string -} - -func ExtractAndScanImage(image string) (*ImageExtractionResult, error) { - tempDir, err := core.GetTmpDir(image) - if err != nil { - return nil, err - } - // defer core.DeleteTmpDir(tempDir) - - imageScan := ImageScan{imageName: image, imageId: "", tempDir: tempDir} - err = imageScan.extractImage(true) - - if err != nil { - return nil, err - } - - secrets, err := imageScan.scan(nil) - - if err != nil { - return nil, err - } - return &ImageExtractionResult{ImageId: imageScan.imageId, Secrets: secrets}, nil -} - -func ExtractAndScanImageStream(image string, scanCtx *tasks.ScanContext) (chan output.SecretFound, error) { - tempDir, err := core.GetTmpDir(image) - if err != nil { - return nil, err - } - - imageScan := ImageScan{imageName: image, imageId: "", tempDir: tempDir} - err = imageScan.extractImage(true) - - if err != nil { - core.DeleteTmpDir(tempDir) - return nil, err - } - - stream, err := imageScan.scanStream(scanCtx) - - if err != nil { - core.DeleteTmpDir(tempDir) - return nil, err - } - - res := make(chan output.SecretFound, secret_pipeline_size) - - go func() { - defer core.DeleteTmpDir(tempDir) - defer close(res) - for i := range stream { - res <- i - } - }() - - return res, nil - -} - -func ExtractAndScanFromTar(tarFolder string, imageName string, - scanCtx *tasks.ScanContext) (*ImageExtractionResult, error) { - // defer core.DeleteTmpDir(tarFolder) - - imageScan := ImageScan{imageName: imageName, imageId: "", tempDir: tarFolder} - err := imageScan.extractImage(false) - - if err != nil { - return nil, err - } - - secrets, err := imageScan.scan(scanCtx) - - if err != nil { - return nil, err - } - return &ImageExtractionResult{ImageId: imageScan.imageId, Secrets: secrets}, nil -} diff --git a/scan/scanner.go b/scan/scanner.go new file mode 100644 index 0000000..dedde88 --- /dev/null +++ b/scan/scanner.go @@ -0,0 +1,102 @@ +package scan + +import ( + "context" + "fmt" + "io" + "path/filepath" + + "github.com/deepfence/SecretScanner/output" + "github.com/deepfence/SecretScanner/signature" + tasks "github.com/deepfence/golang_deepfence_sdk/utils/tasks" + "github.com/deepfence/match-scanner/pkg/extractor" + genscan "github.com/deepfence/match-scanner/pkg/scanner" + + cfg "github.com/deepfence/match-scanner/pkg/config" + "github.com/sirupsen/logrus" +) + +type ScanType int + +const ( + DirScan ScanType = iota + ImageScan + ContainerScan +) + +func ScanTypeString(st ScanType) string { + switch st { + case DirScan: + return "host" + case ImageScan: + return "image" + case ContainerScan: + return "container" + } + return "" +} + +func scanFile(contents io.ReadSeeker, relPath, fileName, fileExtension, layer string, numSecrets *uint, matchedRuleSet map[uint]uint) ([]output.SecretFound, error) { + + secrets, err := signature.MatchPatternSignatures(contents, relPath, fileName, fileExtension, layer, numSecrets, matchedRuleSet) + if err != nil { + return nil, err + } + return secrets, nil +} + +func Scan(ctx *tasks.ScanContext, + stype ScanType, + filters cfg.Filters, + namespace, id, scanID string, + outputFn func(output.SecretFound, string)) error { + var ( + extract extractor.FileExtractor + err error + ) + switch stype { + case DirScan: + extract, err = extractor.NewDirectoryExtractor(filters, id, true) + case ImageScan: + extract, err = extractor.NewImageExtractor(filters, namespace, id) + case ContainerScan: + extract, err = extractor.NewContainerExtractor(filters, namespace, id) + default: + err = fmt.Errorf("invalid request") + } + if err != nil { + return err + } + defer extract.Close() + + // results has to be 1 element max + // to avoid overwriting the buffer entries + results := make(chan []output.SecretFound) + defer close(results) + + go func() { + for malwares := range results { + for _, malware := range malwares { + outputFn(malware, scanID) + } + } + }() + + genscan.ApplyScan(context.Background(), extract, func(f extractor.ExtractedFile) { + if ctx != nil { + err := ctx.Checkpoint("scan_phase") + if err != nil { + return + } + } + matchedRuleSet := map[uint]uint{} + var count uint + m, err := scanFile(f.Content, f.Filename, filepath.Base(f.Filename), filepath.Ext(f.Filename), "", &count, matchedRuleSet) + if err != nil { + logrus.Infof("file: %v, err: %v", f.Filename, err) + } + + results <- m + }) + return nil +} diff --git a/signature/signatures.go b/signature/signatures.go index 472c1c8..7953d12 100644 --- a/signature/signatures.go +++ b/signature/signatures.go @@ -4,10 +4,13 @@ import ( // "regexp" // "regexp/syntax" // "strings" + "bufio" "bytes" "errors" + "io" "math" "regexp" + "strings" "github.com/deepfence/SecretScanner/core" "github.com/deepfence/SecretScanner/output" @@ -44,6 +47,7 @@ var ( simpleSignatureMap map[string][]core.ConfigSignature patternSignatureMap map[string][]core.ConfigSignature hyperscanBlockDbMap map[string]hyperscan.BlockDatabase + regexpMap map[string]*regexp.Regexp signatureIDMap map[int]core.ConfigSignature ) @@ -53,6 +57,7 @@ func init() { simpleSignatureMap = make(map[string][]core.ConfigSignature) patternSignatureMap = make(map[string][]core.ConfigSignature) hyperscanBlockDbMap = make(map[string]hyperscan.BlockDatabase) + regexpMap = make(map[string]*regexp.Regexp) signatureIDMap = make(map[int]core.ConfigSignature) } @@ -99,50 +104,63 @@ func MatchSimpleSignatures(path string, filename string, extension string, layer // @returns // []output.SecretFound - List of all secrets found // Error - Errors if any. Otherwise, returns nil -func MatchPatternSignatures(contents []byte, path string, filename string, extension string, layerID string, +func MatchPatternSignatures(contents io.ReadSeeker, path string, filename string, extension string, layerID string, numSecrets *uint, matchedRuleSet map[uint]uint) ([]output.SecretFound, error) { var tempSecretsFound []output.SecretFound - var hsIOData HsInputOutputData + //var hsIOData HsInputOutputData var matchingPart string - var matchingStr []byte + var matchingStr io.RuneReader for _, part := range []string{ContentsPart, FilenamePart, PathPart, ExtPart} { switch part { case FilenamePart: matchingPart = part - matchingStr = []byte(filename) + matchingStr = bufio.NewReader(strings.NewReader(filename)) case PathPart: matchingPart = part - matchingStr = []byte(path) + matchingStr = bufio.NewReader(strings.NewReader(path)) case ExtPart: matchingPart = part - matchingStr = []byte(extension) + matchingStr = bufio.NewReader(strings.NewReader(extension)) case ContentsPart: matchingPart = part - matchingStr = contents + matchingStr = bufio.NewReader(contents) } - // Ignore if string to match is empty, otherwise hyperscan can return errors - if len(matchingStr) == 0 { - continue - } - - hsIOData = HsInputOutputData{ - inputData: matchingStr, - inputDataLowerCase: bytes.ToLower(matchingStr), - completeFilename: path, - layerID: layerID, - secretsFound: &tempSecretsFound, - numSecrets: numSecrets, - matchedRuleSet: matchedRuleSet, - } - err := RunHyperscan(hyperscanBlockDbMap[matchingPart], hsIOData) - if err != nil { - log.Infof("part: %s, path: %s, filename: %s, extenstion: %s, layerID: %s", - part, path, filename, extension, layerID) - log.Warnf("MatchPatternSignatures: %s", err) - return tempSecretsFound, err + //hsIOData = HsInputOutputData{ + // inputData: matchingStr, + // inputDataLowerCase: bytes.ToLower(matchingStr), + // completeFilename: path, + // layerID: layerID, + // secretsFound: &tempSecretsFound, + // numSecrets: numSecrets, + // matchedRuleSet: matchedRuleSet, + //} + indexes := regexpMap[matchingPart].FindReaderSubmatchIndex(matchingStr) + if indexes != nil { + tempSecretsFound = append(tempSecretsFound, output.SecretFound{ + LayerID: layerID, + RuleID: 0, + RuleName: "", + PartToMatch: part, + Match: matchingPart[indexes[0]:indexes[1]], + Regex: regexpMap[matchingPart].String(), + Severity: "", + SeverityScore: 0, + PrintBufferStartIndex: 0, + MatchFromByte: 0, + MatchToByte: 0, + CompleteFilename: filename, + MatchedContents: "", + }) } + //err := RunHyperscan(hyperscanBlockDbMap[matchingPart], hsIOData) + //if err != nil { + // log.Infof("part: %s, path: %s, filename: %s, extenstion: %s, layerID: %s", + // part, path, filename, extension, layerID) + // log.Warnf("MatchPatternSignatures: %s", err) + // return tempSecretsFound, err + //} } return tempSecretsFound, nil