From d71aa1fb9e18ce93e9ba130c3eb52cb8aac942ce Mon Sep 17 00:00:00 2001 From: Thomas Legris Date: Mon, 8 Jul 2024 17:35:06 +0900 Subject: [PATCH] use new grpc abstraction --- Dockerfile | 46 +++++++++++++++++-- Makefile | 6 ++- go.mod | 4 +- go.sum | 2 + jobs/common.go | 4 +- jobs/scan.go | 64 +------------------------- jobs/status.go | 2 +- main.go | 25 ++++++++-- output/output.go | 35 +++++++------- rules/yara.rules | 11 +++++ server/grpc.go | 116 ++++++++++++++++------------------------------- 11 files changed, 141 insertions(+), 174 deletions(-) create mode 100644 rules/yara.rules diff --git a/Dockerfile b/Dockerfile index db3d8a9..701ada7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,13 +8,49 @@ MAINTAINER DeepFence RUN apk update \ && apk add --upgrade gcc musl-dev pkgconfig g++ make git -COPY --from=vectorscan /vectorscan.tar.bz2 / -RUN tar -xjf /vectorscan.tar.bz2 -C / && rm /vectorscan.tar.bz2 +RUN apk add --no-cache \ + git \ + make \ + build-base \ + pkgconfig \ + libpcap-dev \ + libcap-dev \ + openssl-dev \ + file \ + jansson-dev \ + jansson-static \ + bison \ + tini \ + su-exec + +RUN apk add --no-cache -t .build-deps py-setuptools \ + openssl-libs-static \ + jansson-dev \ + build-base \ + libc-dev \ + file-dev \ + automake \ + autoconf \ + libtool \ + libcrypto3 \ + flex \ + git \ + libmagic-static \ + linux-headers + +RUN cd /root && wget https://github.com/VirusTotal/yara/archive/refs/tags/v4.3.2.tar.gz \ + && tar -zxf v4.3.2.tar.gz \ + && cd yara-4.3.2 \ + && ./bootstrap.sh \ + && ./configure --prefix=/usr/local/yara --disable-dotnet --enable-magic --enable-cuckoo --disable-shared --enable-static\ + && make \ + && make install \ + && cd /usr/local/ \ + && tar -czf yara.tar.gz yara WORKDIR /home/deepfence/src/SecretScanner COPY . . -RUN make clean -RUN make +RUN make clean && make all FROM alpine:3.18 MAINTAINER DeepFence @@ -30,7 +66,7 @@ RUN apk add --no-cache --upgrade tar libstdc++ libgcc docker skopeo bash podman RUN < ./agent-plugins-grpc -replace github.com/deepfence/YaraHunter => ../YaraHunter - require ( - github.com/deepfence/YaraHunter v0.0.0-00010101000000-000000000000 + github.com/deepfence/YaraHunter v0.0.0-20240708090804-4196e3bbd2c1 github.com/deepfence/agent-plugins-grpc v0.0.0-00010101000000-000000000000 github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240626143546-e4ec9311fdf9 github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20240626143546-e4ec9311fdf9 diff --git a/go.sum b/go.sum index 6270cf7..0903485 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,8 @@ github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3H github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deepfence/YaraHunter v0.0.0-20240708090804-4196e3bbd2c1 h1:VuKkE8OKmM+hII/fViwYuLL1DTq5ncczMKWuOKg+EZc= +github.com/deepfence/YaraHunter v0.0.0-20240708090804-4196e3bbd2c1/go.mod h1:SSZ34MXU0qR30p6/rJ/PpY7OgwljWF3go6K/ZfYqHZ0= github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240626143546-e4ec9311fdf9 h1:hI/Fv6XabkERGza4E8g7XLhlkuzWjoXQwTmtq0WIC+Y= github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240626143546-e4ec9311fdf9/go.mod h1:+rchMc4YNjCoHo0YAwKsT+DRBNr1hdDG0WrvAOOCc5k= github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20240626143546-e4ec9311fdf9 h1:/1olNPTiYUFxuYsP79DKLEEieNWIdPDFxs+B58jysRA= diff --git a/jobs/common.go b/jobs/common.go index a29b216..feb8441 100644 --- a/jobs/common.go +++ b/jobs/common.go @@ -6,8 +6,8 @@ import ( ) var ( - scanStatusFilename = getDfInstallDir() + "/var/log/fenced/secret-scan-log/secret_scan_log.log" - scanFilename = getDfInstallDir() + "/var/log/fenced/secret-scan/secret_scan.log" + scanStatusFilename = GetDfInstallDir() + "/var/log/fenced/secret-scan-log/secret_scan_log.log" + scanFilename = GetDfInstallDir() + "/var/log/fenced/secret-scan/secret_scan.log" SecretScanDir = "/" ) diff --git a/jobs/scan.go b/jobs/scan.go index 61606b4..4bde5e0 100644 --- a/jobs/scan.go +++ b/jobs/scan.go @@ -2,77 +2,15 @@ package jobs import ( "encoding/json" - "fmt" "strings" "sync" - "time" - - "github.com/deepfence/SecretScanner/core" - "github.com/deepfence/SecretScanner/output" - "github.com/deepfence/SecretScanner/scan" - "github.com/deepfence/golang_deepfence_sdk/utils/tasks" pb "github.com/deepfence/agent-plugins-grpc/srcgo" - cfg "github.com/deepfence/match-scanner/pkg/config" log "github.com/sirupsen/logrus" ) var ScanMap sync.Map -func DispatchScan(r *pb.FindRequest) { - go func() { - startScanJob() - defer stopScanJob() - - var err error - res, scanCtx := tasks.StartStatusReporter( - r.ScanId, - func(ss tasks.ScanStatus) error { - return writeSecretScanStatus(ss.ScanStatus, ss.ScanId, ss.ScanMessage) - }, - tasks.StatusValues{ - IN_PROGRESS: "IN_PROGRESS", - CANCELLED: "CANCELLED", - FAILED: "ERROR", - SUCCESS: "COMPLETE", - }, - time.Minute*20, - ) - - ScanMap.Store(r.ScanId, scanCtx) - - defer func() { - ScanMap.Delete(r.ScanId) - res <- err - close(res) - }() - - var ( - scanType scan.ScanType - nodeID string - ) - - if r.GetPath() != "" { - scanType = scan.DirScan - nodeID = r.GetPath() - } else if r.GetImage() != nil && r.GetImage().Name != "" { - scanType = scan.ImageScan - nodeID = r.GetImage().Name - } else if r.GetContainer() != nil && r.GetContainer().Id != "" { - scanType = scan.ContainerScan - nodeID = r.GetContainer().Id - } else { - err = fmt.Errorf("Invalid request") - return - } - - filters := cfg.Config2Filter(core.GetSession().ExtractorConfig) - err = scan.Scan(scanCtx, scanType, filters, "", nodeID, r.GetScanId(), func(sf output.SecretFound, s string) { - writeSingleScanData(output.SecretToSecretInfo(sf), r.ScanId) - }) - }() -} - type SecretScanDoc struct { pb.SecretInfo ScanID string `json:"scan_id,omitempty"` @@ -100,7 +38,7 @@ func writeMultiScanData(secrets []*pb.SecretInfo, scan_id string) { } } -func writeSingleScanData(secret *pb.SecretInfo, scan_id string) { +func WriteSingleScanData(secret *pb.SecretInfo, scan_id string) { if SecretScanDir == HostMountDir { secret.GetMatch().FullFilename = strings.Replace(secret.GetMatch().GetFullFilename(), SecretScanDir, "", 1) } diff --git a/jobs/status.go b/jobs/status.go index 856f200..81d2d5a 100644 --- a/jobs/status.go +++ b/jobs/status.go @@ -41,7 +41,7 @@ func writeScanDataToFile(secretScanMsg string, filename string) error { return nil } -func getDfInstallDir() string { +func GetDfInstallDir() string { installDir, exists := os.LookupEnv("DF_INSTALL_DIR") if exists { return installDir diff --git a/main.go b/main.go index c82101b..e33c628 100644 --- a/main.go +++ b/main.go @@ -37,14 +37,20 @@ import ( "time" "github.com/deepfence/SecretScanner/core" + "github.com/deepfence/SecretScanner/jobs" "github.com/deepfence/SecretScanner/output" "github.com/deepfence/SecretScanner/scan" - "github.com/deepfence/SecretScanner/signature" + "github.com/deepfence/SecretScanner/server" "github.com/deepfence/golang_deepfence_sdk/utils/tasks" "github.com/deepfence/match-scanner/pkg/config" log "github.com/sirupsen/logrus" + "google.golang.org/grpc" + out "github.com/deepfence/YaraHunter/pkg/output" "github.com/deepfence/YaraHunter/pkg/runner" + yaraserver "github.com/deepfence/YaraHunter/pkg/server" + + pb "github.com/deepfence/agent-plugins-grpc/srcgo" ) const ( @@ -171,8 +177,6 @@ func main() { return "", " " + path.Base(f.File) + ":" + strconv.Itoa(f.Line) }, }) - // Process and store the read signatures - signature.ProcessSignatures(session.Config.Signatures) flag.Parse() @@ -182,6 +186,9 @@ func main() { ctx, _ := signal.NotifyContext(context.Background(), os.Interrupt) + out.ScanStatusFilename = jobs.GetDfInstallDir() + "/var/log/fenced/secret-scan-log/secret_scan_log.log" + out.ScanFilename = jobs.GetDfInstallDir() + "/var/log/fenced/secret-scan/secret_scan.log" + runnerOpts := runner.RunnerOptions{ SocketPath: *socketPath, RulesPath: *core.GetSession().Options.RulesPath, @@ -206,5 +213,15 @@ func main() { go runner.ScheduleYaraHunterUpdater(ctx, runnerOpts) } - runner.StartYaraHunter(ctx, runnerOpts, core.GetSession().ExtractorConfig) + runner.StartYaraHunter(ctx, runnerOpts, core.GetSession().ExtractorConfig, + + func(base *yaraserver.GRPCScannerServer) server.SecretGRPCServer { + return server.SecretGRPCServer{ + GRPCScannerServer: base, + UnimplementedSecretScannerServer: pb.UnimplementedSecretScannerServer{}, + } + }, + func(s grpc.ServiceRegistrar, impl any) { + pb.RegisterSecretScannerServer(s, impl.(pb.SecretScannerServer)) + }) } diff --git a/output/output.go b/output/output.go index 8db7250..f9d560e 100644 --- a/output/output.go +++ b/output/output.go @@ -6,6 +6,7 @@ import ( "os" "time" + "github.com/deepfence/YaraHunter/pkg/output" pb "github.com/deepfence/agent-plugins-grpc/srcgo" "github.com/fatih/color" tw "github.com/olekukonko/tablewriter" @@ -180,33 +181,31 @@ func removeFirstLastChar(input string) string { return input[1 : len(input)-1] } -func SecretsToSecretInfos(out []SecretFound) []*pb.SecretInfo { - res := make([]*pb.SecretInfo, 0) - for _, v := range out { - res = append(res, SecretToSecretInfo(v)) +func SecretToSecretInfo(out output.IOCFound) *pb.SecretInfo { + matchedContent := "" + if len(out.Meta) != 0 { + matchedContent = out.Meta[0] + } + signature := "" + if len(out.StringsToMatch) != 0 { + signature = out.StringsToMatch[0] + } + severity := "low" + if out.Severity != "" { + severity = out.Severity } - return res -} - -func SecretToSecretInfo(out SecretFound) *pb.SecretInfo { return &pb.SecretInfo{ ImageLayerId: out.LayerID, Rule: &pb.MatchRule{ - Id: int32(out.RuleID), Name: out.RuleName, - Part: out.PartToMatch, - StringToMatch: out.Match, - SignatureToMatch: out.Regex, + SignatureToMatch: signature, }, Match: &pb.Match{ - StartingIndex: int64(out.PrintBufferStartIndex), - RelativeStartingIndex: int64(out.MatchFromByte), - RelativeEndingIndex: int64(out.MatchToByte), - FullFilename: out.CompleteFilename, - MatchedContent: jsonMarshal(out.MatchedContents), + FullFilename: out.CompleteFilename, + MatchedContent: matchedContent, }, Severity: &pb.Severity{ - Level: out.Severity, + Level: severity, Score: float32(out.SeverityScore), }, } diff --git a/rules/yara.rules b/rules/yara.rules new file mode 100644 index 0000000..088487a --- /dev/null +++ b/rules/yara.rules @@ -0,0 +1,11 @@ +rule github_personal_access_token { + meta: + description = "Rule to match GitHub Personal Access Tokens (classic), Fine-grained & Github Actions Token" + author = "deepfence.io" + + strings: + $github_pat = /^gh[ps]_[a-zA-Z0-9]{36}|github_pat_[a-zA-Z0-9]{22}_[a-zA-Z0-9]{59}/ + + condition: + $github_pat +} diff --git a/server/grpc.go b/server/grpc.go index 0cdf8bd..7f6b081 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -2,95 +2,59 @@ package server import ( "context" - "fmt" - "net" - "sync" "github.com/deepfence/SecretScanner/jobs" + "github.com/deepfence/SecretScanner/output" + out "github.com/deepfence/YaraHunter/pkg/output" + "github.com/deepfence/YaraHunter/pkg/server" pb "github.com/deepfence/agent-plugins-grpc/srcgo" - + "github.com/sirupsen/logrus" //nolint:typecheck - "github.com/deepfence/golang_deepfence_sdk/utils/tasks" - log "github.com/sirupsen/logrus" - "google.golang.org/grpc" ) -type gRPCServer struct { - socket_path string - plugin_name string +type SecretGRPCServer struct { + *server.GRPCScannerServer pb.UnimplementedSecretScannerServer - pb.UnimplementedAgentPluginServer - pb.UnimplementedScannersServer -} - -func (s *gRPCServer) ReportJobsStatus(context.Context, *pb.Empty) (*pb.JobReports, error) { - return &pb.JobReports{ - RunningJobs: jobs.GetRunningJobCount(), - }, nil -} - -func (s *gRPCServer) StopScan(c context.Context, req *pb.StopScanRequest) (*pb.StopScanResult, error) { - log.Errorf("Received StopScanRequest: %v", *req) - scanID := req.ScanId - result := &pb.StopScanResult{ - Success: true, - Description: "", - } - - obj, found := jobs.ScanMap.Load(scanID) - if !found { - log.Errorf("SecretScanner::Failed to Stop scan, may have already completed successfully or errored out, scan_id: %s", scanID) - result.Success = false - result.Description = "SecretScanner::Failed to Stop scan" - return result, nil - } else { - log.Errorf("SecretScanner::Stop request submitted") - result.Description = "SecretScanner::Stop request submitted" - } - - scanCtx := obj.(*tasks.ScanContext) - scanCtx.StopTriggered.Store(true) - scanCtx.Cancel() - return result, nil -} - -func (s *gRPCServer) GetName(context.Context, *pb.Empty) (*pb.Name, error) { - return &pb.Name{Str: s.plugin_name}, nil } -func (s *gRPCServer) GetUID(context.Context, *pb.Empty) (*pb.Uid, error) { - return &pb.Uid{Str: fmt.Sprintf("%s-%s", s.plugin_name, s.socket_path)}, nil -} - -func (s *gRPCServer) FindSecretInfo(c context.Context, r *pb.FindRequest) (*pb.FindResult, error) { - jobs.DispatchScan(r) - return &pb.FindResult{}, nil -} - -func RunServer(ctx context.Context, socket_path string, plugin_name string) error { - - lis, err := net.Listen("unix", fmt.Sprintf("%s", socket_path)) +func (s *SecretGRPCServer) FindSecretInfo(c context.Context, r *pb.FindRequest) (*pb.FindResult, error) { + yaraScanner, err := s.YaraRules.NewScanner() if err != nil { - return err + return &pb.FindResult{}, err } - s := grpc.NewServer() - jobs.ScanMap = sync.Map{} - - impl := &gRPCServer{socket_path: socket_path, plugin_name: plugin_name} - pb.RegisterAgentPluginServer(s, impl) - pb.RegisterSecretScannerServer(s, impl) - pb.RegisterScannersServer(s, impl) - log.Infof("main: server listening at %v", lis.Addr()) go func() { - if err := s.Serve(lis); err != nil { - log.Errorf("server: %v", err) + logrus.Infof("request to scan %+v", r) + + namespace := "" + container := "" + image := "" + path := "" + switch { + case r.GetContainer() != nil: + namespace = r.GetContainer().GetNamespace() + container = r.GetContainer().GetId() + case r.GetImage() != nil: + image = r.GetImage().GetName() + default: + path = r.GetPath() } - }() - - <-ctx.Done() - s.GracefulStop() - log.Infof("main: exiting gracefully") - return nil + server.DoScan( + r.ScanId, + s.HostMountPath, + s.ExtractorConfig, + s.InactiveThreshold, + &s.ScanMap, + namespace, + path, + image, + container, + yaraScanner, + func(res out.IOCFound, scanID string) { + jobs.WriteSingleScanData(output.SecretToSecretInfo(res), scanID) + }, + ) + }() + return &pb.FindResult{}, nil }