use new grpc abstraction

Dockerfile

@@ -8,13 +8,49 @@ MAINTAINER DeepFence
 RUN apk update  \
     && apk add --upgrade gcc musl-dev pkgconfig g++ make git
 
-COPY --from=vectorscan /vectorscan.tar.bz2 /
-RUN tar -xjf /vectorscan.tar.bz2 -C / && rm /vectorscan.tar.bz2
+RUN apk add --no-cache \
+    git \
+    make  \
+    build-base \
+    pkgconfig \
+    libpcap-dev \
+    libcap-dev \
+    openssl-dev \
+    file \
+    jansson-dev \
+    jansson-static \
+    bison \
+    tini \
+    su-exec
+
+RUN apk add --no-cache -t .build-deps py-setuptools \
+    openssl-libs-static \
+    jansson-dev \
+    build-base \
+    libc-dev \
+    file-dev \
+    automake \
+    autoconf \
+    libtool \
+    libcrypto3 \
+    flex \
+    git \
+    libmagic-static \
+    linux-headers
+
+RUN cd /root && wget https://github.com/VirusTotal/yara/archive/refs/tags/v4.3.2.tar.gz \
+    && tar -zxf v4.3.2.tar.gz \
+    && cd yara-4.3.2 \
+    && ./bootstrap.sh \
+    && ./configure --prefix=/usr/local/yara --disable-dotnet --enable-magic --enable-cuckoo --disable-shared --enable-static\
+    && make \
+    && make install \
+    && cd /usr/local/ \
+    && tar -czf yara.tar.gz yara
 
 WORKDIR /home/deepfence/src/SecretScanner
 COPY . .
-RUN make clean
-RUN make
+RUN make clean && make all
 
 FROM alpine:3.18
 MAINTAINER DeepFence
@@ -30,7 +66,7 @@ RUN apk add --no-cache --upgrade tar libstdc++ libgcc docker skopeo bash podman
 RUN <<EOF
 set -eux
 
-apk update && apk add --no-cache --upgrade curl 
+apk update && apk add --no-cache --upgrade curl
 
 NERDCTL_VERSION=1.4.0
 curl -fsSLO https://github.com/containerd/nerdctl/releases/download/v${NERDCTL_VERSION}/nerdctl-${NERDCTL_VERSION}-linux-${TARGETARCH}.tar.gz

Makefile

@@ -6,10 +6,12 @@ bootstrap:
 clean:
 	-rm ./SecretScanner
 
-SecretScanner: $(PWD)/**/*.go $(PWD)/agent-plugins-grpc/**/*.go
+vendor: go.mod
 	go mod tidy -v
 	go mod vendor
-	go build -ldflags="-extldflags=-static" -buildvcs=false -v .
+
+SecretScanner: vendor $(PWD)/**/*.go $(PWD)/agent-plugins-grpc/**/*.go
+	CGO_LDFLAGS="-ljansson -lcrypto -lmagic" PKG_CONFIG_PATH=/usr/local/yara/lib/pkgconfig:$(PKG_CONFIG_PATH) go build -buildmode=pie -ldflags="-s -w -extldflags=-static" -buildvcs=false -v .
 
 .PHONY: clean bootstrap
 

go.mod

@@ -4,10 +4,8 @@ go 1.21.0
 
 replace github.com/deepfence/agent-plugins-grpc => ./agent-plugins-grpc
 
-replace github.com/deepfence/YaraHunter => ../YaraHunter
-
 require (
-	github.com/deepfence/YaraHunter v0.0.0-00010101000000-000000000000
+	github.com/deepfence/YaraHunter v0.0.0-20240708090804-4196e3bbd2c1
 	github.com/deepfence/agent-plugins-grpc v0.0.0-00010101000000-000000000000
 	github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240626143546-e4ec9311fdf9
 	github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20240626143546-e4ec9311fdf9

go.sum

@@ -34,6 +34,8 @@ github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3H
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/deepfence/YaraHunter v0.0.0-20240708090804-4196e3bbd2c1 h1:VuKkE8OKmM+hII/fViwYuLL1DTq5ncczMKWuOKg+EZc=
+github.com/deepfence/YaraHunter v0.0.0-20240708090804-4196e3bbd2c1/go.mod h1:SSZ34MXU0qR30p6/rJ/PpY7OgwljWF3go6K/ZfYqHZ0=
 github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240626143546-e4ec9311fdf9 h1:hI/Fv6XabkERGza4E8g7XLhlkuzWjoXQwTmtq0WIC+Y=
 github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240626143546-e4ec9311fdf9/go.mod h1:+rchMc4YNjCoHo0YAwKsT+DRBNr1hdDG0WrvAOOCc5k=
 github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20240626143546-e4ec9311fdf9 h1:/1olNPTiYUFxuYsP79DKLEEieNWIdPDFxs+B58jysRA=

jobs/common.go

@@ -6,8 +6,8 @@ import (
 )
 
 var (
-	scanStatusFilename = getDfInstallDir() + "/var/log/fenced/secret-scan-log/secret_scan_log.log"
-	scanFilename       = getDfInstallDir() + "/var/log/fenced/secret-scan/secret_scan.log"
+	scanStatusFilename = GetDfInstallDir() + "/var/log/fenced/secret-scan-log/secret_scan_log.log"
+	scanFilename       = GetDfInstallDir() + "/var/log/fenced/secret-scan/secret_scan.log"
 	SecretScanDir      = "/"
 )
 

jobs/scan.go

@@ -2,77 +2,15 @@ package jobs
 
 import (
 	"encoding/json"
-	"fmt"
 	"strings"
 	"sync"
-	"time"
-
-	"github.com/deepfence/SecretScanner/core"
-	"github.com/deepfence/SecretScanner/output"
-	"github.com/deepfence/SecretScanner/scan"
-	"github.com/deepfence/golang_deepfence_sdk/utils/tasks"
 
 	pb "github.com/deepfence/agent-plugins-grpc/srcgo"
-	cfg "github.com/deepfence/match-scanner/pkg/config"
 	log "github.com/sirupsen/logrus"
 )
 
 var ScanMap sync.Map
 
-func DispatchScan(r *pb.FindRequest) {
-	go func() {
-		startScanJob()
-		defer stopScanJob()
-
-		var err error
-		res, scanCtx := tasks.StartStatusReporter(
-			r.ScanId,
-			func(ss tasks.ScanStatus) error {
-				return writeSecretScanStatus(ss.ScanStatus, ss.ScanId, ss.ScanMessage)
-			},
-			tasks.StatusValues{
-				IN_PROGRESS: "IN_PROGRESS",
-				CANCELLED:   "CANCELLED",
-				FAILED:      "ERROR",
-				SUCCESS:     "COMPLETE",
-			},
-			time.Minute*20,
-		)
-
-		ScanMap.Store(r.ScanId, scanCtx)
-
-		defer func() {
-			ScanMap.Delete(r.ScanId)
-			res <- err
-			close(res)
-		}()
-
-		var (
-			scanType scan.ScanType
-			nodeID   string
-		)
-
-		if r.GetPath() != "" {
-			scanType = scan.DirScan
-			nodeID = r.GetPath()
-		} else if r.GetImage() != nil && r.GetImage().Name != "" {
-			scanType = scan.ImageScan
-			nodeID = r.GetImage().Name
-		} else if r.GetContainer() != nil && r.GetContainer().Id != "" {
-			scanType = scan.ContainerScan
-			nodeID = r.GetContainer().Id
-		} else {
-			err = fmt.Errorf("Invalid request")
-			return
-		}
-
-		filters := cfg.Config2Filter(core.GetSession().ExtractorConfig)
-		err = scan.Scan(scanCtx, scanType, filters, "", nodeID, r.GetScanId(), func(sf output.SecretFound, s string) {
-			writeSingleScanData(output.SecretToSecretInfo(sf), r.ScanId)
-		})
-	}()
-}
-
 type SecretScanDoc struct {
 	pb.SecretInfo
 	ScanID string `json:"scan_id,omitempty"`
@@ -100,7 +38,7 @@ func writeMultiScanData(secrets []*pb.SecretInfo, scan_id string) {
 	}
 }
 
-func writeSingleScanData(secret *pb.SecretInfo, scan_id string) {
+func WriteSingleScanData(secret *pb.SecretInfo, scan_id string) {
 	if SecretScanDir == HostMountDir {
 		secret.GetMatch().FullFilename = strings.Replace(secret.GetMatch().GetFullFilename(), SecretScanDir, "", 1)
 	}

jobs/status.go

@@ -41,7 +41,7 @@ func writeScanDataToFile(secretScanMsg string, filename string) error {
 	return nil
 }
 
-func getDfInstallDir() string {
+func GetDfInstallDir() string {
 	installDir, exists := os.LookupEnv("DF_INSTALL_DIR")
 	if exists {
 		return installDir

main.go

@@ -37,14 +37,20 @@ import (
 	"time"
 
 	"github.com/deepfence/SecretScanner/core"
+	"github.com/deepfence/SecretScanner/jobs"
 	"github.com/deepfence/SecretScanner/output"
 	"github.com/deepfence/SecretScanner/scan"
-	"github.com/deepfence/SecretScanner/signature"
+	"github.com/deepfence/SecretScanner/server"
 	"github.com/deepfence/golang_deepfence_sdk/utils/tasks"
 	"github.com/deepfence/match-scanner/pkg/config"
 	log "github.com/sirupsen/logrus"
+	"google.golang.org/grpc"
 
+	out "github.com/deepfence/YaraHunter/pkg/output"
 	"github.com/deepfence/YaraHunter/pkg/runner"
+	yaraserver "github.com/deepfence/YaraHunter/pkg/server"
+
+	pb "github.com/deepfence/agent-plugins-grpc/srcgo"
 )
 
 const (
@@ -171,8 +177,6 @@ func main() {
 			return "", " " + path.Base(f.File) + ":" + strconv.Itoa(f.Line)
 		},
 	})
-	// Process and store the read signatures
-	signature.ProcessSignatures(session.Config.Signatures)
 
 	flag.Parse()
 
@@ -182,6 +186,9 @@ func main() {
 
 	ctx, _ := signal.NotifyContext(context.Background(), os.Interrupt)
 
+	out.ScanStatusFilename = jobs.GetDfInstallDir() + "/var/log/fenced/secret-scan-log/secret_scan_log.log"
+	out.ScanFilename = jobs.GetDfInstallDir() + "/var/log/fenced/secret-scan/secret_scan.log"
+
 	runnerOpts := runner.RunnerOptions{
 		SocketPath:           *socketPath,
 		RulesPath:            *core.GetSession().Options.RulesPath,
@@ -206,5 +213,15 @@ func main() {
 		go runner.ScheduleYaraHunterUpdater(ctx, runnerOpts)
 	}
 
-	runner.StartYaraHunter(ctx, runnerOpts, core.GetSession().ExtractorConfig)
+	runner.StartYaraHunter(ctx, runnerOpts, core.GetSession().ExtractorConfig,
+
+		func(base *yaraserver.GRPCScannerServer) server.SecretGRPCServer {
+			return server.SecretGRPCServer{
+				GRPCScannerServer:                base,
+				UnimplementedSecretScannerServer: pb.UnimplementedSecretScannerServer{},
+			}
+		},
+		func(s grpc.ServiceRegistrar, impl any) {
+			pb.RegisterSecretScannerServer(s, impl.(pb.SecretScannerServer))
+		})
 }

output/output.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/deepfence/YaraHunter/pkg/output"
 	pb "github.com/deepfence/agent-plugins-grpc/srcgo"
 	"github.com/fatih/color"
 	tw "github.com/olekukonko/tablewriter"
@@ -180,33 +181,31 @@ func removeFirstLastChar(input string) string {
 	return input[1 : len(input)-1]
 }
 
-func SecretsToSecretInfos(out []SecretFound) []*pb.SecretInfo {
-	res := make([]*pb.SecretInfo, 0)
-	for _, v := range out {
-		res = append(res, SecretToSecretInfo(v))
+func SecretToSecretInfo(out output.IOCFound) *pb.SecretInfo {
+	matchedContent := ""
+	if len(out.Meta) != 0 {
+		matchedContent = out.Meta[0]
+	}
+	signature := ""
+	if len(out.StringsToMatch) != 0 {
+		signature = out.StringsToMatch[0]
+	}
+	severity := "low"
+	if out.Severity != "" {
+		severity = out.Severity
 	}
-	return res
-}
-
-func SecretToSecretInfo(out SecretFound) *pb.SecretInfo {
 	return &pb.SecretInfo{
 		ImageLayerId: out.LayerID,
 		Rule: &pb.MatchRule{
-			Id:               int32(out.RuleID),
 			Name:             out.RuleName,
-			Part:             out.PartToMatch,
-			StringToMatch:    out.Match,
-			SignatureToMatch: out.Regex,
+			SignatureToMatch: signature,
 		},
 		Match: &pb.Match{
-			StartingIndex:         int64(out.PrintBufferStartIndex),
-			RelativeStartingIndex: int64(out.MatchFromByte),
-			RelativeEndingIndex:   int64(out.MatchToByte),
-			FullFilename:          out.CompleteFilename,
-			MatchedContent:        jsonMarshal(out.MatchedContents),
+			FullFilename:   out.CompleteFilename,
+			MatchedContent: matchedContent,
 		},
 		Severity: &pb.Severity{
-			Level: out.Severity,
+			Level: severity,
 			Score: float32(out.SeverityScore),
 		},
 	}

rules/yara.rules

@@ -0,0 +1,11 @@
+rule github_personal_access_token {
+    meta:
+        description = "Rule to match GitHub Personal Access Tokens (classic), Fine-grained & Github Actions Token"
+        author = "deepfence.io"
+
+    strings:
+        $github_pat = /^gh[ps]_[a-zA-Z0-9]{36}|github_pat_[a-zA-Z0-9]{22}_[a-zA-Z0-9]{59}/
+
+    condition:
+        $github_pat
+}