feat: add ngqa and RMSNormEps parameters (mudler#860)

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
EvilFreelancer · Aug 2, 2023 · 5ca21ee · 5ca21ee
1 parent 953e308
commit 5ca21ee
Show file tree

Hide file tree

Showing 7 changed files with 138 additions and 104 deletions.
diff --git a/Makefile b/Makefile
@@ -4,7 +4,7 @@ GOVET=$(GOCMD) vet
 BINARY_NAME=local-ai
 
 # llama.cpp versions
-GOLLAMA_VERSION?=6ba16de8e965e5aa0f32d25ef9d6149bb6586565
+GOLLAMA_VERSION?=50cee7712066d9e38306eccadcfbb44ea87df4b7
 
 # gpt4all version
 GPT4ALL_REPO?=https://github.com/nomic-ai/gpt4all

diff --git a/api/backend/options.go b/api/backend/options.go
@@ -18,6 +18,8 @@ func gRPCModelOpts(c config.Config) *pb.ModelOptions {
 		ContextSize:   int32(c.ContextSize),
 		Seed:          int32(c.Seed),
 		NBatch:        int32(b),
+		NGQA:          c.NGQA,
+		RMSNormEps:    c.RMSNormEps,
 		F16Memory:     c.F16,
 		MLock:         c.MMlock,
 		RopeFreqBase:  c.RopeFreqBase,

diff --git a/api/config/config.go b/api/config/config.go
@@ -51,6 +51,9 @@ type Config struct {
 	FunctionsConfig Functions `yaml:"function"`
 
 	SystemPrompt string `yaml:"system_prompt"`
+
+	RMSNormEps float32 `yaml:"rms_norm_eps"`
+	NGQA       int32   `yaml:"ngqa"`
 }
 
 type Functions struct {

diff --git a/extra/grpc/huggingface/backend_pb2.py b/extra/grpc/huggingface/backend_pb2.py
diff --git a/pkg/grpc/llm/llama/llama.go b/pkg/grpc/llm/llama/llama.go
@@ -33,6 +33,14 @@ func (llm *LLM) Load(opts *pb.ModelOptions) error {
 		llama.WithRopeFreqScale(ropeFreqScale),
 	}
 
+	if opts.NGQA != 0 {
+		llamaOpts = append(llamaOpts, llama.WithGQA(int(opts.NGQA)))
+	}
+
+	if opts.RMSNormEps != 0 {
+		llamaOpts = append(llamaOpts, llama.WithRMSNormEPS(opts.RMSNormEps))
+	}
+
 	if opts.ContextSize != 0 {
 		llamaOpts = append(llamaOpts, llama.SetContext(int(opts.ContextSize)))
 	}