add module update functional test, add hclog formatting fixes, disabl…

…e… (#10)

* add module update functional test, add hclog formating fixes, disable stacktraces, and caller logs, add ref updated for routes in the resource manager

* fix collection/namespace switching, and fix tests

* fix collection and namespace switching

functional-tests/admin_tests/change_checker.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -eo xtrace
+
+ADMIN_URL=${ADMIN_URL:-"http://localhost:9080"}
+PROXY_URL=${PROXY_URL:-"http://localhost"}
+
+DIR="$( cd "$( dirname "$0" )" && pwd )"
+
+export DGATE_ADMIN_API=$ADMIN_URL
+
+dgate-cli namespace create \
+    name=change_checker-ns
+
+dgate-cli domain create \
+    name=change_checker-dm \
+    patterns:='["change_checker.com"]' \
+    namespace=change_checker-ns
+
+dgate-cli module create name=change_checker-mod \
+    payload@=$DIR/change_checker_1.ts \
+    namespace=change_checker-ns
+
+dgate-cli route create \
+    name=base_rt paths:='["/", "/{id}"]' \
+    modules:='["change_checker-mod"]' \
+    methods:='["GET","POST"]' \
+    stripPath:=true \
+    preserveHost:=true \
+    namespace=change_checker-ns
+
+MODID1=$(curl -sG -H Host:change_checker.com ${PROXY_URL}/ | jq -r '.mod')
+
+if [ "$MODID1" != "module1" ]; then
+    echo "Initial assert failed"
+    exit 1
+fi
+
+
+dgate-cli module create name=change_checker-mod \
+    payload@=$DIR/change_checker_2.ts \
+    namespace=change_checker-ns
+
+# dgate-cli r.ker-ns
+
+MODID2=$(curl -sG -H Host:change_checker.com ${PROXY_URL}/ | jq -r '.mod')
+
+if [ "$MODID2" != "module2" ]; then
+    echo "module update failed"
+    exit 1
+fi

functional-tests/admin_tests/change_checker_1.ts

@@ -0,0 +1,6 @@
+
+export const requestHandler = async (ctx: any) => {
+    ctx.response().status(201).json({
+        mod: "module1",
+    });
+}

functional-tests/admin_tests/change_checker_2.ts

@@ -0,0 +1,6 @@
+
+export const requestHandler = async (ctx: any) => {
+    ctx.response().status(201).json({
+        mod: "module2",
+    });
+};

functional-tests/admin_tests/modify_request.ts

@@ -1,4 +1,6 @@
+// @ts-ignore
 import { fetch } from "dgate/http";
+// @ts-ignore
 import { getCache, setCache } from "dgate/storage";
 
 export const requestModifier = async (ctx) => {
@@ -23,9 +25,10 @@ export const requestModifier = async (ctx) => {
             console.error(JSON.stringify(georesp));
             throw new Error(("IP API: " + geodata?.message) ?? "Failed to fetch geodata");
         }
-        setCache('geodata:'+remoteAddr, geodata, {
-            ttl: 3600,
-        });
+
+        // setCache('geodata:'+remoteAddr, geodata, {
+        //     ttl: 3600,
+        // });
     }
 
     req.headers.set("X-Geo-Country", geodata.country);

functional-tests/admin_tests/performance_test_prep.ts

@@ -1,3 +1,4 @@
+// @ts-ignore
 import { sleep } from "dgate";
 
 export const fetchUpstream = async (ctx) =>

functional-tests/admin_tests/url_shortener.ts

@@ -13,7 +13,7 @@ export const requestHandler = (ctx: any) => {
             return;
         }
         // get the document with the ID from the collection
-        return getDocument("short_link", pathId)
+        return getDocument(pathId, "short_link")
             .then((doc: any) => {
                 // check if the document contains the URL
                 if (!doc?.data?.url) {

functional-tests/raft_tests/test1.yaml

@@ -1,5 +1,4 @@
 version: v1
-debug: true
 log_level: info
 
 tags:

functional-tests/raft_tests/test2.yaml

@@ -1,5 +1,4 @@
 version: v1
-debug: true
 log_level: info
 tags:
  - "dev"

internal/admin/admin_raft.go

@@ -30,7 +30,6 @@ func setupRaft(
 	adminConfig := conf.AdminConfig
 	var sstore raft.StableStore
 	var lstore raft.LogStore
-	snapstore := raft.NewInmemSnapshotStore()
 	switch conf.Storage.StorageType {
 	case config.StorageTypeMemory:
 		sstore = raft.NewInmemStore()
@@ -55,6 +54,9 @@ func setupRaft(
 		panic(fmt.Errorf("invalid storage type: %s", conf.Storage.StorageType))
 	}
 	raftId := adminConfig.Replication.RaftID
+	if raftId == "" {
+		raftId = conf.NodeId
+	}
 
 	raftConfig := adminConfig.Replication.LoadRaftConfig(
 		&raft.Config{
@@ -90,7 +92,7 @@ func setupRaft(
 	)
 	raftNode, err := raft.NewRaft(
 		raftConfig, newDGateAdminFSM(logger.Named("fsm"), cs),
-		lstore, sstore, snapstore, transport,
+		lstore, sstore, raft.NewInmemSnapshotStore(), transport,
 	)
 	if err != nil {
 		panic(err)

internal/admin/routes/collection_routes.go

@@ -48,7 +48,10 @@ func ConfigureCollectionAPI(server chi.Router, logger *zap.Logger, cs changestat
 		if oldCollection, ok := rm.GetCollection(collection.Name, collection.NamespaceName); ok {
 			if oldCollection.Type == spec.CollectionTypeDocument {
 				docs, err := dm.GetDocuments(
-					collection.NamespaceName, collection.Name, 0, 0)
+					collection.Name,
+					collection.NamespaceName,
+					0, 0,
+				)
 				if err != nil {
 					util.JsonError(w, http.StatusInternalServerError, err.Error())
 					return
@@ -142,7 +145,7 @@ func ConfigureCollectionAPI(server chi.Router, logger *zap.Logger, cs changestat
 			util.JsonError(w, http.StatusBadRequest, "offset must be an integer")
 			return
 		}
-		docs, err := dm.GetDocuments(namespaceName, collectionName, offset, limit)
+		docs, err := dm.GetDocuments(collectionName, namespaceName, offset, limit)
 		if err != nil {
 			util.JsonError(w, http.StatusInternalServerError, err.Error())
 			return
@@ -194,7 +197,7 @@ func ConfigureCollectionAPI(server chi.Router, logger *zap.Logger, cs changestat
 			return
 		}
 
-		document, err := dm.GetDocumentByID(namespaceName, collectionName, documentId)
+		document, err := dm.GetDocumentByID(documentId, collectionName, namespaceName)
 		if err != nil {
 			util.JsonError(w, http.StatusNotFound, err.Error())
 			return
@@ -348,7 +351,7 @@ func ConfigureCollectionAPI(server chi.Router, logger *zap.Logger, cs changestat
 			util.JsonError(w, http.StatusBadRequest, "document_id is required")
 			return
 		}
-		document, err := dm.GetDocumentByID(namespaceName, collectionName, documentId)
+		document, err := dm.GetDocumentByID(documentId, collectionName, namespaceName)
 		if err != nil {
 			util.JsonError(w, http.StatusNotFound, err.Error())
 			return
@@ -386,7 +389,10 @@ func ConfigureCollectionAPI(server chi.Router, logger *zap.Logger, cs changestat
 		}
 		if collection.Type == spec.CollectionTypeDocument {
 			docs, err := dm.GetDocuments(
-				namespaceName, collectionName, 0, 1)
+				collectionName,
+				namespaceName,
+				1, 1,
+			)
 			if err != nil {
 				util.JsonError(w, http.StatusInternalServerError, err.Error())
 				return

internal/admin/routes/module_routes_test.go

@@ -22,7 +22,7 @@ import (
 func TestAdminRoutes_Module(t *testing.T) {
 	namespaces := []string{"default", "test"}
 	for _, ns := range namespaces {
-		config := configtest.NewTest3DGateConfig()
+		config := configtest.NewTest4DGateConfig()
 		ps := proxy.NewProxyState(zap.NewNop(), config)
 		mux := chi.NewMux()
 		mux.Route("/api/v1", func(r chi.Router) {

internal/admin/routes/service_routes_test.go

@@ -21,7 +21,7 @@ import (
 func TestAdminRoutes_Service(t *testing.T) {
 	namespaces := []string{"default", "test"}
 	for _, ns := range namespaces {
-		config := configtest.NewTest3DGateConfig()
+		config := configtest.NewTest4DGateConfig()
 		ps := proxy.NewProxyState(zap.NewNop(), config)
 		mux := chi.NewMux()
 		mux.Route("/api/v1", func(r chi.Router) {

internal/config/config.go

@@ -226,6 +226,8 @@ func (conf *DGateConfig) GetLogger() (*zap.Logger, error) {
 	if conf.Logging.ZapConfig == nil {
 		config := zap.NewProductionConfig()
 		config.Level = level
+		config.DisableCaller = true
+		config.DisableStacktrace = true
 		config.Development = conf.Debug
 		config.EncoderConfig.EncodeTime = zapcore.RFC3339TimeEncoder
 		config.OutputPaths = []string{"stdout"}

internal/config/configtest/dgate_configs.go

@@ -107,6 +107,23 @@ func NewTest3DGateConfig() *config.DGateConfig {
 	return conf
 }
 
+func NewTest4DGateConfig() *config.DGateConfig {
+	conf := NewTestDGateConfig()
+	conf.DisableDefaultNamespace = false
+	conf.ProxyConfig = config.DGateProxyConfig{
+		Host: "localhost",
+		Port: 16436,
+		InitResources: &config.DGateResources{
+			Namespaces: []spec.Namespace{
+				{
+					Name: "test",
+				},
+			},
+		},
+	}
+	return conf
+}
+
 func NewTestDGateConfig_DomainAndNamespaces() *config.DGateConfig {
 	conf := NewTestDGateConfig()
 	conf.ProxyConfig.InitResources.Namespaces = []spec.Namespace{

internal/proxy/change_log.go

@@ -86,7 +86,7 @@ func (ps *ProxyState) processChangeLog(cl *spec.ChangeLog, reload, store bool) (
 		}
 	}
 	if reload {
-		if cl.Cmd.Resource().IsRelatedTo(spec.Routes) || cl.Cmd.IsNoop() {
+		if cl.Cmd.IsNoop() || cl.Cmd.Resource().IsRelatedTo(spec.Routes) {
 			ps.logger.Debug("Registering change log", zap.Stringer("cmd", cl.Cmd))
 			err = ps.reconfigureState(false, cl)
 			if err != nil {

internal/proxy/dynamic_proxy.go

@@ -11,7 +11,6 @@ import (
 	"github.com/dgate-io/dgate/pkg/modules/extractors"
 	"github.com/dgate-io/dgate/pkg/spec"
 	"github.com/dgate-io/dgate/pkg/typescript"
-	"github.com/dgate-io/dgate/pkg/util/tree/avl"
 	"github.com/dop251/goja"
 	"go.uber.org/zap"
 	"golang.org/x/net/http2"
@@ -103,48 +102,48 @@ func (ps *ProxyState) setupModules() error {
 
 func (ps *ProxyState) setupRoutes() (err error) {
 	ps.logger.Debug("Setting up routes")
-	reqCtxProviders := avl.NewTree[string, *RequestContextProvider]()
+	// reqCtxProviders := avl.NewTree[string, *RequestContextProvider]()
 	for namespaceName, routes := range ps.rm.GetRouteNamespaceMap() {
 		mux := router.NewMux()
-		for _, r := range routes {
-			reqCtxProvider := NewRequestContextProvider(r, ps)
-			reqCtxProviders.Insert(r.Namespace.Name+"/"+r.Name, reqCtxProvider)
-			if len(r.Modules) > 0 {
-				modPool, err := NewModulePool(
-					256, 1024, reqCtxProvider,
-					ps.createModuleExtractorFunc(r),
-				)
-				if err != nil {
+		for _, rt := range routes {
+			reqCtxProvider := NewRequestContextProvider(rt, ps)
+			if len(rt.Modules) > 0 {
+				modExtFunc := ps.createModuleExtractorFunc(rt)
+				if modPool, err := NewModulePool(
+					256, 1024, reqCtxProvider, modExtFunc,
+				); err != nil {
 					ps.logger.Error("Error creating module buffer", zap.Error(err))
 					return err
+				} else {
+					reqCtxProvider.SetModulePool(modPool)
 				}
-				reqCtxProvider.SetModulePool(modPool)
 			}
-			err = func() (err error) {
+			ps.providers.Insert(rt.Namespace.Name+"/"+rt.Name, reqCtxProvider)
+			err = func(rt *spec.DGateRoute) (err error) {
 				defer func() {
 					if r := recover(); r != nil {
 						err = fmt.Errorf("%v", r)
 					}
 				}()
-				for _, path := range r.Paths {
-					if len(r.Methods) > 0 && r.Methods[0] == "*" {
-						if len(r.Methods) > 1 {
+				for _, path := range rt.Paths {
+					if len(rt.Methods) > 0 && rt.Methods[0] == "*" {
+						if len(rt.Methods) > 1 {
 							return errors.New("route methods cannot have other methods with *")
 						}
 						mux.Handle(path, ps.HandleRoute(reqCtxProvider, path))
 					} else {
-						if len(r.Methods) == 0 {
+						if len(rt.Methods) == 0 {
 							return errors.New("route must have at least one method")
-						} else if err = ValidateMethods(r.Methods); err != nil {
+						} else if err = ValidateMethods(rt.Methods); err != nil {
 							return err
 						}
-						for _, method := range r.Methods {
+						for _, method := range rt.Methods {
 							mux.Method(method, path, ps.HandleRoute(reqCtxProvider, path))
 						}
 					}
 				}
 				return nil
-			}()
+			}(rt)
 		}
 
 		ps.logger.Debug("Routes have changed, reloading")
@@ -158,18 +157,18 @@ func (ps *ProxyState) setupRoutes() (err error) {
 	return
 }
 
-func (ps *ProxyState) createModuleExtractorFunc(r *spec.DGateRoute) ModuleExtractorFunc {
+func (ps *ProxyState) createModuleExtractorFunc(rt *spec.DGateRoute) ModuleExtractorFunc {
 	return func(reqCtx *RequestContextProvider) (_ ModuleExtractor, err error) {
-		if len(r.Modules) == 0 {
-			return nil, fmt.Errorf("no modules found for route: %s/%s", r.Name, r.Namespace.Name)
+		if len(rt.Modules) == 0 {
+			return nil, fmt.Errorf("no modules found for route: %s/%s", rt.Name, rt.Namespace.Name)
 		}
 		// TODO: Perhaps have some entrypoint flag to determine which module to use
-		m := r.Modules[0]
-		if program, ok := ps.modPrograms.Find(m.Name + "/" + r.Namespace.Name); !ok {
+		m := rt.Modules[0]
+		if program, ok := ps.modPrograms.Find(m.Name + "/" + rt.Namespace.Name); !ok {
 			ps.logger.Error("Error getting module program: invalid state", zap.Error(err))
-			return nil, fmt.Errorf("cannot find module program: %s/%s", m.Name, r.Namespace.Name)
+			return nil, fmt.Errorf("cannot find module program: %s/%s", m.Name, rt.Namespace.Name)
 		} else {
-			rtCtx := NewRuntimeContext(ps, r, r.Modules...)
+			rtCtx := NewRuntimeContext(ps, rt, rt.Modules...)
 			if err := extractors.SetupModuleEventLoop(ps.printer, rtCtx, program); err != nil {
 				ps.logger.Error("Error creating runtime for route",
 					zap.String("route", reqCtx.route.Name),

internal/proxy/proxy_documents.go

@@ -15,19 +15,19 @@ func (ps *ProxyState) GetDocuments(collection, namespace string, limit, offset i
 	if _, ok := ps.rm.GetNamespace(namespace); !ok {
 		return nil, spec.ErrNamespaceNotFound(namespace)
 	}
-	if _, ok := ps.rm.GetCollection(namespace, collection); !ok {
+	if _, ok := ps.rm.GetCollection(collection, namespace); !ok {
 		return nil, spec.ErrCollectionNotFound(collection)
 	}
 	return ps.store.FetchDocuments(namespace, collection, limit, offset)
 }
 
 // GetDocumentByID is a function that returns a document in a collection by its ID.
-func (ps *ProxyState) GetDocumentByID(namespace, collection, docId string) (*spec.Document, error) {
+func (ps *ProxyState) GetDocumentByID(docId, collection, namespace string) (*spec.Document, error) {
 	if _, ok := ps.rm.GetNamespace(namespace); !ok {
 		return nil, spec.ErrNamespaceNotFound(namespace)
 	}
 	if _, ok := ps.rm.GetCollection(collection, namespace); !ok {
 		return nil, spec.ErrCollectionNotFound(collection)
 	}
-	return ps.store.FetchDocument(namespace, collection, docId)
+	return ps.store.FetchDocument(docId, collection, namespace)
 }