feat: otel tracing (#899)

api/build/approve.go

@@ -3,6 +3,7 @@
 package build
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"strings"
@@ -110,7 +111,7 @@ func ApproveBuild(c *gin.Context) {
 
 	// publish the build to the queue
 	go Enqueue(
-		ctx,
+		context.WithoutCancel(ctx),
 		queue.FromGinContext(c),
 		database.FromContext(c),
 		models.ToItem(b),

api/build/create.go

@@ -3,6 +3,7 @@
 package build
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 
@@ -149,7 +150,7 @@ func CreateBuild(c *gin.Context) {
 
 	// publish the build to the queue
 	go Enqueue(
-		ctx,
+		context.WithoutCancel(ctx),
 		queue.FromGinContext(c),
 		database.FromContext(c),
 		item,

api/build/restart.go

@@ -3,6 +3,7 @@
 package build
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"strings"
@@ -157,7 +158,7 @@ func RestartBuild(c *gin.Context) {
 
 	// publish the build to the queue
 	go Enqueue(
-		ctx,
+		context.WithoutCancel(ctx),
 		queue.FromGinContext(c),
 		database.FromContext(c),
 		item,

api/webhook/post.go

@@ -489,8 +489,6 @@ func PostWebhook(c *gin.Context) {
 		}
 	}
 
-	c.JSON(http.StatusCreated, b)
-
 	// regardless of whether the build is published to queue, we want to attempt to auto-cancel if no errors
 	defer func() {
 		if err == nil && build.ShouldAutoCancel(p.Metadata.AutoCancel, b, repo.GetBranch()) {
@@ -531,6 +529,10 @@ func PostWebhook(c *gin.Context) {
 		}
 	}()
 
+	// track if we have already responded to the http request
+	// helps prevent multiple responses to the same request in the event of errors
+	responded := false
+
 	// if the webhook was from a Pull event from a forked repository, verify it is allowed to run
 	if webhook.PullRequest.IsFromFork {
 		l.Tracef("inside %s workflow for fork PR build %s/%d", repo.GetApproveBuild(), repo.GetFullName(), b.GetNumber())
@@ -540,6 +542,8 @@ func PostWebhook(c *gin.Context) {
 			err = gatekeepBuild(c, b, repo)
 			if err != nil {
 				util.HandleError(c, http.StatusInternalServerError, err)
+			} else {
+				c.JSON(http.StatusCreated, b)
 			}
 
 			return
@@ -550,6 +554,8 @@ func PostWebhook(c *gin.Context) {
 				err = gatekeepBuild(c, b, repo)
 				if err != nil {
 					util.HandleError(c, http.StatusInternalServerError, err)
+				} else {
+					c.JSON(http.StatusCreated, b)
 				}
 
 				return
@@ -564,12 +570,16 @@ func PostWebhook(c *gin.Context) {
 			contributor, err := scm.FromContext(c).RepoContributor(ctx, repo.GetOwner(), b.GetSender(), repo.GetOrg(), repo.GetName())
 			if err != nil {
 				util.HandleError(c, http.StatusInternalServerError, err)
+
+				responded = true
 			}
 
 			if !contributor {
 				err = gatekeepBuild(c, b, repo)
 				if err != nil {
 					util.HandleError(c, http.StatusInternalServerError, err)
+				} else if !responded {
+					c.JSON(http.StatusCreated, b)
 				}
 
 				return
@@ -591,12 +601,17 @@ func PostWebhook(c *gin.Context) {
 
 	// publish the build to the queue
 	go build.Enqueue(
-		ctx,
+		context.WithoutCancel(ctx),
 		queue.FromGinContext(c),
 		database.FromContext(c),
 		item,
 		b.GetHost(),
 	)
+
+	// respond only when necessary
+	if !responded {
+		c.JSON(http.StatusCreated, b)
+	}
 }
 
 // handleRepositoryEvent is a helper function that processes repository events from the SCM and updates

cmd/vela-server/main.go

@@ -17,6 +17,7 @@ import (
 	"github.com/go-vela/server/queue"
 	"github.com/go-vela/server/scm"
 	"github.com/go-vela/server/secret"
+	"github.com/go-vela/server/tracing"
 	"github.com/go-vela/server/version"
 	"github.com/go-vela/types/constants"
 )
@@ -279,6 +280,9 @@ func main() {
 	// Add Source Flags
 	app.Flags = append(app.Flags, scm.Flags...)
 
+	// Add Tracing Flags
+	app.Flags = append(app.Flags, tracing.Flags...)
+
 	if err = app.Run(os.Args); err != nil {
 		logrus.Fatal(err)
 	}

cmd/vela-server/schedule.go

@@ -226,7 +226,7 @@ func processSchedule(ctx context.Context, s *api.Schedule, settings *settings.Pl
 
 	// publish the build to the queue
 	go build.Enqueue(
-		ctx,
+		context.WithoutCancel(ctx),
 		queue,
 		database,
 		item,

cmd/vela-server/scm.go

@@ -7,10 +7,11 @@ import (
 	"github.com/urfave/cli/v2"
 
 	"github.com/go-vela/server/scm"
+	"github.com/go-vela/server/tracing"
 )
 
 // helper function to setup the scm from the CLI arguments.
-func setupSCM(c *cli.Context) (scm.Service, error) {
+func setupSCM(c *cli.Context, tc *tracing.Client) (scm.Service, error) {
 	logrus.Debug("creating scm client from CLI configuration")
 
 	// scm configuration
@@ -24,6 +25,7 @@ func setupSCM(c *cli.Context) (scm.Service, error) {
 		StatusContext:        c.String("scm.context"),
 		WebUIAddress:         c.String("webui-addr"),
 		Scopes:               c.StringSlice("scm.scopes"),
+		Tracing:              tc,
 	}
 
 	// setup the scm

cmd/vela-server/server.go

@@ -26,6 +26,7 @@ import (
 	"github.com/go-vela/server/queue"
 	"github.com/go-vela/server/router"
 	"github.com/go-vela/server/router/middleware"
+	"github.com/go-vela/server/tracing"
 )
 
 //nolint:funlen,gocyclo // ignore function length and cyclomatic complexity
@@ -78,7 +79,21 @@ func server(c *cli.Context) error {
 		return err
 	}
 
-	database, err := database.FromCLIContext(c)
+	tc, err := tracing.FromCLIContext(c)
+	if err != nil {
+		return err
+	}
+
+	if tc.EnableTracing {
+		defer func() {
+			err := tc.TracerProvider.Shutdown(context.Background())
+			if err != nil {
+				logrus.Errorf("unable to shutdown tracer provider: %v", err)
+			}
+		}()
+	}
+
+	database, err := database.FromCLIContext(c, tc)
 	if err != nil {
 		return err
 	}
@@ -93,7 +108,7 @@ func server(c *cli.Context) error {
 		return err
 	}
 
-	scm, err := setupSCM(c)
+	scm, err := setupSCM(c, tc)
 	if err != nil {
 		return err
 	}
@@ -189,6 +204,8 @@ func server(c *cli.Context) error {
 		middleware.DefaultRepoEventsMask(c.Int64("default-repo-events-mask")),
 		middleware.DefaultRepoApproveBuild(c.String("default-repo-approve-build")),
 		middleware.ScheduleFrequency(c.Duration("schedule-minimum-frequency")),
+		middleware.TracingClient(tc),
+		middleware.TracingInstrumentation(tc),
 	)
 
 	addr, err := url.Parse(c.String("server-addr"))

database/context.go

@@ -7,6 +7,8 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
+
+	"github.com/go-vela/server/tracing"
 )
 
 const key = "database"
@@ -38,7 +40,7 @@ func ToContext(c Setter, d Interface) {
 }
 
 // FromCLIContext creates and returns a database engine from the urfave/cli context.
-func FromCLIContext(c *cli.Context) (Interface, error) {
+func FromCLIContext(c *cli.Context, tc *tracing.Client) (Interface, error) {
 	logrus.Debug("creating database engine from CLI configuration")
 
 	return New(
@@ -54,5 +56,6 @@ func FromCLIContext(c *cli.Context) (Interface, error) {
 		WithLogSlowThreshold(c.Duration("database.log.slow_threshold")),
 		WithLogShowSQL(c.Bool("database.log.show_sql")),
 		WithSkipCreation(c.Bool("database.skip_creation")),
+		WithTracing(tc),
 	)
 }

database/context_test.go

@@ -10,6 +10,8 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/urfave/cli/v2"
+
+	"github.com/go-vela/server/tracing"
 )
 
 func TestDatabase_FromContext(t *testing.T) {
@@ -132,7 +134,7 @@ func TestDatabase_FromCLIContext(t *testing.T) {
 	// run tests
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			_, err := FromCLIContext(test.context)
+			_, err := FromCLIContext(test.context, &tracing.Client{Config: tracing.Config{EnableTracing: false}})
 
 			if test.failure {
 				if err == nil {

database/database.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/sirupsen/logrus"
+	"github.com/uptrace/opentelemetry-go-extra/otelgorm"
 	"gorm.io/driver/postgres"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
@@ -28,6 +29,7 @@ import (
 	"github.com/go-vela/server/database/step"
 	"github.com/go-vela/server/database/user"
 	"github.com/go-vela/server/database/worker"
+	"github.com/go-vela/server/tracing"
 	"github.com/go-vela/types/constants"
 )
 
@@ -70,6 +72,8 @@ type (
 		ctx context.Context
 		// sirupsen/logrus logger used in database functions
 		logger *logrus.Entry
+		// configurations related to telemetry/tracing
+		tracing *tracing.Client
 
 		settings.SettingsInterface
 		build.BuildInterface
@@ -105,7 +109,7 @@ func New(opts ...EngineOpt) (Interface, error) {
 	e.client = new(gorm.DB)
 	e.config = new(config)
 	e.logger = new(logrus.Entry)
-	e.ctx = context.TODO()
+	e.ctx = context.Background()
 
 	// apply all provided configuration options
 	for _, opt := range opts {
@@ -191,6 +195,19 @@ func New(opts ...EngineOpt) (Interface, error) {
 		return nil, err
 	}
 
+	// initialize otel tracing if enabled
+	if e.tracing.EnableTracing {
+		otelPlugin := otelgorm.NewPlugin(
+			otelgorm.WithTracerProvider(e.tracing.TracerProvider),
+			otelgorm.WithoutQueryVariables(),
+		)
+
+		err := e.client.Use(otelPlugin)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// set the maximum amount of time a connection may be reused
 	db.SetConnMaxLifetime(e.config.ConnectionLife)
 	// set the maximum number of connections in the idle connection pool
@@ -230,5 +247,6 @@ func NewTest() (Interface, error) {
 		WithLogShowSQL(false),
 		WithLogSkipNotFound(true),
 		WithLogSlowThreshold(200*time.Millisecond),
+		WithTracing(&tracing.Client{Config: tracing.Config{EnableTracing: false}}),
 	)
 }

database/database_test.go

@@ -11,6 +11,8 @@ import (
 	"gorm.io/driver/postgres"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
+
+	"github.com/go-vela/server/tracing"
 )
 
 func TestDatabase_New(t *testing.T) {
@@ -110,6 +112,7 @@ func TestDatabase_New(t *testing.T) {
 				WithLogSlowThreshold(test.config.LogSlowThreshold),
 				WithEncryptionKey(test.config.EncryptionKey),
 				WithSkipCreation(test.config.SkipCreation),
+				WithTracing(&tracing.Client{Config: tracing.Config{EnableTracing: false}}),
 			)
 
 			if test.failure {

database/integration_test.go

@@ -33,6 +33,7 @@ import (
 	"github.com/go-vela/server/database/testutils"
 	"github.com/go-vela/server/database/user"
 	"github.com/go-vela/server/database/worker"
+	"github.com/go-vela/server/tracing"
 	"github.com/go-vela/types/constants"
 	"github.com/go-vela/types/library"
 	"github.com/go-vela/types/raw"
@@ -114,6 +115,7 @@ func TestDatabase_Integration(t *testing.T) {
 				WithDriver(test.config.Driver),
 				WithEncryptionKey(test.config.EncryptionKey),
 				WithSkipCreation(test.config.SkipCreation),
+				WithTracing(&tracing.Client{Config: tracing.Config{EnableTracing: false}}),
 			)
 			if err != nil {
 				t.Errorf("unable to create new database engine for %s: %v", test.name, err)

database/opts.go

@@ -5,6 +5,8 @@ package database
 import (
 	"context"
 	"time"
+
+	"github.com/go-vela/server/tracing"
 )
 
 // EngineOpt represents a configuration option to initialize the database engine.
@@ -130,6 +132,15 @@ func WithSkipCreation(skipCreation bool) EngineOpt {
 	}
 }
 
+// WithTracing sets the shared tracing config in the database engine.
+func WithTracing(tracing *tracing.Client) EngineOpt {
+	return func(e *engine) error {
+		e.tracing = tracing
+
+		return nil
+	}
+}
+
 // WithContext sets the context in the database engine.
 func WithContext(ctx context.Context) EngineOpt {
 	return func(e *engine) error {