Instrument plugins (getporter#2026)

* Plumb context.Context for the plugins

I wanted to add tracing to the plugins before working on the migration
(which will do being extreme plugin shenanigans so we'll need the extra
logs there). This is a first pass to add context.Context to all
functions that lead to the plugins.

Signed-off-by: Carolyn Van Slyck <me@carolynvanslyck.com>

* Fix integration test use of namespace

The test wasn't setting namespace and was creating data in the global
namespace.

Signed-off-by: Carolyn Van Slyck <me@carolynvanslyck.com>

* Add plugin adapter for secrets

Signed-off-by: Carolyn Van Slyck <me@carolynvanslyck.com>

* Instrument plugin execution

This has lots of semi-related fixes for everything required to get a
plugin instrumented.

* Add support for creating secondary tracers with a different service name
set when interacting with the plugins. In Jaeger, you will see a span
tree for "porter", and now another span tree for each plugin that it
interacts with, e.g. "mongodb", or "azure.storage.blob".
* Support grpc in the storage plugin. Not sure yet if we are keeping
  net/rpc. Only grpc plugins can be instrumented, net/rpc doesn't have a
  mechanism for propogating the context.
* Change storage protocol to include context.Context, and to use
  concrete types instead of interface{} when sending filters, sorts,
  etc.
* Run all plugins through the hashicorp/go-plugin framework. I was
  running internal plugins in-process which meant that I didn't notice
  when we broke external storage plugins. It also was more work to
  instrument a new separate service name when I had to do it both for
  external and internal plugins. This makes the code paths the same
  regardless of who wrote the plugin.
* Improve error handling when serving internal plugins. All errors must
  be logged with hclog, and we must not write to stdout/err as that
  causes the connection to the plugin to fail.
* Dump logs from the plugins into a span as a fallback mechanism to
  collect info from an uninstrumented plugin.
* Set porter into "plugin mode" and have porter log with hclog

Signed-off-by: Carolyn Van Slyck <me@carolynvanslyck.com>

* Standardize Close function

I've made the following changes to make our Close functions consistent:

* Rename Teardown to Close.
* Do not pass a context.Context, use context.Background() instead.
* Make sure if we are passed an interface, that we try to call close.

Signed-off-by: Carolyn Van Slyck <me@carolynvanslyck.com>

* Make context.Context the first parameter

Signed-off-by: Carolyn Van Slyck <me@carolynvanslyck.com>

* Review feedback

Signed-off-by: Carolyn Van Slyck <me@carolynvanslyck.com>

* Review feedback

Signed-off-by: Carolyn Van Slyck <me@carolynvanslyck.com>

* tik tok

Signed-off-by: Carolyn Van Slyck <me@carolynvanslyck.com>

* Update pkg/storage/plugins/mongodb_docker/store.go

Signed-off-by: Carolyn Van Slyck <me@carolynvanslyck.com>

Co-authored-by: Yingrong Zhao <yingrong.zhao@gmail.com>

* Fix tests after merging with release/v1

Signed-off-by: Carolyn Van Slyck <me@carolynvanslyck.com>

Co-authored-by: Yingrong Zhao <yingrong.zhao@gmail.com>

cmd/exec/build.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 
 	"get.porter.sh/porter/pkg/exec"
-
 	"github.com/spf13/cobra"
 )
 

cmd/porter/bundle.go

@@ -4,9 +4,8 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/spf13/cobra"
-
 	"get.porter.sh/porter/pkg/porter"
+	"github.com/spf13/cobra"
 )
 
 func buildBundleCommands(p *porter.Porter) *cobra.Command {

cmd/porter/bundle_test.go

@@ -157,7 +157,7 @@ func TestBuildValidate_Driver(t *testing.T) {
 			}
 
 			p := porter.NewTestPorter(t)
-			defer p.Teardown()
+			defer p.Close()
 
 			rootCmd := buildRootCommandFrom(p.Porter)
 

cmd/porter/copy.go

@@ -1,9 +1,8 @@
 package main
 
 import (
-	"github.com/spf13/cobra"
-
 	"get.porter.sh/porter/pkg/porter"
+	"github.com/spf13/cobra"
 )
 
 func buildBundleCopyCommand(p *porter.Porter) *cobra.Command {

cmd/porter/credentials.go

@@ -43,7 +43,7 @@ You can use the generate and show commands to create the initial file:
 			return opts.Validate(p.Context, args)
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return p.CredentialsApply(opts)
+			return p.CredentialsApply(cmd.Context(), opts)
 		},
 	}
 
@@ -66,7 +66,7 @@ func buildCredentialsEditCommand(p *porter.Porter) *cobra.Command {
 			return opts.Validate(args)
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return p.EditCredential(opts)
+			return p.EditCredential(cmd.Context(), opts)
 		},
 	}
 
@@ -148,7 +148,7 @@ The results may also be filtered by associated labels and the namespace in which
 			return opts.Validate()
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return p.PrintCredentials(opts)
+			return p.PrintCredentials(cmd.Context(), opts)
 		},
 	}
 
@@ -178,8 +178,8 @@ func buildCredentialsDeleteCommand(p *porter.Porter) *cobra.Command {
 		PreRunE: func(_ *cobra.Command, args []string) error {
 			return opts.Validate(args)
 		},
-		RunE: func(_ *cobra.Command, _ []string) error {
-			return p.DeleteCredential(opts)
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			return p.DeleteCredential(cmd.Context(), opts)
 		},
 	}
 
@@ -203,7 +203,7 @@ func buildCredentialsShowCommand(p *porter.Porter) *cobra.Command {
 			return opts.Validate(args)
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return p.ShowCredential(opts)
+			return p.ShowCredential(cmd.Context(), opts)
 		},
 	}
 

cmd/porter/explain.go

@@ -1,9 +1,8 @@
 package main
 
 import (
-	"github.com/spf13/cobra"
-
 	"get.porter.sh/porter/pkg/porter"
+	"github.com/spf13/cobra"
 )
 
 func buildBundleExplainCommand(p *porter.Porter) *cobra.Command {

cmd/porter/inspect.go

@@ -1,9 +1,8 @@
 package main
 
 import (
-	"github.com/spf13/cobra"
-
 	"get.porter.sh/porter/pkg/porter"
+	"github.com/spf13/cobra"
 )
 
 func buildBundleInspectCommand(p *porter.Porter) *cobra.Command {

cmd/porter/main.go

@@ -36,6 +36,15 @@ func main() {
 		// Trace the command that called porter, e.g. porter installation show
 		cmd, commandName, formattedCommand := getCalledCommand(rootCmd)
 
+		// When running an internal plugin, switch how we log to be compatible
+		// with the hashicorp go-plugin framework
+		if commandName == "porter plugins run" {
+			p.IsInternalPlugin = true
+			if len(os.Args) > 3 {
+				p.InternalPluginKey = os.Args[3]
+			}
+		}
+
 		// Only run init logic that could fail for commands that
 		// really need it, skip it for commands that should NEVER
 		// fail.
@@ -46,7 +55,7 @@ func main() {
 			}
 		}
 
-		ctx, log := p.StartRootSpan(context.Background(), commandName, attribute.String("command", formattedCommand))
+		ctx, log := p.StartRootSpan(ctx, commandName, attribute.String("command", formattedCommand))
 		defer func() {
 			// Capture panics and trace them
 			if panicErr := recover(); panicErr != nil {
@@ -58,6 +67,7 @@ func main() {
 				os.Exit(1)
 			} else {
 				log.EndSpan()
+				log.Close()
 				p.Close()
 			}
 		}()
@@ -148,7 +158,7 @@ Try our QuickStart https://porter.sh/quickstart to learn how to use Porter.
 			if err != nil {
 				return err
 			}
-			
+
 			return nil
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {

cmd/porter/main_test.go

@@ -87,7 +87,7 @@ func TestExperimentalFlags(t *testing.T) {
 
 	t.Run("flag unset, env unset", func(t *testing.T) {
 		p := porter.NewTestPorter(t)
-		defer p.Teardown()
+		defer p.Close()
 
 		cmd := buildRootCommandFrom(p.Porter)
 		cmd.SetArgs([]string{"install"})
@@ -97,7 +97,7 @@ func TestExperimentalFlags(t *testing.T) {
 
 	t.Run("flag set", func(t *testing.T) {
 		p := porter.NewTestPorter(t)
-		defer p.Teardown()
+		defer p.Close()
 
 		cmd := buildRootCommandFrom(p.Porter)
 		cmd.SetArgs([]string{"install", "--experimental", experimental.StructuredLogs})
@@ -111,7 +111,7 @@ func TestExperimentalFlags(t *testing.T) {
 		defer os.Unsetenv(expEnvVar)
 
 		p := porter.NewTestPorter(t)
-		defer p.Teardown()
+		defer p.Close()
 
 		cmd := buildRootCommandFrom(p.Porter)
 		cmd.SetArgs([]string{"install"})

cmd/porter/mixins_test.go

@@ -10,7 +10,7 @@ import (
 
 func TestBuildListMixinsCommand_DefaultFormat(t *testing.T) {
 	p := porter.NewTestPorter(t)
-	defer p.Teardown()
+	defer p.Close()
 
 	cmd := buildMixinsListCommand(p.Porter)
 
@@ -22,7 +22,7 @@ func TestBuildListMixinsCommand_DefaultFormat(t *testing.T) {
 
 func TestBuildListMixinsCommand_AlternateFormat(t *testing.T) {
 	p := porter.NewTestPorter(t)
-	defer p.Teardown()
+	defer p.Close()
 
 	cmd := buildMixinsListCommand(p.Porter)
 	cmd.ParseFlags([]string{"-o", "json"})
@@ -35,7 +35,7 @@ func TestBuildListMixinsCommand_AlternateFormat(t *testing.T) {
 
 func TestBuildListMixinsCommand_BadFormat(t *testing.T) {
 	p := porter.NewTestPorter(t)
-	defer p.Teardown()
+	defer p.Close()
 
 	cmd := buildMixinsListCommand(p.Porter)
 	cmd.ParseFlags([]string{"-o", "flarts"})
@@ -48,7 +48,7 @@ func TestBuildListMixinsCommand_BadFormat(t *testing.T) {
 
 func TestBuildMixinInstallCommand(t *testing.T) {
 	p := porter.NewTestPorter(t)
-	defer p.Teardown()
+	defer p.Close()
 
 	cmd := BuildMixinInstallCommand(p.Porter)
 	err := cmd.ParseFlags([]string{"--url", "https://example.com/mixins/helm"})
@@ -60,7 +60,7 @@ func TestBuildMixinInstallCommand(t *testing.T) {
 
 func TestBuildMixinInstallCommand_NoMixinName(t *testing.T) {
 	p := porter.NewTestPorter(t)
-	defer p.Teardown()
+	defer p.Close()
 
 	cmd := BuildMixinInstallCommand(p.Porter)
 

cmd/porter/parameters.go

@@ -42,7 +42,7 @@ You can use the generate and show commands to create the initial file:
 			return opts.Validate(p.Context, args)
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return p.ParametersApply(opts)
+			return p.ParametersApply(cmd.Context(), opts)
 		},
 	}
 
@@ -65,7 +65,7 @@ func buildParametersEditCommand(p *porter.Porter) *cobra.Command {
 			return opts.Validate(args)
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return p.EditParameter(opts)
+			return p.EditParameter(cmd.Context(), opts)
 		},
 	}
 
@@ -147,7 +147,7 @@ The results may also be filtered by associated labels and the namespace in which
 			return opts.Validate()
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return p.PrintParameters(opts)
+			return p.PrintParameters(cmd.Context(), opts)
 		},
 	}
 
@@ -176,8 +176,8 @@ func buildParametersDeleteCommand(p *porter.Porter) *cobra.Command {
 		PreRunE: func(_ *cobra.Command, args []string) error {
 			return opts.Validate(args)
 		},
-		RunE: func(_ *cobra.Command, _ []string) error {
-			return p.DeleteParameter(opts)
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			return p.DeleteParameter(cmd.Context(), opts)
 		},
 	}
 
@@ -200,7 +200,7 @@ func buildParametersShowCommand(p *porter.Porter) *cobra.Command {
 			return opts.Validate(args)
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return p.ShowParameter(opts)
+			return p.ShowParameter(cmd.Context(), opts)
 		},
 	}
 

cmd/porter/plugins.go

@@ -22,6 +22,7 @@ func buildPluginsCommands(p *porter.Porter) *cobra.Command {
 	cmd.AddCommand(buildPluginShowCommand(p))
 	cmd.AddCommand(BuildPluginInstallCommand(p))
 	cmd.AddCommand(BuildPluginUninstallCommand(p))
+	cmd.AddCommand(buildPluginRunCommand(p))
 
 	return cmd
 }
@@ -147,3 +148,18 @@ func BuildPluginUninstallCommand(p *porter.Porter) *cobra.Command {
 
 	return cmd
 }
+
+func buildPluginRunCommand(p *porter.Porter) *cobra.Command {
+	var opts porter.RunInternalPluginOpts
+	cmd := &cobra.Command{
+		Use:   "run PLUGIN_KEY",
+		Short: "Serve internal plugins",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			opts.ApplyArgs(args)
+			return p.RunInternalPlugins(cmd.Context(), opts)
+		},
+		Hidden: true, // This should ALWAYS be hidden, it is not a user-facing command
+	}
+
+	return cmd
+}

cmd/porter/run_test.go

@@ -11,7 +11,7 @@ import (
 
 func TestRun_Validate(t *testing.T) {
 	p := porter.NewTestPorter(t)
-	defer p.Teardown()
+	defer p.Close()
 
 	configTpl, err := p.Templates.GetManifest()
 	require.NoError(t, err)
@@ -26,7 +26,7 @@ func TestRun_Validate(t *testing.T) {
 
 func TestRun_ValidateCustomAction(t *testing.T) {
 	p := porter.NewTestPorter(t)
-	defer p.Teardown()
+	defer p.Close()
 
 	configTpl, err := p.Templates.GetManifest()
 	require.NoError(t, err)

cmd/porter/storage.go

@@ -1,9 +1,8 @@
 package main
 
 import (
-	"github.com/spf13/cobra"
-
 	"get.porter.sh/porter/pkg/porter"
+	"github.com/spf13/cobra"
 )
 
 func buildStorageCommand(p *porter.Porter) *cobra.Command {
@@ -31,7 +30,7 @@ func buildStorageMigrateCommand(p *porter.Porter) *cobra.Command {
 
 Always back up Porter's data before performing a migration. Instructions for backing up are at https://porter.sh/storage-migrate.`,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return p.MigrateStorage()
+			return p.MigrateStorage(cmd.Context())
 		},
 	}
 }