Skip to content

Commit

Permalink
First pass at an ExternalClient that wraps Terraform
Browse files Browse the repository at this point in the history 
Signed-off-by: Nic Cope <negz@rk0n.org>
  • Loading branch information
@negz
negz committed Mar 10, 2021
1 parent 65ff308 commit 1f96281
Show file tree
Hide file tree
Showing 11 changed files with 417 additions and 121 deletions.
21 changes: 19 additions & 2 deletions apis/workspace/v1alpha1/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,12 +78,29 @@ type KeyReference struct {
Key string `json:"key"`
}

// A ModuleSource represents the source of a Terraform module.
// +kubebuilder:validation:Enum=Remote;Inline
type ModuleSource string

// Module sources.
const (
ModuleSourceRemote ModuleSource = "Remote"
ModuleSourceInline ModuleSource = "Inline"
)

// WorkspaceParameters are the configurable fields of a Workspace.
type WorkspaceParameters struct {
// The root module of this workspace; i.e. the path to the directory that
// contains the main.tf file of the Terraform configuration.
// The root module of this workspace; i.e. the module containing its main.tf
// file. When the workspace's source is 'Remote' (the default) this can be
// any address supported by terraform init -from-module, for example a git
// repository or an S3 bucket. When the workspace's source is 'Inline' the
// content of a simple main.tf file may be written inline.
Module string `json:"module"`

// Source of the root module of this workspace.
// +kubebuilder:default=Remote
Source ModuleSource `json:"source"`

// Configuration variables.
// +optional
Vars []Var `json:"vars,omitempty"`
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ require (
github.com/google/go-cmp v0.5.2
github.com/pkg/errors v0.9.1
gopkg.in/alecthomas/kingpin.v2 v2.2.6
k8s.io/api v0.20.1
k8s.io/apimachinery v0.20.1
k8s.io/client-go v0.20.1
sigs.k8s.io/controller-runtime v0.8.0
Expand Down
5 changes: 5 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,8 +121,10 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0 h1:w3NnFcKR5241cfmQU5ZZAsf0xcpId6mWOupTvJlUX2U=
github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
Expand Down Expand Up @@ -484,6 +486,7 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
Expand Down Expand Up @@ -831,6 +834,7 @@ google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfG
google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a h1:pOwg4OoaRYScjmR4LlLgdtnyoHYTSAVhhqe5uPdpII8=
google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
Expand All @@ -840,6 +844,7 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk=
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
Expand Down
224 changes: 156 additions & 68 deletions internal/controller/workspace/workspace.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,23 +18,29 @@ package workspace

import (
"context"
"fmt"
"io/ioutil"
"os"
"path/filepath"

"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/util/workqueue"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"

xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1"
"github.com/crossplane/crossplane-runtime/pkg/event"
"github.com/crossplane/crossplane-runtime/pkg/logging"
"github.com/crossplane/crossplane-runtime/pkg/meta"
"github.com/crossplane/crossplane-runtime/pkg/ratelimiter"
"github.com/crossplane/crossplane-runtime/pkg/reconciler/managed"
"github.com/crossplane/crossplane-runtime/pkg/resource"

tfv1alpha1 "github.com/negz/provider-terraform/apis/v1alpha1"
"github.com/negz/provider-terraform/apis/workspace/v1alpha1"
"github.com/negz/provider-terraform/internal/terraform"
)

const (
Expand All @@ -43,16 +49,37 @@ const (
errGetPC = "cannot get ProviderConfig"
errGetCreds = "cannot get credentials"

errNewClient = "cannot create new Service"
errMkdir = "cannot make Terraform configuration directory"
errWriteCreds = "cannot write Terraform credentials"
errWriteMain = "cannot write Terraform configuration " + tfMain
errInit = "cannot initialize Terraform configuration"
errWorkspace = "cannot select Terraform workspace"
errResources = "cannot list Terraform resources"
errOutputs = "cannot list Terraform outputs"
errOptions = "cannot determine Terraform options"
errApply = "cannot apply Terraform configuration"
errDestroy = "cannot apply Terraform configuration"
errVarFile = "cannot get tfvars"
)

// A NoOpService does nothing.
type NoOpService struct{}

var (
newNoOpService = func(_ []byte) (interface{}, error) { return &NoOpService{}, nil }
const (
// TODO(negz): Make the Terraform binary path and work dir configurable.
tfPath = "terraform"
tfDir = "/tf"
tfCreds = "credentials"
tfMain = "main.tf"
)

type tfclient interface {
Init(ctx context.Context, o ...terraform.InitOption) error
Validate(ctx context.Context) error
Workspace(ctx context.Context, name string) error
Outputs(ctx context.Context) ([]terraform.Output, error)
Resources(ctx context.Context) ([]string, error)
Apply(ctx context.Context, o ...terraform.Option) error
Destroy(ctx context.Context, o ...terraform.Option) error
}

// Setup adds a controller that reconciles Workspace managed resources.
func Setup(mgr ctrl.Manager, l logging.Logger, rl workqueue.RateLimiter) error {
name := managed.ControllerName(v1alpha1.WorkspaceGroupKind)
Expand All @@ -64,9 +91,8 @@ func Setup(mgr ctrl.Manager, l logging.Logger, rl workqueue.RateLimiter) error {
r := managed.NewReconciler(mgr,
resource.ManagedKind(v1alpha1.WorkspaceGroupVersionKind),
managed.WithExternalConnecter(&connector{
kube: mgr.GetClient(),
usage: resource.NewProviderConfigUsageTracker(mgr.GetClient(), &tfv1alpha1.ProviderConfigUsage{}),
newServiceFn: newNoOpService}),
kube: mgr.GetClient(),
usage: resource.NewProviderConfigUsageTracker(mgr.GetClient(), &tfv1alpha1.ProviderConfigUsage{})}),
managed.WithLogger(l.WithValues("controller", name)),
managed.WithRecorder(event.NewAPIRecorder(mgr.GetEventRecorderFor(name))))

Expand All @@ -77,25 +103,23 @@ func Setup(mgr ctrl.Manager, l logging.Logger, rl workqueue.RateLimiter) error {
Complete(r)
}

// A connector is expected to produce an ExternalClient when its Connect method
// is called.
type connector struct {
kube client.Client
usage resource.Tracker
newServiceFn func(creds []byte) (interface{}, error)
kube client.Client
usage resource.Tracker
}

// Connect typically produces an ExternalClient by:
// 1. Tracking that the managed resource is using a ProviderConfig.
// 2. Getting the managed resource's ProviderConfig.
// 3. Getting the credentials specified by the ProviderConfig.
// 4. Using the credentials to form a client.
func (c *connector) Connect(ctx context.Context, mg resource.Managed) (managed.ExternalClient, error) {
cr, ok := mg.(*v1alpha1.Workspace)
if !ok {
return nil, errors.New(errNotWorkspace)
}

// TODO(negz): Garbage collect this directory.
dir := filepath.Join(tfDir, string(cr.GetUID()))
if err := os.MkdirAll(dir, 0600); resource.Ignore(os.IsExist, err) != nil {
return nil, errors.Wrap(err, errMkdir)
}

if err := c.usage.Track(ctx, mg); err != nil {
return nil, errors.Wrap(err, errTrackPCUsage)
}
Expand All @@ -111,61 +135,59 @@ func (c *connector) Connect(ctx context.Context, mg resource.Managed) (managed.E
return nil, errors.Wrap(err, errGetCreds)
}

svc, err := c.newServiceFn(data)
if err != nil {
return nil, errors.Wrap(err, errNewClient)
if err := ioutil.WriteFile(filepath.Join(dir, tfCreds), data, 0600); err != nil {
return nil, errors.Wrap(err, errWriteCreds)
}

// TODO(negz): Allow this implementation to be swapped out for testing.
tf := terraform.Harness{Path: tfPath, Dir: dir}

io := []terraform.InitOption{terraform.FromModule(cr.Spec.ForProvider.Module)}
if cr.Spec.ForProvider.Source == v1alpha1.ModuleSourceInline {
if err := ioutil.WriteFile(filepath.Join(dir, tfMain), []byte(cr.Spec.ForProvider.Module), 0600); err != nil {
return nil, errors.Wrap(err, errWriteMain)
}
io = nil
}

if err := tf.Init(ctx, io...); err != nil {
return nil, errors.Wrap(err, errInit)
}

return &external{service: svc}, nil
return &external{tf: tf, kube: c.kube}, errors.Wrap(tf.Workspace(ctx, meta.GetExternalName(cr)), errWorkspace)
}

// An ExternalClient observes, then either creates, updates, or deletes an
// external resource to ensure it reflects the managed resource's desired state.
type external struct {
// A 'client' used to connect to the external resource API. In practice this
// would be something like an AWS SDK client.
service interface{}
tf tfclient
kube client.Reader
}

func (c *external) Observe(ctx context.Context, mg resource.Managed) (managed.ExternalObservation, error) {
cr, ok := mg.(*v1alpha1.Workspace)
if !ok {
return managed.ExternalObservation{}, errors.New(errNotWorkspace)
func (c *external) Observe(ctx context.Context, _ resource.Managed) (managed.ExternalObservation, error) {
r, err := c.tf.Resources(ctx)
if err != nil {
return managed.ExternalObservation{}, errors.Wrap(err, errResources)
}

// These fmt statements should be removed in the real implementation.
fmt.Printf("Observing: %+v", cr)
// TODO(negz): Include any non-sensitive outputs in our status?
o, err := c.tf.Outputs(ctx)
if err != nil {
return managed.ExternalObservation{}, errors.Wrap(err, errOutputs)
}

// TODO(negz): Is there any value in running terraform plan to determine
// whether the workspace is up-to-date, or should we just YOLO apply?
return managed.ExternalObservation{
// Return false when the external resource does not exist. This lets
// the managed resource reconciler know that it needs to call Create to
// (re)create the resource, or that it has successfully been deleted.
ResourceExists: true,

// Return false when the external resource exists, but it not up to date
// with the desired managed resource state. This lets the managed
// resource reconciler know that it needs to call Update.
ResourceUpToDate: true,

// Return any details that may be required to connect to the external
// resource. These will be stored as the connection secret.
ConnectionDetails: managed.ConnectionDetails{},
ResourceExists: len(r) > 0,
ResourceUpToDate: false,
ResourceLateInitialized: false,
ConnectionDetails: op2cd(o),
}, nil
}

func (c *external) Create(ctx context.Context, mg resource.Managed) (managed.ExternalCreation, error) {
cr, ok := mg.(*v1alpha1.Workspace)
if !ok {
return managed.ExternalCreation{}, errors.New(errNotWorkspace)
}

fmt.Printf("Creating: %+v", cr)

return managed.ExternalCreation{
// Optionally return any details that may be required to connect to the
// external resource. These will be stored as the connection secret.
ConnectionDetails: managed.ConnectionDetails{},
}, nil
// Terraform does not have distinct 'create' and 'update' operations.
u, err := c.Update(ctx, mg)
return managed.ExternalCreation{ConnectionDetails: u.ConnectionDetails}, err
}

func (c *external) Update(ctx context.Context, mg resource.Managed) (managed.ExternalUpdate, error) {
Expand All @@ -174,13 +196,25 @@ func (c *external) Update(ctx context.Context, mg resource.Managed) (managed.Ext
return managed.ExternalUpdate{}, errors.New(errNotWorkspace)
}

fmt.Printf("Updating: %+v", cr)
o, err := c.options(ctx, cr.Spec.ForProvider)
if err != nil {
return managed.ExternalUpdate{}, errors.Wrap(err, errOptions)
}

if err := c.tf.Apply(ctx, o...); err != nil {
return managed.ExternalUpdate{}, errors.Wrap(err, errApply)
}

return managed.ExternalUpdate{
// Optionally return any details that may be required to connect to the
// external resource. These will be stored as the connection secret.
ConnectionDetails: managed.ConnectionDetails{},
}, nil
op, err := c.tf.Outputs(ctx)
if err != nil {
return managed.ExternalUpdate{}, errors.Wrap(err, errOutputs)
}

// TODO(negz): Allow Workspaces to optionally derive their readiness from an
// output - similar to the logic XRs use to derive readiness from a field of
// a composed resource.
mg.SetConditions(xpv1.Available())
return managed.ExternalUpdate{ConnectionDetails: op2cd(op)}, nil
}

func (c *external) Delete(ctx context.Context, mg resource.Managed) error {
Expand All @@ -189,7 +223,61 @@ func (c *external) Delete(ctx context.Context, mg resource.Managed) error {
return errors.New(errNotWorkspace)
}

fmt.Printf("Deleting: %+v", cr)
o, err := c.options(ctx, cr.Spec.ForProvider)
if err != nil {
return errors.Wrap(err, errOptions)
}

return nil
return errors.Wrap(c.tf.Destroy(ctx, o...), errDestroy)
}

func (c *external) options(ctx context.Context, p v1alpha1.WorkspaceParameters) ([]terraform.Option, error) {
o := make([]terraform.Option, 0, len(p.Vars)+len(p.VarFiles))

for _, v := range p.Vars {
o = append(o, terraform.WithVar(v.Key, v.Value))
}

for _, vf := range p.VarFiles {
fmt := terraform.HCL
if vf.Format == v1alpha1.VarFileFormatJSON {
fmt = terraform.JSON
}

switch vf.Source {
case v1alpha1.VarFileSourceConfigMapKey:
cm := &corev1.ConfigMap{}
r := vf.ConfigMapKeyReference
nn := types.NamespacedName{Namespace: r.Namespace, Name: r.Name}
if err := c.kube.Get(ctx, nn, cm); err != nil {
return nil, errors.Wrap(err, errVarFile)
}
o = append(o, terraform.WithVarFile(cm.BinaryData[r.Key], fmt))

case v1alpha1.VarFileSourceSecretKey:
s := &corev1.Secret{}
r := vf.SecretKeyReference
nn := types.NamespacedName{Namespace: r.Namespace, Name: r.Name}
if err := c.kube.Get(ctx, nn, s); err != nil {
return nil, errors.Wrap(err, errVarFile)
}
o = append(o, terraform.WithVarFile(s.Data[r.Key], fmt))
}
}

return o, nil
}

func op2cd(o []terraform.Output) managed.ConnectionDetails {
cd := managed.ConnectionDetails{}
for _, op := range o {
if op.Type == terraform.OutputTypeString {
cd[op.Name] = []byte(op.StringValue())
continue
}
if j, err := op.JSONValue(); err == nil {
cd[op.Name] = j
}
}
return cd
}
Loading

0 comments on commit 1f96281

Please sign in to comment.