diff --git a/driver.go b/driver.go index 6d12bc8..1b83c2b 100644 --- a/driver.go +++ b/driver.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "math" "os" "strconv" "strings" @@ -21,7 +22,26 @@ var trueVar = true var falseVar = false type hetznerDriver struct { - client hetznerClienter + client hetznerClienter + nextTry time.Time + failuresInARow int +} + +func (hd *hetznerDriver) checkBackoff() error { + if time.Now().Before(hd.nextTry) { + return fmt.Errorf("last failure too recent; waiting a bit before retrying") + } + return nil +} + +func (hd *hetznerDriver) handleBackoff(err error) { + if err == nil { + hd.failuresInARow = 0 + } + if err != nil { + hd.nextTry = time.Now().Add(time.Duration(int64(math.Pow(2, float64(hd.failuresInARow)))) * time.Second) + hd.failuresInARow++ + } } func newHetznerDriver() *hetznerDriver { @@ -36,7 +56,7 @@ func (hd *hetznerDriver) Capabilities() *volume.CapabilitiesResponse { } } -func (hd *hetznerDriver) Create(req *volume.CreateRequest) error { +func (hd *hetznerDriver) createInternal(req *volume.CreateRequest) error { validateOptions(req.Name, req.Options) prefixedName := prefixName(req.Name) @@ -61,7 +81,7 @@ func (hd *hetznerDriver) Create(req *volume.CreateRequest) error { } switch f := getOption("fstype", req.Options); f { case "xfs", "ext4": - opts.Format = hcloud.String(f) + opts.Format = hcloud.Ptr(f) } resp, _, err := hd.client.Volume().Create(context.Background(), opts) @@ -118,7 +138,17 @@ func (hd *hetznerDriver) Create(req *volume.CreateRequest) error { return nil } -func (hd *hetznerDriver) List() (*volume.ListResponse, error) { +func (hd *hetznerDriver) Create(req *volume.CreateRequest) error { + if err := hd.checkBackoff(); err != nil { + return err + } + + err := hd.createInternal(req) + hd.handleBackoff(err) + return err +} + +func (hd *hetznerDriver) listInternal() (*volume.ListResponse, error) { logrus.Infof("got list request") vols, err := hd.client.Volume().All(context.Background()) @@ -150,7 +180,17 @@ func (hd *hetznerDriver) List() (*volume.ListResponse, error) { return &resp, nil } -func (hd *hetznerDriver) Get(req *volume.GetRequest) (*volume.GetResponse, error) { +func (hd *hetznerDriver) List() (*volume.ListResponse, error) { + if err := hd.checkBackoff(); err != nil { + return nil, err + } + + resp, err := hd.listInternal() + hd.handleBackoff(err) + return resp, err +} + +func (hd *hetznerDriver) getInternal(req *volume.GetRequest) (*volume.GetResponse, error) { prefixedName := prefixName(req.Name) logrus.Infof("fetching information for volume %q", prefixedName) @@ -186,7 +226,17 @@ func (hd *hetznerDriver) Get(req *volume.GetRequest) (*volume.GetResponse, error return &resp, nil } -func (hd *hetznerDriver) Remove(req *volume.RemoveRequest) error { +func (hd *hetznerDriver) Get(req *volume.GetRequest) (*volume.GetResponse, error) { + if err := hd.checkBackoff(); err != nil { + return nil, err + } + + resp, err := hd.getInternal(req) + hd.handleBackoff(err) + return resp, err +} + +func (hd *hetznerDriver) removeInternal(req *volume.RemoveRequest) error { prefixedName := prefixName(req.Name) logrus.Infof("starting volume removal for %q", prefixedName) @@ -228,7 +278,17 @@ func (hd *hetznerDriver) Remove(req *volume.RemoveRequest) error { return nil } -func (hd *hetznerDriver) Path(req *volume.PathRequest) (*volume.PathResponse, error) { +func (hd *hetznerDriver) Remove(req *volume.RemoveRequest) error { + if err := hd.checkBackoff(); err != nil { + return err + } + + err := hd.removeInternal(req) + hd.handleBackoff(err) + return err +} + +func (hd *hetznerDriver) pathInternal(req *volume.PathRequest) (*volume.PathResponse, error) { prefixedName := prefixName(req.Name) logrus.Infof("got path request for volume %q", prefixedName) @@ -241,7 +301,17 @@ func (hd *hetznerDriver) Path(req *volume.PathRequest) (*volume.PathResponse, er return &volume.PathResponse{Mountpoint: resp.Volume.Mountpoint}, nil } -func (hd *hetznerDriver) Mount(req *volume.MountRequest) (*volume.MountResponse, error) { +func (hd *hetznerDriver) Path(req *volume.PathRequest) (*volume.PathResponse, error) { + if err := hd.checkBackoff(); err != nil { + return nil, err + } + + resp, err := hd.pathInternal(req) + hd.handleBackoff(err) + return resp, err +} + +func (hd *hetznerDriver) mountInternal(req *volume.MountRequest) (*volume.MountResponse, error) { prefixedName := prefixName(req.Name) logrus.Infof("received mount request for %q as %q", prefixedName, req.ID) @@ -313,7 +383,17 @@ func (hd *hetznerDriver) Mount(req *volume.MountRequest) (*volume.MountResponse, return &volume.MountResponse{Mountpoint: mountpoint}, nil } -func (hd *hetznerDriver) Unmount(req *volume.UnmountRequest) error { +func (hd *hetznerDriver) Mount(req *volume.MountRequest) (*volume.MountResponse, error) { + if err := hd.checkBackoff(); err != nil { + return nil, err + } + + resp, err := hd.mountInternal(req) + hd.handleBackoff(err) + return resp, err +} + +func (hd *hetznerDriver) unmountInternal(req *volume.UnmountRequest) error { prefixedName := prefixName(req.Name) logrus.Infof("received unmount request for %q as %q", prefixedName, req.ID) @@ -357,6 +437,16 @@ func (hd *hetznerDriver) Unmount(req *volume.UnmountRequest) error { return nil } +func (hd *hetznerDriver) Unmount(req *volume.UnmountRequest) error { + if err := hd.checkBackoff(); err != nil { + return err + } + + err := hd.unmountInternal(req) + hd.handleBackoff(err) + return err +} + func (hd *hetznerDriver) getServerForLocalhost() (*hcloud.Server, error) { hostname, err := os.Hostname() if err != nil {