Skip to content

Commit

Permalink
Merge branch 'master' into builtin-operations
Browse files Browse the repository at this point in the history
  • Loading branch information
@taran-p
taran-p authored Oct 2, 2024
2 parents 6392c27 + ce0b434 commit 87c4ca6
Show file tree
Hide file tree
Showing 3 changed files with 119 additions and 82 deletions.
123 changes: 80 additions & 43 deletions cmd/admin-heal.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,11 @@
package cmd

import (
"bufio"
"fmt"
"math"
"net/url"
"os"
"path/filepath"
"sort"
"strings"
Expand Down Expand Up @@ -76,6 +79,10 @@ var adminHealFlags = []cli.Flag{
Usage: "force stop a running heal sequence",
Hidden: true,
},
cli.BoolFlag{
Name: "force",
Usage: "avoid showing a warning prompt",
},
cli.BoolFlag{
Name: "remove",
Usage: "remove dangling objects in heal sequence",
Expand All @@ -95,6 +102,10 @@ var adminHealFlags = []cli.Flag{
Name: "verbose, v",
Usage: "show verbose information",
},
cli.BoolFlag{
Name: "all-drives, a",
Usage: "select all drives for verbose printing",
},
}

var adminHealCmd = cli.Command{
Expand Down Expand Up @@ -162,8 +173,11 @@ type poolInfo struct {
}

type setInfo struct {
maxUsedSpace uint64
totalDisks int
totalDisks int

readyDisksCount int // disks online and not in healing state
readyDisksUsedSpace uint64 // the total used space of ready disks

incapableDisks int
}

Expand Down Expand Up @@ -218,11 +232,11 @@ func generateSetsStatus(disks []madmin.Disk) map[setIndex]setInfo {
setSt = setInfo{}
}
setSt.totalDisks++
if d.UsedSpace > setSt.maxUsedSpace {
setSt.maxUsedSpace = d.UsedSpace
}
if d.State != "ok" || d.Healing {
setSt.incapableDisks++
} else {
setSt.readyDisksCount++
setSt.readyDisksUsedSpace += d.UsedSpace
}
m[idx] = setSt
}
Expand Down Expand Up @@ -327,6 +341,8 @@ type verboseBackgroundHealStatusMessage struct {
Status string `json:"status"`
HealInfo madmin.BgHealState

allDrives bool

// Specify storage class to show servers/disks tolerance
ToleranceForSC string `json:"-"`
}
Expand Down Expand Up @@ -374,14 +390,16 @@ func (s verboseBackgroundHealStatusMessage) String() string {
fmt.Fprintf(&msg, " %s: %s\n", endpoint, stateText)
continue
}
var serverHeader strings.Builder
var serverHeaderPrinted bool
serverStatus := serversStatus[endpoint]
switch {
case showTolerance:
serverHeader := " %s: (Tolerance: %d server(s))\n"
fmt.Fprintf(&msg, serverHeader, endpoint, poolsInfo[serverStatus.pool].tolerance)
hdr := " %s: (Tolerance: %d server(s))\n"
fmt.Fprintf(&serverHeader, hdr, endpoint, poolsInfo[serverStatus.pool].tolerance)
default:
serverHeader := " %s:\n"
fmt.Fprintf(&msg, serverHeader, endpoint)
hdr := " %s:\n"
fmt.Fprintf(&serverHeader, hdr, endpoint)
}

for _, d := range serverStatus.disks {
Expand All @@ -393,29 +411,49 @@ func (s verboseBackgroundHealStatusMessage) String() string {
case d.State == "ok" && d.Healing:
stateText = console.Colorize("DiskHealing", "HEALING")
case d.State == "ok":
if !s.allDrives {
continue
}
stateText = console.Colorize("DiskOK", "OK")
default:
stateText = console.Colorize("DiskFailed", d.State)
}
fmt.Fprintf(&msg, " + %s : %s\n", d.DrivePath, stateText)
if !serverHeaderPrinted {
serverHeaderPrinted = true
fmt.Fprint(&msg, serverHeader.String())
}
drivePath := d.DrivePath
if drivePath == "" {
if u, e := url.Parse(d.Endpoint); e == nil {
drivePath = u.Path
}
}
fmt.Fprintf(&msg, " + %s : %s\n", drivePath, stateText)

thisSet := setsStatus[setIndex{d.PoolIndex, d.SetIndex}]
if d.Healing && d.HealInfo != nil && !d.HealInfo.Finished {
if d.HealInfo.RetryAttempts == 0 {
now := time.Now().UTC()
scanSpeed := float64(d.UsedSpace) / float64(now.Sub(d.HealInfo.Started))
remainingTime := time.Duration(float64(setsStatus[setIndex{d.PoolIndex, d.SetIndex}].maxUsedSpace-d.UsedSpace) / scanSpeed)
estimationText := humanize.RelTime(now, now.Add(remainingTime), "", "")
fmt.Fprintf(&msg, " |__ Estimated: %s\n", estimationText)
} else {
fmt.Fprintf(&msg, " |__ Retry attempts: %d\n", d.HealInfo.RetryAttempts)
refUsedSpace := uint64(math.MaxUint64)
if thisSet.readyDisksCount > 0 { // to avoid crashing
refUsedSpace = thisSet.readyDisksUsedSpace / uint64(thisSet.readyDisksCount)
}
if refUsedSpace < d.UsedSpace { // normalize
refUsedSpace = d.UsedSpace
}
fmt.Fprintf(&msg, " |__ Progress: %d%%\n", 100*d.UsedSpace/refUsedSpace)
fmt.Fprintf(&msg, " |__ Started: %s\n", humanize.Time(d.HealInfo.Started))
if d.HealInfo.RetryAttempts > 0 {
fmt.Fprintf(&msg, " |__ Retries: %d\n", d.HealInfo.RetryAttempts)
}
}
fmt.Fprintf(&msg, " |__ Capacity: %s/%s\n", humanize.IBytes(d.UsedSpace), humanize.IBytes(d.TotalSpace))
fmt.Fprintf(&msg, " |__ Capacity: %s/%s\n", humanize.IBytes(d.UsedSpace), humanize.IBytes(d.TotalSpace))
if showTolerance {
fmt.Fprintf(&msg, " |__ Tolerance: %d drive(s)\n", parity-setsStatus[setIndex{d.PoolIndex, d.SetIndex}].incapableDisks)
fmt.Fprintf(&msg, " |__ Tolerance: %d drive(s)\n", parity-thisSet.incapableDisks)
}
}

fmt.Fprintf(&msg, "\n")
if serverHeaderPrinted {
fmt.Fprintf(&msg, "\n")
}
}
}

Expand Down Expand Up @@ -476,9 +514,6 @@ func (s shortBackgroundHealStatusMessage) String() string {
// The addition of Elapsed time of each parallel healing operation
// this is needed to calculate the rate of healing
accumulatedElapsedTime time.Duration

// ETA of healing - it is the latest ETA of all drives currently healing
healingRemaining time.Duration
)

var problematicDisks int
Expand All @@ -502,24 +537,23 @@ func (s shortBackgroundHealStatusMessage) String() string {
if disk.HealInfo != nil && !disk.HealInfo.Finished {
missingInSet++

diskSet := setIndex{pool: disk.PoolIndex, set: disk.SetIndex}
if maxUsedSpace := setsStatus[diskSet].maxUsedSpace; maxUsedSpace > 0 {
if pct := float64(disk.UsedSpace) / float64(maxUsedSpace); pct < leastPct {
thisSet := setsStatus[setIndex{pool: disk.PoolIndex, set: disk.SetIndex}]
refUsedSpace := uint64(math.MaxUint64)
if thisSet.readyDisksCount > 0 {
refUsedSpace = thisSet.readyDisksUsedSpace / uint64(thisSet.readyDisksCount)
}
if refUsedSpace < disk.UsedSpace {
refUsedSpace = disk.UsedSpace
}
if refUsedSpace > 0 {
if pct := float64(disk.UsedSpace) / float64(refUsedSpace); pct < leastPct {
leastPct = pct
}
} else {
// Unlikely to have max used space in an erasure set to be zero, but still set this to zero
leastPct = 0
}

if disk.HealInfo.RetryAttempts == 0 {
scanSpeed := float64(disk.UsedSpace) / float64(time.Since(disk.HealInfo.Started))
remainingTime := time.Duration(float64(setsStatus[diskSet].maxUsedSpace-disk.UsedSpace) / scanSpeed)
if remainingTime > healingRemaining {
healingRemaining = remainingTime
}
}

disk := disk
if furthestHealingDisk == nil {
furthestHealingDisk = &disk
Expand Down Expand Up @@ -582,14 +616,6 @@ func (s shortBackgroundHealStatusMessage) String() string {
healPrettyMsg += fmt.Sprintf("Heal rate: %d obj/s, %s/s\n", int64(itemsHealedPerSec), humanize.IBytes(uint64(bytesHealedPerSec)))
}

// Estimation completion
eta := "<unknown>"
if healingRemaining > 0 {
now := time.Now()
eta = humanize.RelTime(now, now.Add(healingRemaining), "", "")
}
healPrettyMsg += fmt.Sprintf("Estimated Completion: %s\n", eta)

if problematicDisks > 0 {
healPrettyMsg += "\n"
healPrettyMsg += fmt.Sprintf("%d offline disk(s) found.", problematicDisks)
Expand Down Expand Up @@ -669,6 +695,7 @@ func mainAdminHeal(ctx *cli.Context) error {
printMsg(verboseBackgroundHealStatusMessage{
Status: "success",
HealInfo: bgHealStatus,
allDrives: ctx.Bool("all-drives"),
ToleranceForSC: strings.ToUpper(ctx.String("storage-class")),
})
} else {
Expand Down Expand Up @@ -715,6 +742,16 @@ func mainAdminHeal(ctx *cli.Context) error {
return nil
}

if opts.Recursive && opts.Pool == nil && opts.Set == nil && isTerminal() && !ctx.Bool("force") {
fmt.Printf("You are about to scan and heal the whole namespace in all pools and sets, please confirm [y/N]: ")
answer, e := bufio.NewReader(os.Stdin).ReadString('\n')
fatalIf(probe.NewError(e), "Unable to parse user input.")
if answer = strings.TrimSpace(strings.ToLower(answer)); answer != "y" && answer != "yes" {
fmt.Println("Heal aborted!")
return nil
}
}

healStart, _, e := adminClnt.Heal(globalContext, bucket, prefix, opts, "", forceStart, false)
fatalIf(probe.NewError(e), "Unable to start healing.")

Expand Down
26 changes: 13 additions & 13 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ module github.com/minio/mc
go 1.22

require (
github.com/charmbracelet/bubbles v0.19.0
github.com/charmbracelet/bubbletea v0.27.1
github.com/charmbracelet/bubbles v0.20.0
github.com/charmbracelet/bubbletea v1.1.1
github.com/charmbracelet/lipgloss v0.13.0
github.com/cheggaaa/pb v1.0.29
github.com/dustin/go-humanize v1.0.1
Expand All @@ -21,7 +21,7 @@ require (
github.com/minio/cli v1.24.2
github.com/minio/colorjson v1.0.8
github.com/minio/filepath v1.0.0
github.com/minio/madmin-go/v3 v3.0.68
github.com/minio/madmin-go/v3 v3.0.70
github.com/minio/minio-go/v7 v7.0.77
github.com/minio/pkg/v3 v3.0.20
github.com/minio/selfupdate v0.6.0
Expand All @@ -31,13 +31,13 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/xattr v0.4.10
github.com/posener/complete v1.2.3
github.com/prometheus/client_golang v1.20.2
github.com/prometheus/client_golang v1.20.4
github.com/prometheus/procfs v0.15.1
github.com/rjeczalik/notify v0.9.3
github.com/rs/xid v1.6.0
github.com/shirou/gopsutil/v3 v3.24.5
github.com/tidwall/gjson v1.17.3
github.com/vbauerster/mpb/v8 v8.8.2
github.com/vbauerster/mpb/v8 v8.8.3
golang.org/x/net v0.29.0
golang.org/x/sys v0.25.0
golang.org/x/term v0.24.0
Expand All @@ -53,7 +53,7 @@ require (
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/charmbracelet/x/ansi v0.2.3 // indirect
github.com/charmbracelet/x/ansi v0.3.2 // indirect
github.com/charmbracelet/x/term v0.2.0 // indirect
github.com/coreos/go-semver v0.3.1 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
Expand Down Expand Up @@ -87,12 +87,12 @@ require (
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 // indirect
github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.59.1 // indirect
github.com/prometheus/prom2json v1.4.0 // indirect
github.com/prometheus/common v0.60.0 // indirect
github.com/prometheus/prom2json v1.4.1 // indirect
github.com/prometheus/prometheus v0.54.1 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
Expand All @@ -101,7 +101,7 @@ require (
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tinylib/msgp v1.2.1 // indirect
github.com/tinylib/msgp v1.2.2 // indirect
github.com/tklauser/go-sysconf v0.3.14 // indirect
github.com/tklauser/numcpus v0.8.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
Expand All @@ -112,9 +112,9 @@ require (
go.uber.org/zap v1.27.0 // indirect
golang.org/x/crypto v0.27.0 // indirect
golang.org/x/sync v0.8.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
google.golang.org/grpc v1.66.2 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240930140551-af27646dc61f // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240930140551-af27646dc61f // indirect
google.golang.org/grpc v1.67.1 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
Loading

0 comments on commit 87c4ca6

Please sign in to comment.