copyblocks: Configure s3 upload part size (#8292)

* objtools: configure part-size in s3 client-side upload Signed-off-by: Vladimir Varankin <vladimir.varankin@grafana.com> * copyblocks: enable pprof Signed-off-by: Vladimir Varankin <vladimir.varankin@grafana.com> * update changelog Signed-off-by: Vladimir Varankin <vladimir.varankin@grafana.com> * objtools/s3: don't define NumThreads Signed-off-by: Vladimir Varankin <vladimir.varankin@grafana.com> --------- Signed-off-by: Vladimir Varankin <vladimir.varankin@grafana.com>
grafana · Jun 6, 2024 · d1731fc · d1731fc
1 parent 5988b23
commit d1731fc
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -162,6 +162,8 @@
 ### Tools
 
 * [ENHANCEMENT] ulidtime: add option to show random part of ULID, timestamp in milliseconds and header. #7615
+* [ENHANCEMENT] copyblocks: add a flag to configure part-size for multipart uploads in s3 client-side copying. #8292
+* [ENHANCEMENT] copyblocks: enable pprof HTTP endpoints. #8292
 
 ## 2.12.0
 

diff --git a/pkg/util/objtools/s3.go b/pkg/util/objtools/s3.go
@@ -18,6 +18,7 @@ type S3ClientConfig struct {
 	AccessKeyID     string
 	SecretAccessKey string
 	Secure          bool
+	PartSize        uint64
 }
 
 func (c *S3ClientConfig) RegisterFlags(prefix string, f *flag.FlagSet) {
@@ -26,6 +27,7 @@ func (c *S3ClientConfig) RegisterFlags(prefix string, f *flag.FlagSet) {
 	f.StringVar(&c.AccessKeyID, prefix+"access-key-id", "", "The access key ID used in AWS Signature Version 4 authentication.")
 	f.StringVar(&c.SecretAccessKey, prefix+"secret-access-key", "", "The secret access key used in AWS Signature Version 4 authentication.")
 	f.BoolVar(&c.Secure, prefix+"secure", true, "If true (default), use HTTPS when connecting to the bucket. If false, insecure HTTP is used.")
+	f.Uint64Var(&c.PartSize, prefix+"part-size", 0, "If 0, and object's size is known and optimal for multipart upload, the default value is the minimum allowed size 16MiB.")
 }
 
 func (c *S3ClientConfig) Validate(prefix string) error {
@@ -55,12 +57,14 @@ func (c *S3ClientConfig) ToBucket() (Bucket, error) {
 	return &s3Bucket{
 		client:     client,
 		bucketName: c.BucketName,
+		partSize:   c.PartSize,
 	}, nil
 }
 
 type s3Bucket struct {
 	client     *minio.Client
 	bucketName string
+	partSize   uint64
 }
 
 func (bkt *s3Bucket) Get(ctx context.Context, objectName string, options GetOptions) (io.ReadCloser, error) {
@@ -163,7 +167,10 @@ func (bkt *s3Bucket) RestoreVersion(ctx context.Context, objectName string, vers
 }
 
 func (bkt *s3Bucket) Upload(ctx context.Context, objectName string, reader io.Reader, contentLength int64) error {
-	_, err := bkt.client.PutObject(ctx, bkt.bucketName, objectName, reader, contentLength, minio.PutObjectOptions{})
+	opts := minio.PutObjectOptions{
+		PartSize: bkt.partSize,
+	}
+	_, err := bkt.client.PutObject(ctx, bkt.bucketName, objectName, reader, contentLength, opts)
 	return err
 }
 

diff --git a/tools/copyblocks/main.go b/tools/copyblocks/main.go
@@ -12,6 +12,7 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	_ "net/http/pprof" // anonymous import to get the pprof handler registered
 	"os"
 	"os/signal"
 	"path/filepath"