Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

copyblocks: Configure s3 upload part size #8292

Merged
merged 4 commits into from
Jun 6, 2024
Merged

copyblocks: Configure s3 upload part size #8292

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4613a8d
objtools: configure part-size in s3 client-side upload
narqo Jun 6, 2024
880fac8
copyblocks: enable pprof
narqo Jun 6, 2024
aee2162
update changelog
narqo Jun 6, 2024
a15e320
objtools/s3: don't define NumThreads
narqo Jun 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,8 @@
### Tools

* [ENHANCEMENT] ulidtime: add option to show random part of ULID, timestamp in milliseconds and header. #7615
* [ENHANCEMENT] copyblocks: add a flag to configure part-size for multipart uploads in s3 client-side copying. #8292
* [ENHANCEMENT] copyblocks: enable pprof HTTP endpoints. #8292

## 2.12.0

Expand Down
11 changes: 10 additions & 1 deletion pkg/util/objtools/s3.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ type S3ClientConfig struct {
AccessKeyID string
SecretAccessKey string
Secure bool
PartSize uint64
}

func (c *S3ClientConfig) RegisterFlags(prefix string, f *flag.FlagSet) {
Expand All @@ -26,6 +27,7 @@ func (c *S3ClientConfig) RegisterFlags(prefix string, f *flag.FlagSet) {
f.StringVar(&c.AccessKeyID, prefix+"access-key-id", "", "The access key ID used in AWS Signature Version 4 authentication.")
f.StringVar(&c.SecretAccessKey, prefix+"secret-access-key", "", "The secret access key used in AWS Signature Version 4 authentication.")
f.BoolVar(&c.Secure, prefix+"secure", true, "If true (default), use HTTPS when connecting to the bucket. If false, insecure HTTP is used.")
f.Uint64Var(&c.PartSize, prefix+"part-size", 0, "If 0, and object's size is known and optimal for multipart upload, the default value is the minimum allowed size 16MiB.")
}

func (c *S3ClientConfig) Validate(prefix string) error {
Expand Down Expand Up @@ -55,12 +57,14 @@ func (c *S3ClientConfig) ToBucket() (Bucket, error) {
return &s3Bucket{
client: client,
bucketName: c.BucketName,
partSize: c.PartSize,
}, nil
}

type s3Bucket struct {
client *minio.Client
bucketName string
partSize uint64
}

func (bkt *s3Bucket) Get(ctx context.Context, objectName string, options GetOptions) (io.ReadCloser, error) {
Expand Down Expand Up @@ -163,7 +167,12 @@ func (bkt *s3Bucket) RestoreVersion(ctx context.Context, objectName string, vers
}

func (bkt *s3Bucket) Upload(ctx context.Context, objectName string, reader io.Reader, contentLength int64) error {
_, err := bkt.client.PutObject(ctx, bkt.bucketName, objectName, reader, contentLength, minio.PutObjectOptions{})
opts := minio.PutObjectOptions{
PartSize: bkt.partSize,
// Refer to https://github.com/thanos-io/objstore/blob/71ef2d0cf7c4b42a6b25bc019de41bf0acecd30c/providers/s3/s3.go#L513-L516
NumThreads: 4,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe this is already the default in Minio when there's no value set.

}
_, err := bkt.client.PutObject(ctx, bkt.bucketName, objectName, reader, contentLength, opts)
return err
}

Expand Down
1 change: 1 addition & 0 deletions tools/copyblocks/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"flag"
"fmt"
"net/http"
_ "net/http/pprof" // anonymous import to get the pprof handler registered
"os"
"os/signal"
"path/filepath"
Expand Down
Loading