Use Azure BlobURL.Download instead of in-memory buffer

Modify the azure.Bucket get methods to use BlobURL.Download for fetching
blobs and blob ranges. This avoids the need to allocate a buffer for storing
the entire expected size of the object in memory. Instead, use a ReaderCloser
view of the body returned by the download method.

See grafana/mimir#2229

Signed-off-by: Nick Pillitteri <nick.pillitteri@grafana.com>

CHANGELOG.md

@@ -22,6 +22,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 ### Changed
 
 - [#5447](https://github.com/thanos-io/thanos/pull/5447) Promclient: Ignore 405 status codes for Prometheus buildVersion requests
+- [#5451](https://github.com/thanos-io/thanos/pull/5451) Azure: Reduce memory usage by not buffering file downloads entirely in memory.
 
 ### Removed
 

pkg/objstore/azure/azure.go

@@ -4,10 +4,8 @@
 package azure
 
 import (
-	"bytes"
 	"context"
 	"io"
-	"io/ioutil"
 	"os"
 	"strings"
 	"testing"
@@ -298,39 +296,15 @@ func (b *Bucket) getBlobReader(ctx context.Context, name string, offset, length
 	if err != nil {
 		return nil, errors.Wrapf(err, "cannot get Azure blob URL, address: %s", name)
 	}
-	var props *blob.BlobGetPropertiesResponse
-	props, err = blobURL.GetProperties(ctx, blob.BlobAccessConditions{}, blob.ClientProvidedKeyOptions{})
-	if err != nil {
-		return nil, errors.Wrapf(err, "cannot get properties for container: %s", name)
-	}
-
-	var size int64
-	// If a length is specified and it won't go past the end of the file,
-	// then set it as the size.
-	if length > 0 && length <= props.ContentLength()-offset {
-		size = length
-		level.Debug(b.logger).Log("msg", "set size to length", "size", size, "length", length, "offset", offset, "name", name)
-	} else {
-		size = props.ContentLength() - offset
-		level.Debug(b.logger).Log("msg", "set size to go to EOF", "contentlength", props.ContentLength(), "size", size, "length", length, "offset", offset, "name", name)
-	}
 
-	destBuffer := make([]byte, size)
-
-	if err := blob.DownloadBlobToBuffer(context.Background(), blobURL.BlobURL, offset, size,
-		destBuffer, blob.DownloadFromBlobOptions{
-			BlockSize:   blob.BlobDefaultDownloadBlockSize,
-			Parallelism: uint16(3),
-			Progress:    nil,
-			RetryReaderOptionsPerBlock: blob.RetryReaderOptions{
-				MaxRetryRequests: b.config.ReaderConfig.MaxRetryRequests,
-			},
-		},
-	); err != nil {
-		return nil, errors.Wrapf(err, "cannot download blob, address: %s", blobURL.BlobURL)
+	dl, err := blobURL.Download(ctx, offset, length, blob.BlobAccessConditions{}, false, blob.ClientProvidedKeyOptions{})
+	if err != nil {
+		return nil, errors.Wrapf(err, "cannot download Azure blob, address: %s", name)
 	}
 
-	return ioutil.NopCloser(bytes.NewReader(destBuffer)), nil
+	return dl.Body(blob.RetryReaderOptions{
+		MaxRetryRequests: b.config.ReaderConfig.MaxRetryRequests,
+	}), nil
 }
 
 // Get returns a reader for the given object name.