feat: Add cache HTTP client to cache downloads in .chezmoiexternal.<f…

…ormat>

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/google/gops v0.3.22
 	github.com/google/renameio/v2 v2.0.0
 	github.com/google/uuid v1.3.0 // indirect
+	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79
 	github.com/huandu/xstrings v1.3.2 // indirect
 	github.com/kevinburke/ssh_config v1.1.0 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
@@ -29,6 +30,7 @@ require (
 	github.com/muesli/reflow v0.3.0 // indirect
 	github.com/muesli/termenv v0.9.0 // indirect
 	github.com/pelletier/go-toml v1.9.4
+	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/rogpeppe/go-internal v1.8.0
 	github.com/rs/zerolog v1.26.0
 	github.com/sergi/go-diff v1.1.0

go.sum

@@ -297,6 +297,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
 github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
+github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA=
+github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
@@ -465,6 +467,8 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM=
 github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
+github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

internal/chezmoi/sourcestate.go

@@ -77,6 +77,7 @@ type SourceState struct {
 	encryption              Encryption
 	ignore                  *patternSet
 	interpreters            map[string]*Interpreter
+	httpClient              *http.Client
 	logger                  *zerolog.Logger
 	minVersion              semver.Version
 	mode                    Mode
@@ -122,6 +123,13 @@ func WithEncryption(encryption Encryption) SourceStateOption {
 	}
 }
 
+// WithHTTPClient sets the HTTP client.
+func WithHTTPClient(httpClient *http.Client) SourceStateOption {
+	return func(s *SourceState) {
+		s.httpClient = httpClient
+	}
+}
+
 // WithInterpreters sets the interpreters.
 func WithInterpreters(interpreters map[string]*Interpreter) SourceStateOption {
 	return func(s *SourceState) {
@@ -202,6 +210,7 @@ func NewSourceState(options ...SourceStateOption) *SourceState {
 		umask:                Umask,
 		encryption:           NoEncryption{},
 		ignore:               newPatternSet(),
+		httpClient:           http.DefaultClient,
 		logger:               &log.Logger,
 		readTemplateData:     true,
 		priorityTemplateData: make(map[string]interface{}),
@@ -1092,9 +1101,6 @@ func (s *SourceState) executeTemplate(templateAbsPath AbsPath) ([]byte, error) {
 // getExternalDataRaw returns the raw data for external at externalRelPath,
 // possibly from the external cache.
 func (s *SourceState) getExternalDataRaw(ctx context.Context, externalRelPath RelPath, external External, options *ReadOptions) ([]byte, error) {
-	// FIXME be more intelligent about HTTP caching, e.g. following RFC 7234,
-	// rather than blindly re-downloading each time
-
 	var now time.Time
 	if options != nil && options.TimeNow != nil {
 		now = options.TimeNow()
@@ -1122,7 +1128,7 @@ func (s *SourceState) getExternalDataRaw(ctx context.Context, externalRelPath Re
 	if err != nil {
 		return nil, err
 	}
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := s.httpClient.Do(req)
 	s.logger.Err(err).
 		Str("method", req.Method).
 		Int("statusCode", resp.StatusCode).

internal/cmd/config.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"net/http"
 	"os"
 	"os/exec"
 	"os/user"
@@ -160,6 +161,7 @@ type Config struct {
 	destSystem             chezmoi.System
 	persistentStateAbsPath chezmoi.AbsPath
 	persistentState        chezmoi.PersistentState
+	httpClient             *http.Client
 	logger                 *zerolog.Logger
 
 	// Computed configuration.
@@ -185,6 +187,7 @@ type configState struct {
 var (
 	chezmoiRelPath             = chezmoi.NewRelPath("chezmoi")
 	persistentStateFileRelPath = chezmoi.NewRelPath("chezmoistate.boltdb")
+	httpCacheDirRelPath        = chezmoi.NewRelPath("httpcache")
 
 	configStateKey = []byte("configState")
 
@@ -229,9 +232,11 @@ func newConfig(options ...configOption) (*Config, error) {
 		return nil, err
 	}
 
+	cacheDirAbsPath := chezmoi.NewAbsPath(bds.CacheHome).Join(chezmoiRelPath)
+
 	c := &Config{
 		// Global configuration, settable in the config file.
-		CacheDirAbsPath: chezmoi.NewAbsPath(bds.CacheHome).Join(chezmoiRelPath),
+		CacheDirAbsPath: cacheDirAbsPath,
 		Color: autoBool{
 			auto: true,
 		},
@@ -386,6 +391,7 @@ func newConfig(options ...configOption) (*Config, error) {
 		// Configuration.
 		fileSystem: vfs.OSFS,
 		bds:        bds,
+		httpClient: newCacheHTTPClient(cacheDirAbsPath.Join(httpCacheDirRelPath)),
 
 		// Computed configuration.
 		homeDirAbsPath: homeDirAbsPath,
@@ -1302,6 +1308,7 @@ func (c *Config) newSourceState(ctx context.Context, options ...chezmoi.SourceSt
 		chezmoi.WithDefaultTemplateDataFunc(c.defaultTemplateData),
 		chezmoi.WithDestDir(c.DestDirAbsPath),
 		chezmoi.WithEncryption(c.encryption),
+		chezmoi.WithHTTPClient(c.httpClient),
 		chezmoi.WithInterpreters(c.Interpreters),
 		chezmoi.WithLogger(&sourceStateLogger),
 		chezmoi.WithMode(c.Mode),

internal/cmd/upgradecmd.go

@@ -240,7 +240,7 @@ func (c *Config) downloadURL(ctx context.Context, url string) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := c.httpClient.Do(req)
 	c.logger.Err(err).
 		Str("method", req.Method).
 		Int("statusCode", resp.StatusCode).

internal/cmd/util.go

@@ -11,7 +11,11 @@ import (
 	"unicode"
 
 	"github.com/google/go-github/v40/github"
+	"github.com/gregjones/httpcache"
+	"github.com/gregjones/httpcache/diskcache"
 	"golang.org/x/oauth2"
+
+	"github.com/twpayne/chezmoi/v2/internal/chezmoi"
 )
 
 var (
@@ -72,6 +76,12 @@ func firstNonEmptyString(ss ...string) string {
 	return ""
 }
 
+// newCacheHTTPClient returns a new http.Client that will cache responses
+// according to the HTTP RFC.
+func newCacheHTTPClient(cacheDirAbsPath chezmoi.AbsPath) *http.Client {
+	return httpcache.NewTransport(diskcache.New(cacheDirAbsPath.String())).Client()
+}
+
 // newGitHubClient returns a new github.Client configured with an access token,
 // if available.
 func newGitHubClient(ctx context.Context) *github.Client {