Config API v1 (grafana#3898)

* Committing all prometheus prototypes in case we want to reference them again before finalizing the approach.

Signed-off-by: erikbaranowski <39704712+erikbaranowski@users.noreply.github.com>

* Implement first version of config converter.

Signed-off-by: erikbaranowski <39704712+erikbaranowski@users.noreply.github.com>

---------

Signed-off-by: erikbaranowski <39704712+erikbaranowski@users.noreply.github.com>

converter/converter.go

@@ -0,0 +1,37 @@
+// Package converter exposes utilities to convert config files from other
+// programs to Grafana Agent Flow configurations.
+package converter
+
+import (
+	"fmt"
+
+	"github.com/grafana/agent/converter/internal/prometheusconvert"
+)
+
+// Input represents the type of config file being fed into the converter.
+type Input string
+
+const (
+	// InputPrometheus indicates that the input file is a prometheus.yaml file.
+	InputPrometheus Input = "prometheus"
+)
+
+// Convert generates a Grafana Agent Flow config given an input configuration
+// file.
+//
+// Conversions are made as literally as possible, so the resulting config files
+// may be unoptimized (i.e., lacking component reuse). A converted config file
+// should just be the starting point rather than the final destination.
+//
+// Note that not all functionality defined in the input configuration may have
+// an equivalent in Grafana Agent Flow. If the conversion could not complete
+// because of mismatched functionality, an error is returned with no resulting
+// config. If the conversion completed successfully but generated warnings, an
+// error is returned alongside the resulting config.
+func Convert(in []byte, kind Input) ([]byte, error) {
+	switch kind {
+	case InputPrometheus:
+		return prometheusconvert.Convert(in)
+	}
+	return nil, fmt.Errorf("unrecognized kind %q", kind)
+}

converter/internal/prometheusconvert/common.go

@@ -0,0 +1,84 @@
+package prometheusconvert
+
+import (
+	"github.com/grafana/agent/component/common/config"
+	"github.com/grafana/agent/pkg/river/rivertypes"
+	promconfig "github.com/prometheus/common/config"
+)
+
+func toHttpClientConfig(httpClientConfig *promconfig.HTTPClientConfig) *config.HTTPClientConfig {
+	if httpClientConfig == nil {
+		return nil
+	}
+
+	return &config.HTTPClientConfig{
+		BasicAuth:       toBasicAuth(httpClientConfig.BasicAuth),
+		Authorization:   toAuthorization(httpClientConfig.Authorization),
+		OAuth2:          toOAuth2(httpClientConfig.OAuth2),
+		BearerToken:     rivertypes.Secret(httpClientConfig.BearerToken),
+		BearerTokenFile: httpClientConfig.BearerTokenFile,
+		ProxyURL:        config.URL(httpClientConfig.ProxyURL),
+		TLSConfig:       *toTLSConfig(&httpClientConfig.TLSConfig),
+		FollowRedirects: httpClientConfig.FollowRedirects,
+		EnableHTTP2:     httpClientConfig.EnableHTTP2,
+	}
+}
+
+func toBasicAuth(basicAuth *promconfig.BasicAuth) *config.BasicAuth {
+	if basicAuth == nil {
+		return nil
+	}
+
+	return &config.BasicAuth{
+		Username:     basicAuth.Username,
+		Password:     rivertypes.Secret(basicAuth.Password),
+		PasswordFile: basicAuth.PasswordFile,
+	}
+}
+
+func toAuthorization(authorization *promconfig.Authorization) *config.Authorization {
+	if authorization == nil {
+		return nil
+	}
+
+	return &config.Authorization{
+		Type:            authorization.Type,
+		Credentials:     rivertypes.Secret(authorization.Credentials),
+		CredentialsFile: authorization.CredentialsFile,
+	}
+}
+
+func toOAuth2(oAuth2 *promconfig.OAuth2) *config.OAuth2Config {
+	if oAuth2 == nil {
+		return nil
+	}
+
+	return &config.OAuth2Config{
+		ClientID:         oAuth2.ClientID,
+		ClientSecret:     rivertypes.Secret(oAuth2.ClientSecret),
+		ClientSecretFile: oAuth2.ClientSecretFile,
+		Scopes:           oAuth2.Scopes,
+		TokenURL:         oAuth2.TokenURL,
+		EndpointParams:   oAuth2.EndpointParams,
+		ProxyURL:         config.URL(oAuth2.ProxyURL),
+		TLSConfig:        toTLSConfig(&oAuth2.TLSConfig),
+	}
+}
+
+func toTLSConfig(tlsConfig *promconfig.TLSConfig) *config.TLSConfig {
+	if tlsConfig == nil {
+		return nil
+	}
+
+	return &config.TLSConfig{
+		CA:                 tlsConfig.CA,
+		CAFile:             tlsConfig.CAFile,
+		Cert:               tlsConfig.Cert,
+		CertFile:           tlsConfig.CertFile,
+		Key:                rivertypes.Secret(tlsConfig.Key),
+		KeyFile:            tlsConfig.KeyFile,
+		ServerName:         tlsConfig.ServerName,
+		InsecureSkipVerify: tlsConfig.InsecureSkipVerify,
+		MinVersion:         config.TLSVersion(tlsConfig.MinVersion),
+	}
+}

converter/internal/prometheusconvert/prometheusconvert.go

@@ -0,0 +1,58 @@
+package prometheusconvert
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/go-kit/log"
+	"github.com/grafana/agent/pkg/river/token/builder"
+	promconfig "github.com/prometheus/prometheus/config"
+
+	_ "github.com/prometheus/prometheus/discovery/install" // Register Prometheus SDs
+)
+
+// Convert implements a Prometheus config converter.
+//
+// TODO...
+// The implementation of this API is a work in progress.
+// Additional components must be implemented:
+//
+//	prometheus.relabel
+//	discovery.azure
+//	discovery.consul
+//	discovery.digitalocean
+//	discovery.dns
+//	discovery.docker
+//	discovery.ec2
+//	discovery.file
+//	discovery.gce
+//	discovery.kubernetes
+//	discovery.lightsail
+//	discovery.relabel
+func Convert(in []byte) ([]byte, error) {
+	promConfig, err := promconfig.Load(string(in), false, log.NewNopLogger())
+	if err != nil {
+		return nil, err
+	}
+	f := builder.NewFile()
+
+	remoteWriteArgs := toRemotewriteArguments(promConfig)
+	remoteWriteBlock := builder.NewBlock([]string{"prometheus", "remote_write"}, "default")
+	remoteWriteBlock.Body().AppendFrom(remoteWriteArgs)
+	f.Body().AppendBlock(remoteWriteBlock)
+
+	for _, scrapeConfig := range promConfig.ScrapeConfigs {
+		scrapeArgs := toScrapeArguments(scrapeConfig)
+
+		scrapeBlock := builder.NewBlock([]string{"prometheus", "scrape"}, scrapeArgs.JobName)
+		scrapeBlock.Body().AppendFrom(scrapeArgs)
+
+		f.Body().AppendBlock(scrapeBlock)
+	}
+
+	var buf bytes.Buffer
+	if _, err := f.WriteTo(&buf); err != nil {
+		return nil, fmt.Errorf("failed to render Flow config: %w", err)
+	}
+	return buf.Bytes(), nil
+}

converter/internal/prometheusconvert/prometheusconvert_test.go

@@ -0,0 +1,60 @@
+package prometheusconvert_test
+
+import (
+	"bytes"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/grafana/agent/converter/internal/prometheusconvert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	promSuffix = ".yaml"
+	flowSuffix = ".river"
+)
+
+func TestConvert(t *testing.T) {
+	filepath.WalkDir("testdata", func(path string, d fs.DirEntry, _ error) error {
+		if d.IsDir() {
+			return nil
+		}
+
+		if strings.HasSuffix(path, promSuffix) {
+			inputFile := path
+			expectFile := strings.TrimSuffix(path, promSuffix) + flowSuffix
+
+			inputBytes, err := os.ReadFile(inputFile)
+			require.NoError(t, err)
+			expectBytes, err := os.ReadFile(expectFile)
+			require.NoError(t, err)
+
+			caseName := filepath.Base(path)
+			caseName = strings.TrimSuffix(caseName, promSuffix)
+
+			t.Run(caseName, func(t *testing.T) {
+				testConverter(t, inputBytes, expectBytes)
+			})
+		}
+
+		return nil
+	})
+}
+
+func testConverter(t *testing.T, input, expect []byte) {
+	t.Helper()
+
+	actual, err := prometheusconvert.Convert(input)
+
+	require.NoError(t, err)
+	require.Equal(t, string(normalizeLineEndings(expect)), string(normalizeLineEndings(actual))+"\n")
+}
+
+// Replace '\r\n' with '\n'
+func normalizeLineEndings(data []byte) []byte {
+	normalized := bytes.ReplaceAll(data, []byte{'\r', '\n'}, []byte{'\n'})
+	return normalized
+}

converter/internal/prometheusconvert/remote_write.go

@@ -0,0 +1,59 @@
+package prometheusconvert
+
+import (
+	"time"
+
+	"github.com/grafana/agent/component/prometheus/remotewrite"
+	promconfig "github.com/prometheus/prometheus/config"
+)
+
+func toRemotewriteArguments(promConfig *promconfig.Config) *remotewrite.Arguments {
+	return &remotewrite.Arguments{
+		ExternalLabels: promConfig.GlobalConfig.ExternalLabels.Map(),
+		Endpoints:      getEndpointOptions(promConfig.RemoteWriteConfigs),
+		WALOptions:     remotewrite.DefaultWALOptions,
+	}
+}
+
+func getEndpointOptions(remoteWriteConfigs []*promconfig.RemoteWriteConfig) []*remotewrite.EndpointOptions {
+	endpoints := make([]*remotewrite.EndpointOptions, 0)
+
+	for _, remoteWriteConfig := range remoteWriteConfigs {
+		endpoint := &remotewrite.EndpointOptions{
+			Name:                 remoteWriteConfig.Name,
+			URL:                  remoteWriteConfig.URL.String(),
+			RemoteTimeout:        time.Duration(remoteWriteConfig.RemoteTimeout),
+			Headers:              remoteWriteConfig.Headers,
+			SendExemplars:        remoteWriteConfig.SendExemplars,
+			SendNativeHistograms: remoteWriteConfig.SendNativeHistograms,
+			HTTPClientConfig:     toHttpClientConfig(&remoteWriteConfig.HTTPClientConfig),
+			QueueOptions:         toQueueOptions(&remoteWriteConfig.QueueConfig),
+			MetadataOptions:      toMetadataOptions(&remoteWriteConfig.MetadataConfig),
+		}
+
+		endpoints = append(endpoints, endpoint)
+	}
+
+	return endpoints
+}
+
+func toQueueOptions(queueConfig *promconfig.QueueConfig) *remotewrite.QueueOptions {
+	return &remotewrite.QueueOptions{
+		Capacity:          queueConfig.Capacity,
+		MaxShards:         queueConfig.MaxShards,
+		MinShards:         queueConfig.MinShards,
+		MaxSamplesPerSend: queueConfig.MaxSamplesPerSend,
+		BatchSendDeadline: time.Duration(queueConfig.BatchSendDeadline),
+		MinBackoff:        time.Duration(queueConfig.MinBackoff),
+		MaxBackoff:        time.Duration(queueConfig.MaxBackoff),
+		RetryOnHTTP429:    queueConfig.RetryOnRateLimit,
+	}
+}
+
+func toMetadataOptions(metadataConfig *promconfig.MetadataConfig) *remotewrite.MetadataOptions {
+	return &remotewrite.MetadataOptions{
+		Send:              metadataConfig.Send,
+		SendInterval:      time.Duration(metadataConfig.SendInterval),
+		MaxSamplesPerSend: metadataConfig.MaxSamplesPerSend,
+	}
+}

converter/internal/prometheusconvert/scrape.go

@@ -0,0 +1,58 @@
+package prometheusconvert
+
+import (
+	"time"
+
+	"github.com/grafana/agent/component/discovery"
+	"github.com/grafana/agent/component/prometheus/scrape"
+	promconfig "github.com/prometheus/prometheus/config"
+	promdiscovery "github.com/prometheus/prometheus/discovery"
+	"github.com/prometheus/prometheus/storage"
+)
+
+func toScrapeArguments(scrapeConfig *promconfig.ScrapeConfig) *scrape.Arguments {
+	if scrapeConfig == nil {
+		return nil
+	}
+
+	return &scrape.Arguments{
+		Targets:               getTargets(scrapeConfig),
+		ForwardTo:             []storage.Appendable{}, // TODO
+		JobName:               scrapeConfig.JobName,
+		HonorLabels:           scrapeConfig.HonorLabels,
+		HonorTimestamps:       scrapeConfig.HonorTimestamps,
+		Params:                scrapeConfig.Params,
+		ScrapeInterval:        time.Duration(scrapeConfig.ScrapeInterval),
+		ScrapeTimeout:         time.Duration(scrapeConfig.ScrapeTimeout),
+		MetricsPath:           scrapeConfig.MetricsPath,
+		Scheme:                scrapeConfig.Scheme,
+		BodySizeLimit:         scrapeConfig.BodySizeLimit,
+		SampleLimit:           scrapeConfig.SampleLimit,
+		TargetLimit:           scrapeConfig.TargetLimit,
+		LabelLimit:            scrapeConfig.LabelLimit,
+		LabelNameLengthLimit:  scrapeConfig.LabelNameLengthLimit,
+		LabelValueLengthLimit: scrapeConfig.LabelValueLengthLimit,
+		HTTPClientConfig:      *toHttpClientConfig(&scrapeConfig.HTTPClientConfig),
+		ExtraMetrics:          false,
+		Clustering:            scrape.Clustering{Enabled: false},
+	}
+}
+
+func getTargets(scrapeConfig *promconfig.ScrapeConfig) []discovery.Target {
+	targets := []discovery.Target{}
+
+	for _, serviceDiscoveryConfig := range scrapeConfig.ServiceDiscoveryConfigs {
+		switch sdc := serviceDiscoveryConfig.(type) {
+		case promdiscovery.StaticConfig:
+			for _, target := range sdc {
+				for _, labelSet := range target.Targets {
+					for labelName, labelValue := range labelSet {
+						targets = append(targets, map[string]string{string(labelName): string(labelValue)})
+					}
+				}
+			}
+		}
+	}
+
+	return targets
+}