diff --git a/.gitattributes b/.gitattributes index 940d651664d5..9697c956587e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,7 +1 @@ pkg/operator/crds/*.yaml linguist-generated=true - -# Git on Windows may automatically check out files with crlf line endings, -# which breaks the Fuzz parser in 1.18 (golang/go#52268). A fix is scheduled -# for 1.19, but in the meantime we have to disable autocrlf for Fuzz files and -# ensure that \n is the only end-of-line character used. -**/testdata/fuzz/Fuzz*/** -text eol=lf diff --git a/Makefile b/Makefile index dd2245857e41..289662f6bd4e 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ ## Build, test, and generate code for various parts of Grafana Agent. ## -## At least Go 1.18, git, and a moderately recent version of Docker is required +## At least Go 1.19, git, and a moderately recent version of Docker is required ## to be able to use the Makefile. This list isn't exhaustive and there are other ## dependencies for the generate-* targets. If you do not have the full list of ## build dependencies, you may set USE_CONTAINER=1 to proxy build commands to a diff --git a/docs/developer/contributing.md b/docs/developer/contributing.md index 88d6da1b92e1..1abac93a26cf 100644 --- a/docs/developer/contributing.md +++ b/docs/developer/contributing.md @@ -56,7 +56,7 @@ down the issues involving the components you want to work on. To build Grafana Agent from source code, please install the following tools: 1. [Git](https://git-scm.com/) -2. [Go](https://golang.org/) (version 1.18 and up) +2. [Go](https://golang.org/) (version 1.19 and up) 3. [Make](https://www.gnu.org/software/make/) 4. [Docker](https://www.docker.com/) @@ -163,17 +163,16 @@ To add or update a new dependency, use the `go get` command: ```bash # Pick the latest tagged release. -go install example.com/some/module/pkg@latest +go get example.com/some/module/pkg@latest # Pick a specific version. -go install example.com/some/module/pkg@vX.Y.Z +go get example.com/some/module/pkg@vX.Y.Z ``` Tidy up the `go.mod` and `go.sum` files: ```bash -# The GO111MODULE variable can be omitted when the code isn't located in GOPATH. -GO111MODULE=on go mod tidy +go mod tidy ``` You have to commit the changes to `go.mod` and `go.sum` before submitting the diff --git a/docs/sources/flow/concepts/components.md b/docs/sources/flow/concepts/components.md index 1137dddebd52..34a4562141ad 100644 --- a/docs/sources/flow/concepts/components.md +++ b/docs/sources/flow/concepts/components.md @@ -24,7 +24,7 @@ Components are specified in the config file by first providing the component's name with a user-specified label, and then by providing arguments to configure the component: -``` +```river discovery.kubernetes "pods" { role = "pod" } diff --git a/docs/sources/flow/getting-started/collect-opentelemetry-data.md b/docs/sources/flow/getting-started/collect-opentelemetry-data.md index ef33b223a505..cb794b1fbce1 100644 --- a/docs/sources/flow/getting-started/collect-opentelemetry-data.md +++ b/docs/sources/flow/getting-started/collect-opentelemetry-data.md @@ -104,7 +104,7 @@ data using OTLP, complete the following steps: 2. Add the following line inside of the `client` block of your `otelcol.exporter.otlp` component: - ``` + ```river auth = otelcol.auth.basic.BASIC_AUTH_LABEL.handler ``` diff --git a/docs/sources/flow/reference/components/discovery.relabel.md b/docs/sources/flow/reference/components/discovery.relabel.md index 792b0a93f6e1..5c7b71d14b6e 100644 --- a/docs/sources/flow/reference/components/discovery.relabel.md +++ b/docs/sources/flow/reference/components/discovery.relabel.md @@ -32,7 +32,7 @@ different labels. ## Usage -``` +```river discovery.relabel "LABEL" { targets = TARGET_LIST diff --git a/docs/sources/flow/reference/components/loki.process.md b/docs/sources/flow/reference/components/loki.process.md index fcbc475cfd59..a7b42b73a623 100644 --- a/docs/sources/flow/reference/components/loki.process.md +++ b/docs/sources/flow/reference/components/loki.process.md @@ -186,7 +186,7 @@ provide a custom label using the `drop_counter_reason` argument. The following stage drops log entries that contain the word `debug` _and_ are longer than 1KB. -``` +```river stage.drop { expression = ".*debug.*" longer_than = "1KB" @@ -197,7 +197,7 @@ On the following example, we define multiple `drop` blocks so `loki.process` drops entries that are either 24h or older, are longer than 8KB, _or_ the extracted value of 'app' is equal to foo. -``` +```river stage.drop { older_than = "24h" drop_reason = "too old" @@ -1355,7 +1355,7 @@ The supported actions are: The following stage fetches the `time` value from the shared values map, parses it as a RFC3339 format, and sets it as the log entry's timestamp. -``` +```river stage.timestamp { source = "time" format = "RFC3339" diff --git a/docs/sources/flow/reference/components/loki.source.cloudflare.md b/docs/sources/flow/reference/components/loki.source.cloudflare.md index d3a9fd296000..82360338a6eb 100644 --- a/docs/sources/flow/reference/components/loki.source.cloudflare.md +++ b/docs/sources/flow/reference/components/loki.source.cloudflare.md @@ -5,7 +5,7 @@ title: loki.source.cloudflare # loki.source.cloudflare `loki.source.cloudflare` pulls logs from the Cloudflare Logpull API and -forwards them to other `loki.*` components. +forwards them to other `loki.*` components. These logs contain data related to the connecting client, the request path through the Cloudflare network, and the response from the origin web server and @@ -81,72 +81,72 @@ The last timestamp fetched by the component is recorded in the All incoming Cloudflare log entries are in JSON format. You can make use of the `loki.process` component and a JSON processing stage to extract more labels or change the log line format. A sample log looks like this: -``` +```json { - "CacheCacheStatus": "miss", - "CacheResponseBytes": 8377, - "CacheResponseStatus": 200, - "CacheTieredFill": false, - "ClientASN": 786, - "ClientCountry": "gb", - "ClientDeviceType": "desktop", - "ClientIP": "100.100.5.5", - "ClientIPClass": "noRecord", - "ClientRequestBytes": 2691, - "ClientRequestHost": "www.foo.com", - "ClientRequestMethod": "GET", - "ClientRequestPath": "/comments/foo/", - "ClientRequestProtocol": "HTTP/1.0", - "ClientRequestReferer": "https://www.foo.com/foo/168855/?offset=8625", - "ClientRequestURI": "/foo/15248108/", - "ClientRequestUserAgent": "some bot", - "ClientSSLCipher": "ECDHE-ECDSA-AES128-GCM-SHA256", - "ClientSSLProtocol": "TLSv1.2", - "ClientSrcPort": 39816, - "ClientXRequestedWith": "", - "EdgeColoCode": "MAN", - "EdgeColoID": 341, - "EdgeEndTimestamp": 1637336610671000000, - "EdgePathingOp": "wl", - "EdgePathingSrc": "macro", - "EdgePathingStatus": "nr", - "EdgeRateLimitAction": "", - "EdgeRateLimitID": 0, - "EdgeRequestHost": "www.foo.com", - "EdgeResponseBytes": 14878, - "EdgeResponseCompressionRatio": 1, - "EdgeResponseContentType": "text/html", - "EdgeResponseStatus": 200, - "EdgeServerIP": "8.8.8.8", - "EdgeStartTimestamp": 1637336610517000000, - "FirewallMatchesActions": [], - "FirewallMatchesRuleIDs": [], - "FirewallMatchesSources": [], - "OriginIP": "8.8.8.8", - "OriginResponseBytes": 0, - "OriginResponseHTTPExpires": "", - "OriginResponseHTTPLastModified": "", - "OriginResponseStatus": 200, - "OriginResponseTime": 123000000, - "OriginSSLProtocol": "TLSv1.2", - "ParentRayID": "00", - "RayID": "6b0a...", + "CacheCacheStatus": "miss", + "CacheResponseBytes": 8377, + "CacheResponseStatus": 200, + "CacheTieredFill": false, + "ClientASN": 786, + "ClientCountry": "gb", + "ClientDeviceType": "desktop", + "ClientIP": "100.100.5.5", + "ClientIPClass": "noRecord", + "ClientRequestBytes": 2691, + "ClientRequestHost": "www.foo.com", + "ClientRequestMethod": "GET", + "ClientRequestPath": "/comments/foo/", + "ClientRequestProtocol": "HTTP/1.0", + "ClientRequestReferer": "https://www.foo.com/foo/168855/?offset=8625", + "ClientRequestURI": "/foo/15248108/", + "ClientRequestUserAgent": "some bot", + "ClientSSLCipher": "ECDHE-ECDSA-AES128-GCM-SHA256", + "ClientSSLProtocol": "TLSv1.2", + "ClientSrcPort": 39816, + "ClientXRequestedWith": "", + "EdgeColoCode": "MAN", + "EdgeColoID": 341, + "EdgeEndTimestamp": 1637336610671000000, + "EdgePathingOp": "wl", + "EdgePathingSrc": "macro", + "EdgePathingStatus": "nr", + "EdgeRateLimitAction": "", + "EdgeRateLimitID": 0, + "EdgeRequestHost": "www.foo.com", + "EdgeResponseBytes": 14878, + "EdgeResponseCompressionRatio": 1, + "EdgeResponseContentType": "text/html", + "EdgeResponseStatus": 200, + "EdgeServerIP": "8.8.8.8", + "EdgeStartTimestamp": 1637336610517000000, + "FirewallMatchesActions": [], + "FirewallMatchesRuleIDs": [], + "FirewallMatchesSources": [], + "OriginIP": "8.8.8.8", + "OriginResponseBytes": 0, + "OriginResponseHTTPExpires": "", + "OriginResponseHTTPLastModified": "", + "OriginResponseStatus": 200, + "OriginResponseTime": 123000000, + "OriginSSLProtocol": "TLSv1.2", + "ParentRayID": "00", + "RayID": "6b0a...", "RequestHeaders": [], "ResponseHeaders": [ "x-foo": "bar" ], - "SecurityLevel": "med", - "WAFAction": "unknown", - "WAFFlags": "0", - "WAFMatchedVar": "", - "WAFProfile": "unknown", - "WAFRuleID": "", - "WAFRuleMessage": "", - "WorkerCPUTime": 0, - "WorkerStatus": "unknown", - "WorkerSubrequest": false, - "WorkerSubrequestCount": 0, - "ZoneID": 1234 + "SecurityLevel": "med", + "WAFAction": "unknown", + "WAFFlags": "0", + "WAFMatchedVar": "", + "WAFProfile": "unknown", + "WAFRuleID": "", + "WAFRuleMessage": "", + "WorkerCPUTime": 0, + "WorkerStatus": "unknown", + "WorkerSubrequest": false, + "WorkerSubrequestCount": 0, + "ZoneID": 1234 } ``` diff --git a/docs/sources/flow/reference/components/loki.source.heroku.md b/docs/sources/flow/reference/components/loki.source.heroku.md index 624c5ced4bc3..17946b08056b 100644 --- a/docs/sources/flow/reference/components/loki.source.heroku.md +++ b/docs/sources/flow/reference/components/loki.source.heroku.md @@ -12,7 +12,7 @@ block and fans out incoming entries to the list of receivers in `forward_to`. Before using `loki.source.heroku`, Heroku should be configured with the URL where the Agent will be listening. Follow the steps in [Heroku HTTPS Drain docs](https://devcenter.heroku.com/articles/log-drains#https-drains) for using the Heroku CLI with a command like the following: -``` +```shell heroku drains:add [http|https]://HOSTNAME:PORT/heroku/api/v1/drain -a HEROKU_APP_NAME ``` diff --git a/docs/sources/flow/reference/components/phlare.scrape.md b/docs/sources/flow/reference/components/phlare.scrape.md index 86b073232e53..ff580b0f4fda 100644 --- a/docs/sources/flow/reference/components/phlare.scrape.md +++ b/docs/sources/flow/reference/components/phlare.scrape.md @@ -16,7 +16,7 @@ Multiple `phlare.scrape` components can be specified by giving them different la ## Usage -``` +```river phlare.scrape "LABEL" { targets = TARGET_LIST forward_to = RECEIVER_LIST diff --git a/docs/sources/flow/reference/components/prometheus.exporter.consul.md b/docs/sources/flow/reference/components/prometheus.exporter.consul.md index 964adc2d42c6..247ea934e14a 100644 --- a/docs/sources/flow/reference/components/prometheus.exporter.consul.md +++ b/docs/sources/flow/reference/components/prometheus.exporter.consul.md @@ -16,7 +16,7 @@ The `prometheus.exporter.consul` component embeds ## Usage ```river -prometheus.exporter.consul "LABEL"{ +prometheus.exporter.consul "LABEL" { } ``` diff --git a/docs/sources/flow/reference/components/prometheus.exporter.mysql.md b/docs/sources/flow/reference/components/prometheus.exporter.mysql.md index a4b469d35b9f..9ce06bac531f 100644 --- a/docs/sources/flow/reference/components/prometheus.exporter.mysql.md +++ b/docs/sources/flow/reference/components/prometheus.exporter.mysql.md @@ -17,7 +17,7 @@ The `prometheus.exporter.mysql` component embeds ```river prometheus.exporter.mysql "LABEL" { - data_source_name = "DATA SOURCE NAME" + data_source_name = "DATA_SOURCE_NAME" } ``` diff --git a/docs/sources/flow/reference/stdlib/discovery_target_decode.md b/docs/sources/flow/reference/stdlib/discovery_target_decode.md index 1fbab6cad2f7..65f0dcf35934 100644 --- a/docs/sources/flow/reference/stdlib/discovery_target_decode.md +++ b/docs/sources/flow/reference/stdlib/discovery_target_decode.md @@ -12,7 +12,7 @@ targets matching the exports of `discovery.*` components. The string must match the JSON format used by Prometheus' HTTP and file service discovery: -``` +```json [ { "targets": [ "", ... ], diff --git a/go.sum b/go.sum index 2988b5c82472..1cbcb5b7e52a 100644 --- a/go.sum +++ b/go.sum @@ -1855,8 +1855,6 @@ github.com/grafana/phlare/api v0.1.2 h1:1jrwd3KnsXMzj/tJih9likx5EvbY3pbvLbDqAAYe github.com/grafana/phlare/api v0.1.2/go.mod h1:29vcLwFDmZBDce2jwFIMtzvof7fzPadT8VMKw9ks7FU= github.com/grafana/postgres_exporter v0.8.1-0.20210722175051-db35d7c2f520 h1:HnFWqxhoSF3WC7sKAdMZ+SRXvHLVZlZ3sbQjuUlTqkw= github.com/grafana/postgres_exporter v0.8.1-0.20210722175051-db35d7c2f520/go.mod h1:+HPXgiOV0InDHcZ2jNijL1SOKvo0eEPege5fQA0+ICI= -github.com/grafana/process-exporter v0.7.3-0.20210106202358-831154072e2a h1:JUnP/laSl2GylHT0+fqAqOZY+7XkLh1mLefLN0n8Mmk= -github.com/grafana/process-exporter v0.7.3-0.20210106202358-831154072e2a/go.mod h1:RMjrx3Qn8l2pgCD27g45xbko4UDpVVuHC8Cd2YXPtWA= github.com/grafana/prometheus v1.8.2-0.20230328154716-1d0e5416a0de h1:HjEfLfBCMlZktx4tGEtMSbTZNdHrhzaCouRsXkh13ps= github.com/grafana/prometheus v1.8.2-0.20230328154716-1d0e5416a0de/go.mod h1:Pfqb/MLnnR2KK+0vchiaH39jXxvLMBk+3lnIGP4N7Vk= github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd h1:PpuIBO5P3e9hpqBD0O/HjhShYuM6XE0i/lbE6J94kww= diff --git a/pkg/river/encoding/block.go b/pkg/river/encoding/block.go index 3c0585c7baac..c7d3fc27d122 100644 --- a/pkg/river/encoding/block.go +++ b/pkg/river/encoding/block.go @@ -37,20 +37,38 @@ func (bf *blockField) convertBlock(namePrefix []string, reflectValue reflect.Val } reflectValue = reflectValue.Elem() } - if reflectValue.Kind() != reflect.Struct { - return fmt.Errorf("convertBlock can only be called on struct kinds, got %s", reflectValue.Kind()) - } - bf.Name = strings.Join(mergeStringSlices(namePrefix, f.Name), ".") - bf.Type = "block" - bf.Label = getBlockLabel(reflectValue) + switch reflectValue.Kind() { + case reflect.Struct: + bf.Name = strings.Join(mergeStringSlices(namePrefix, f.Name), ".") + bf.Type = "block" + bf.Label = getBlockLabel(reflectValue) + + fields, err := getFieldsForBlockStruct(namePrefix, reflectValue.Interface()) + if err != nil { + return err + } + bf.Body = fields + return nil + + case reflect.Map: + if reflectValue.Type().Key().Kind() != reflect.String { + return fmt.Errorf("convertBlock given unsupported map type; expected map[string]T, got %s", reflectValue.Type()) + } - fields, err := getFieldsForBlock(namePrefix, reflectValue.Interface()) - if err != nil { - return err + bf.Name = strings.Join(mergeStringSlices(namePrefix, f.Name), ".") + bf.Type = "block" + + fields, err := getFieldsForBlockMap(reflectValue) + if err != nil { + return err + } + bf.Body = fields + return nil + + default: + return fmt.Errorf("convertBlock can only be called on struct or map kinds, got %s", reflectValue.Kind()) } - bf.Body = fields - return nil } // getBlockLabel returns the label for a given block. @@ -65,7 +83,7 @@ func getBlockLabel(rv reflect.Value) string { return "" } -func getFieldsForBlock(namePrefix []string, input interface{}) ([]interface{}, error) { +func getFieldsForBlockStruct(namePrefix []string, input interface{}) ([]interface{}, error) { val := value.Encode(input) reflectVal := val.Reflect() rt := rivertags.Get(reflectVal.Type()) @@ -122,6 +140,27 @@ func getFieldsForBlock(namePrefix []string, input interface{}) ([]interface{}, e return fields, nil } +func getFieldsForBlockMap(val reflect.Value) ([]interface{}, error) { + var fields []interface{} + + it := val.MapRange() + for it.Next() { + // Make a fake field so newAttribute works properly. + field := rivertags.Field{ + Name: []string{it.Key().String()}, + Flags: rivertags.FlagAttr, + } + attr, err := newAttribute(value.FromRaw(it.Value()), field) + if err != nil { + return nil, err + } + + fields = append(fields, attr) + } + + return fields, nil +} + func mergeStringSlices(a, b []string) []string { if len(a) == 0 { return b diff --git a/pkg/river/encoding/encoding.go b/pkg/river/encoding/encoding.go index c25df6db648c..bc7a717d4c0c 100644 --- a/pkg/river/encoding/encoding.go +++ b/pkg/river/encoding/encoding.go @@ -22,7 +22,7 @@ func ConvertRiverBodyToJSON(input interface{}) ([]byte, error) { if input == nil { return nil, nil } - fields, err := getFieldsForBlock(nil, input) + fields, err := getFieldsForBlockStruct(nil, input) if err != nil { return nil, err } diff --git a/pkg/river/encoding/encoding_test.go b/pkg/river/encoding/encoding_test.go index e7102f0584c1..9e4a5c54e232 100644 --- a/pkg/river/encoding/encoding_test.go +++ b/pkg/river/encoding/encoding_test.go @@ -144,3 +144,28 @@ func TestConvertRiverBodyToJSON_Enum_Block(t *testing.T) { require.JSONEq(t, expect, string(actual)) } + +func TestMapBlocks(t *testing.T) { + type Body struct { + Block map[string]string `river:"some_block,block,optional"` + } + + val := Body{ + Block: map[string]string{"key": "value"}, + } + + actual, err := encoding.ConvertRiverBodyToJSON(val) + require.NoError(t, err) + + expect := `[{ + "name": "some_block", + "type": "block", + "body": [{ + "name": "key", + "type": "attr", + "value": { "type": "string", "value": "value" } + }] + }]` + + require.JSONEq(t, expect, string(actual)) +}