Skip to content

Commit

Permalink
Introduce agentctl (#60)
Browse files Browse the repository at this point in the history
* cmd/agentctl: initial commit

* pkg/client: add client for agent http/1 API

* implement agentctl and hook up to CI for builds

* fix test fail

* address review feedback

* agentctl: exit with failure when config-sync fails

* Update docs/scraping-service.md

Co-authored-by: gotjosh <josue.abreu@gmail.com>

* Update pkg/agentctl/sync.go

Co-authored-by: gotjosh <josue.abreu@gmail.com>

* address review feedback

* return error at the end of syncing if anything failed

Co-authored-by: gotjosh <josue.abreu@gmail.com>
  • Loading branch information
rfratto and gotjosh committed May 11, 2020
1 parent 3dd3fe8 commit c54d2c4
Show file tree
Hide file tree
Showing 41 changed files with 4,608 additions and 25 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/containerize.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ jobs:
run: make test

- name: Make Container
run: RELEASE_BUILD=true make agent-image
run: RELEASE_BUILD=true make agent-image agentctl-image

- name: Push Container
run: RELEASE_BUILD=true make push-agent-image
run: RELEASE_BUILD=true make push-agent-image push-agent-ctl-image
4 changes: 2 additions & 2 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,6 @@ jobs:
export PATH="$(go env GOPATH)/bin:$PATH"
export RELEASE_TAG=${GITHUB_REF##*/}
make RELEASE_BUILD=true agent-image
make RELEASE_BUILD=true push-agent-image
make RELEASE_BUILD=true agent-image agentctl-image
make RELEASE_BUILD=true push-agent-image push-agentctl-image
make RELEASE_BUILD=true publish
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,4 +50,4 @@ jobs:
- name: Check out code into the Go module directory
uses: actions/checkout@v2
- name: Build
run: make cmd/agent/agent
run: make cmd/agent/agent cmd/agentctl/agentctl
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,5 @@
cover.out

cmd/agent/agent
cmd/agentctl/agentctl
dist/
2 changes: 1 addition & 1 deletion .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ run:
# include test files or not, default is true
tests: true

# list of build tags, all linters use it. Default is empty list.
# list of build2 tags, all linters use it. Default is empty list.
build-tags:

# which dirs to skip: they won't be analyzed;
Expand Down
25 changes: 22 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# TODO(rfratto): docker images

.DEFAULT_GOAL := all
.PHONY: all agent check-mod int test clean cmd/agent/agent protos
.PHONY: all agent agentctl check-mod int test clean cmd/agent/agent cmd/agentctl/agentctl protos

SHELL = /usr/bin/env bash

Expand Down Expand Up @@ -45,7 +45,7 @@ GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
DONT_FIND := -name tools -prune -o -name vendor -prune -o -name .git -prune -o -name .cache -prune -o -name .pkg -prune -o

# Build flags
VPREFIX := github.com/grafana/agent/cmd/agent/build
VPREFIX := github.com/grafana/agent/pkg/build
GO_LDFLAGS := -X $(VPREFIX).Branch=$(GIT_BRANCH) -X $(VPREFIX).Version=$(IMAGE_TAG) -X $(VPREFIX).Revision=$(GIT_REVISION) -X $(VPREFIX).BuildUser=$(shell whoami)@$(shell hostname) -X $(VPREFIX).BuildDate=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
GO_FLAGS := -ldflags "-extldflags \"-static\" -s -w $(GO_LDFLAGS)" -tags netgo $(MOD_FLAG)
DEBUG_GO_FLAGS := -gcflags "all=-N -l" -ldflags "-extldflags \"-static\" $(GO_LDFLAGS)" -tags netgo $(MOD_FLAG)
Expand Down Expand Up @@ -96,22 +96,40 @@ endif
###################
# Primary Targets #
###################
all: protos agent
all: protos agent agentctl
agent: cmd/agent/agent
agentctl: cmd/agentctl/agentctl

cmd/agent/agent: cmd/agent/main.go
CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D)
$(NETGO_CHECK)

cmd/agentctl/agentctl: cmd/agentctl/main.go
CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D)
$(NETGO_CHECK)

agent-image:
docker build --build-arg RELEASE_BUILD=$(RELEASE_BUILD) --build-arg IMAGE_TAG=$(IMAGE_TAG) \
-t $(IMAGE_PREFIX)/agent:latest -f cmd/agent/Dockerfile .
docker tag $(IMAGE_PREFIX)/agent:latest $(IMAGE_PREFIX)/agent:$(IMAGE_TAG)

agentctl-image:
docker build --build-arg RELEASE_BUILD=$(RELEASE_BUILD) --build-arg IMAGE_TAG=$(IMAGE_TAG) \
-t $(IMAGE_PREFIX)/agentctl:latest -f cmd/agentctl/Dockerfile .
docker tag $(IMAGE_PREFIX)/agentctl:latest $(IMAGE_PREFIX)/agentctl:$(IMAGE_TAG)

push-agent-image:
docker push $(IMAGE_PREFIX)/agent:latest
docker push $(IMAGE_PREFIX)/agent:$(IMAGE_TAG)

push-agentctl-image:
docker push $(IMAGE_PREFIX)/agentctl:latest
docker push $(IMAGE_PREFIX)/agentctl:$(IMAGE_TAG)

install:
CGO_ENABLED=0 go install $(GO_FLAGS) ./cmd/agent
CGO_ENABLED=0 go install $(GO_FLAGS) ./cmd/agentctl

#######################
# Development targets #
#######################
Expand Down Expand Up @@ -140,6 +158,7 @@ example-dashboards:
GOX = gox $(GO_FLAGS) -parallel=2 -output="dist/{{.Dir}}-{{.OS}}-{{.Arch}}"
dist:
CGO_ENABLED=0 $(GOX) -osarch="linux/amd64 darwin/amd64 windows/amd64 freebsd/amd64" ./cmd/agent
CGO_ENABLED=0 $(GOX) -osarch="linux/amd64 darwin/amd64 windows/amd64 freebsd/amd64" ./cmd/agentctl
for i in dist/*; do zip -j -m $$i.zip $$i; done
pushd dist && sha256sum * > SHA256SUMS && popd

Expand Down
3 changes: 1 addition & 2 deletions cmd/agent/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,7 @@ import (
"os"

// Adds version information

_ "github.com/grafana/agent/cmd/agent/build"
_ "github.com/grafana/agent/pkg/build"

"github.com/cortexproject/cortex/pkg/util"
"github.com/go-kit/kit/log/level"
Expand Down
12 changes: 12 additions & 0 deletions cmd/agentctl/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
FROM golang:1.14 as build
COPY . /src/agent
WORKDIR /src/agent
ARG RELEASE_BUILD=false
ARG IMAGE_TAG
RUN make clean && make IMAGE_TAG=${IMAGE_TAG} RELEASE_BUILD=${RELEASE_BUILD} BUILD_IN_CONTAINER=false agentctl

FROM alpine:3.9
RUN apk add --update --no-cache ca-certificates
COPY --from=build /src/agent/cmd/agentctl/agentctl /bin/agentctl

ENTRYPOINT ["/bin/agentctl"]
75 changes: 75 additions & 0 deletions cmd/agentctl/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
// Command agentctl provides utilities for interacting with Grafana Cloud Agent
package main

import (
"os"

// Adds version information
_ "github.com/grafana/agent/pkg/build"
"github.com/prometheus/common/version"

"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/grafana/agent/pkg/agentctl"
"github.com/grafana/agent/pkg/client"
"github.com/spf13/cobra"
)

func main() {
var cmd = &cobra.Command{
Use: "agentctl",
Short: "Tools for interacting with the Grafana Cloud Agent",
Version: version.Print("agentctl"),
}
cmd.SetVersionTemplate("{{ .Version }}\n")

cmd.AddCommand(
configSyncCmd(),
)

_ = cmd.Execute()
}

func configSyncCmd() *cobra.Command {
var (
agentAddr string
)

cmd := &cobra.Command{
Use: "config-sync [directory]",
Short: "Sync config files from a directory to an Agent's config management API",
Long: `config-sync loads all files ending with .yml or .yaml from the specified
directory and uploads them the the config management API. The name of the config
uploaded will be the base name of the file (e.g., the name of the file without
its extension).
The directory is used as the source-of-truth for the entire set of configs that
should be present in the API. config-sync will delete all existing configs from the API
that do not match any of the names of the configs that were uploaded from the
source-of-truth directory.`,
Args: cobra.ExactArgs(1),

Run: func(_ *cobra.Command, args []string) {
directory := args[0]
cli := client.New(agentAddr)

logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stdout))

err := agentctl.ConfigSync(logger, cli.PrometheusClient, directory)
if err != nil {
level.Error(logger).Log("msg", "failed to sync config", "err", err)
os.Exit(1)
}
},
}

cmd.Flags().StringVarP(&agentAddr, "addr", "a", "http://localhost:12345", "address of the agent to connect to")
must(cmd.MarkFlagRequired("addr"))
return cmd
}

func must(err error) {
if err != nil {
panic(err)
}
}
18 changes: 14 additions & 4 deletions docs/scraping-service.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,8 @@ contents.

When Scraping Service Mode is enabled, Agents **disallow** specifying
instance configurations locally in the configuration file; using the KV store
is required.

GitOps-friendly tooling is planned to automatically load instance configuration
files.
is required. [`agentctl`](#agentctl) can be used to manually sync
instance configuration files to the Agent's API server.

## Distributed Hash Ring

Expand Down Expand Up @@ -151,3 +149,15 @@ Note that there are no instance configs present in this example; instance
configs must be passed to the API for the Agent to start scraping metrics.
See [the docker-compose Scraping Service Example](../example/README.md)
for how to run a Scraping Service Agent cluster locally.

## agentctl

`agentctl` is a tool included with this repository that helps users to interact
with the new Config Management API. The `agentctl config-sync` subcommand uses
all YAML files as a source of truth and syncs their contents with the API.
Entries in the API not in the synced directly will be deleted.

`agentctl` is distributed in binary form with each release and as a Docker
container with the `grafana/agentctl` image. Tanka configurations that
utilize `grafana/agentctl` and sync a set of configurations to the API
are planned for the future.
6 changes: 3 additions & 3 deletions example/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,6 @@ configurations. A set of instance configuration files is provided in
[`agent/instance-configs`](./agent/instance-configs) that will work with the
Docker Compose example.

A bash script has been provided to automatically deploy these example
instance configuration files to the running cluster; run
`bash ./scripts/deploy-configs.sh` from this directory to do so.
The `agentctl` tool provided by this repository can automatically deploy
those configs for you; run `agentctl config-sync -a http://localhost:12345 ./agent/instance-configs`
from this directory to do so.
5 changes: 0 additions & 5 deletions example/scripts/deploy-configs.sh

This file was deleted.

1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ require (
github.com/prometheus/common v0.9.1
github.com/prometheus/procfs v0.0.11 // indirect
github.com/prometheus/prometheus v1.8.2-0.20200213233353-b90be6f32a33
github.com/spf13/cobra v0.0.3
github.com/stretchr/testify v1.4.0
github.com/weaveworks/common v0.0.0-20200310113808-2708ba4e60a4
go.uber.org/atomic v1.5.0
Expand Down
5 changes: 5 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,12 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMx
github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q=
Expand Down Expand Up @@ -443,6 +445,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/influxdata/influxdb v1.7.7/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=
github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
Expand Down Expand Up @@ -719,6 +722,7 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
Expand Down Expand Up @@ -1024,6 +1028,7 @@ google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfG
google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
Expand Down
Loading

0 comments on commit c54d2c4

Please sign in to comment.