Website for Thanos' docs using Hugo.

.circleci/config.yml

@@ -8,6 +8,7 @@ jobs:
     working_directory: /go/src/github.com/improbable-eng/thanos
     environment:
       GO111MODULE: 'on'
+      GOBIN: "/go/src/github.com/improbable-eng/thanos/.bin"
     steps:
       - checkout
       - setup_remote_docker:
@@ -48,6 +49,8 @@ jobs:
   cross_build:
     machine: true
     working_directory: /home/circleci/.go_workspace/src/github.com/improbable-eng/thanos
+    environment:
+      GOBIN: "/go/src/github.com/improbable-eng/thanos/.bin"
     steps:
     - checkout
     - run: make crossbuild
@@ -61,6 +64,8 @@ jobs:
       # Available from https://hub.docker.com/r/circleci/golang/
       - image: circleci/golang:1.12
     working_directory: /go/src/github.com/improbable-eng/thanos
+    environment:
+      GOBIN: "/go/src/github.com/improbable-eng/thanos/.bin"
     steps:
       - checkout
       - setup_remote_docker:
@@ -78,6 +83,8 @@ jobs:
       # Available from https://hub.docker.com/r/circleci/golang/
       - image: circleci/golang:1.12
     working_directory: /go/src/github.com/improbable-eng/thanos
+    environment:
+      GOBIN: "/go/src/github.com/improbable-eng/thanos/.bin"
     steps:
       - checkout
       - setup_remote_docker:
@@ -127,4 +134,4 @@ workflows:
           tags:
             only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/
           branches:
-            ignore: /.*/
+            ignore: /.*/

.gitignore

@@ -17,3 +17,7 @@ data/
 
 /.idea
 /*.iml
+
+website/public/
+website/docs-pre-processed/
+website/hugo-generated.yaml

CODE_OF_CONDUCT.md

@@ -58,7 +58,4 @@ face temporary or permanent repercussions as determined by other members of the
 
 ## Attribution
 
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
-
-[homepage]: http://contributor-covenant.org
-[version]: http://contributor-covenant.org/version/1/4/
+This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.4, available at [http://contributor-covenant.org/version/1/4](http://contributor-covenant.org/version/1/4/)

Makefile

@@ -2,53 +2,54 @@ PREFIX            ?= $(shell pwd)
 FILES             ?= $(shell find . -type f -name '*.go' -not -path "./vendor/*")
 DOCKER_IMAGE_NAME ?= thanos
 DOCKER_IMAGE_TAG  ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD))-$(shell date +%Y-%m-%d)-$(shell git rev-parse --short HEAD)
-# $GOPATH/bin might not be in $PATH, so we can't assume `which` would give use
-# the path of promu et al. As for selecting the first GOPATH, we assume:
-# - most people only have one GOPATH at a time;
-# - if you don't have one or any of those tools installed, running `go get`
-#   would place them in the first GOPATH.
-# It's possible that any of the tools would be installed in the other GOPATHs,
-# but for simplicity sake we just make sure they exist in the first one, and
-# then keep using those.
-FIRST_GOPATH      ?= $(firstword $(subst :, ,$(shell go env GOPATH)))
+
 TMP_GOPATH        ?= /tmp/thanos-go
-BIN_DIR           ?= $(FIRST_GOPATH)/bin
+GOBIN             ?= ${GOPATH}/bin
 GO111MODULE       ?= on
 export GO111MODULE
 
 # Tools.
-EMBEDMD           ?= $(BIN_DIR)/embedmd-$(EMBEDMD_VERSION)
+EMBEDMD           ?= $(GOBIN)/embedmd-$(EMBEDMD_VERSION)
 # v2.0.0
 EMBEDMD_VERSION   ?= 97c13d6e41602fc6e397eb51c45f38069371a969
-ERRCHECK          ?= $(BIN_DIR)/errcheck-$(ERRCHECK_VERSION)
+ERRCHECK          ?= $(GOBIN)/errcheck-$(ERRCHECK_VERSION)
 # v1.2.0
 ERRCHECK_VERSION  ?= e14f8d59a22d460d56c5ee92507cd94c78fbf274
-LICHE             ?= $(BIN_DIR)/liche-$(LICHE_VERSION)
+LICHE             ?= $(GOBIN)/liche-$(LICHE_VERSION)
 LICHE_VERSION     ?= 2a2e6e56f6c615c17b2e116669c4cdb31b5453f3
-GOIMPORTS         ?= $(BIN_DIR)/goimports-$(GOIMPORTS_VERSION)
+GOIMPORTS         ?= $(GOBIN)/goimports-$(GOIMPORTS_VERSION)
 GOIMPORTS_VERSION ?= 1c3d964395ce8f04f3b03b30aaed0b096c08c3c6
-PROMU             ?= $(BIN_DIR)/promu-$(PROMU_VERSION)
+PROMU             ?= $(GOBIN)/promu-$(PROMU_VERSION)
 # v0.2.0
 PROMU_VERSION     ?= 264dc36af9ea3103255063497636bd5713e3e9c1
-PROTOC            ?= $(BIN_DIR)/protoc-$(PROTOC_VERSION)
+PROTOC            ?= $(GOBIN)/protoc-$(PROTOC_VERSION)
 PROTOC_VERSION    ?= 3.4.0
+# v0.54.0
+HUGO_VERSION      ?= b1a82c61aba067952fdae2f73b826fe7d0f3fc2f
+HUGO              ?= $(GOBIN)/hugo-$(HUGO_VERSION)
 GIT               ?= $(shell which git)
 BZR               ?= $(shell which bzr)
 
+WEB_DIR           ?= website
+PUBLIC_DIR        ?= $(WEB_DIR)/public
+ME                ?= $(shell whoami)
+
 # E2e test deps.
 # Referenced by github.com/improbable-eng/thanos/blob/master/docs/getting_started.md#prometheus
 
-# Limitied prom version, because testing was not possibe. This should fix it: https://github.com/improbable-eng/thanos/issues/758
+# Limitied prom version, because testing was not possible. This should fix it: https://github.com/improbable-eng/thanos/issues/758
 PROM_VERSIONS           ?=v2.4.3 v2.5.0
 ALERTMANAGER_VERSION    ?=v0.15.2
 MINIO_SERVER_VERSION    ?=RELEASE.2018-10-06T00-15-16Z
 
-# fetch_go_bin_version downloads (go gets) the binary from specific version and installs it in $(BIN_DIR)/<bin>-<version>
+# fetch_go_bin_version downloads (go gets) the binary from specific version and installs it in $(GOBIN)/<bin>-<version>
 # arguments:
 # $(1): Install path. (e.g github.com/campoy/embedmd)
 # $(2): Tag or revision for checkout.
+# TODO(bwplotka): Move to just using modules, however make sure to not use or edit Thanos go.mod file!
 define fetch_go_bin_version
-	@mkdir -p $(BIN_DIR)
+	@mkdir -p $(GOBIN)
+	@mkdir -p $(TMP_GOPATH)
 
 	@echo ">> fetching $(1)@$(2) revision/version"
 	@if [ ! -d '$(TMP_GOPATH)/src/$(1)' ]; then \
@@ -59,8 +60,27 @@ define fetch_go_bin_version
 	@CDPATH='' cd -- '$(TMP_GOPATH)/src/$(1)' && git checkout -f -q '$(2)'
 	@echo ">> installing $(1)@$(2)"
 	@GOBIN='$(TMP_GOPATH)/bin' GOPATH='$(TMP_GOPATH)' GO111MODULE='off' go install '$(1)'
-	@mv -- '$(TMP_GOPATH)/bin/$(shell basename $(1))' '$(BIN_DIR)/$(shell basename $(1))-$(2)'
-	@echo ">> produced $(BIN_DIR)/$(shell basename $(1))-$(2)"
+	@mv -- '$(TMP_GOPATH)/bin/$(shell basename $(1))' '$(GOBIN)/$(shell basename $(1))-$(2)'
+	@echo ">> produced $(GOBIN)/$(shell basename $(1))-$(2)"
+
+endef
+
+define require_clean_work_tree
+	@git update-index -q --ignore-submodules --refresh
+
+    @if ! git diff-files --quiet --ignore-submodules --; then \
+        echo >&2 "cannot $1: you have unstaged changes."; \
+        git diff-files --name-status -r --ignore-submodules -- >&2; \
+        echo >&2 "Please commit or stash them."; \
+        exit 1; \
+    fi
+
+    @if ! git diff-index --cached --quiet HEAD --ignore-submodules --; then \
+        echo >&2 "cannot $1: your index contains uncommitted changes."; \
+        git diff-index --cached --name-status -r --ignore-submodules HEAD -- >&2; \
+        echo >&2 "Please commit or stash them."; \
+        exit 1; \
+    fi
 
 endef
 
@@ -81,7 +101,7 @@ assets:
 # build builds Thanos binary using `promu`.
 .PHONY: build
 build: check-git check-bzr go-mod-tidy $(PROMU)
-	@echo ">> building binaries"
+	@echo ">> building binaries $(GOBIN)"
 	@$(PROMU) build --prefix $(PREFIX)
 
 # crossbuild builds all binaries for all platforms.
@@ -147,7 +167,7 @@ promu: $(PROMU)
 .PHONY: tarball
 tarball: $(PROMU)
 	@echo ">> building release tarball"
-	$(PROMU) tarball --prefix $(PREFIX) $(BIN_DIR)
+	$(PROMU) tarball --prefix $(PREFIX) $(GOBIN)
 
 .PHONY: tarballs-release
 tarballs-release: $(PROMU)
@@ -203,6 +223,37 @@ else
 	@echo >&2 "No bzr binary found."; exit 1
 endif
 
+.PHONY: web-pre-process
+web-pre-process:
+	@echo ">> running documentation website pre processing"
+	@bash scripts/websitepreprocess.sh
+
+.PHONY: web
+web: web-pre-process $(HUGO)
+	@echo ">> building documentation website"
+	# TODO(bwplotka): Make it --gc
+	@cd $(WEB_DIR) && HUGO_ENV=production $(HUGO) --config hugo-generated.yaml --minify -v
+
+.PHONY: web-serve
+web-serve: web-pre-process $(HUGO)
+	@echo ">> serving documentation website"
+	@cd $(WEB_DIR) && $(HUGO) --config hugo-generated.yaml -v server
+
+.PHONY: web-deploy
+web-deploy:
+	# Requires git creds configured beforehand.
+	$(call require_clean_work_tree,"deploy website")
+	@rm -rf $(PUBLIC_DIR)
+	@mkdir $(PUBLIC_DIR)
+	@git worktree prune
+	@rm -rf .git/worktrees/$(PUBLIC_DIR)/
+	@git fetch origin
+	@git worktree add -B gh-pages $(PUBLIC_DIR) origin/gh-pages
+	@rm -rf $(PUBLIC_DIR)/*
+	@make web
+	@cd $(PUBLIC_DIR) && git add --all && git commit -m "Publishing to gh-pages as $(ME)" && cd ..
+	@git push origin gh-pages
+
 # non-phony targets
 $(EMBEDMD):
 	$(call fetch_go_bin_version,github.com/campoy/embedmd,$(EMBEDMD_VERSION))
@@ -219,10 +270,13 @@ $(LICHE):
 $(PROMU):
 	$(call fetch_go_bin_version,github.com/prometheus/promu,$(PROMU_VERSION))
 
+$(HUGO):
+	$(call fetch_go_bin_version,github.com/gohugoio/hugo,$(HUGO_VERSION))
+
 $(PROTOC):
 	@mkdir -p $(TMP_GOPATH)
 	@echo ">> fetching protoc@${PROTOC_VERSION}"
 	@PROTOC_VERSION="$(PROTOC_VERSION)" TMP_GOPATH="$(TMP_GOPATH)" scripts/installprotoc.sh
 	@echo ">> installing protoc@${PROTOC_VERSION}"
-	@mv -- "$(TMP_GOPATH)/bin/protoc" "$(BIN_DIR)/protoc-$(PROTOC_VERSION)"
-	@echo ">> produced $(BIN_DIR)/protoc-$(PROTOC_VERSION)"
+	@mv -- "$(TMP_GOPATH)/bin/protoc" "$(GOBIN)/protoc-$(PROTOC_VERSION)"
+	@echo ">> produced $(GOBIN)/protoc-$(PROTOC_VERSION)"

docs/components/bucket.md

@@ -1,11 +1,17 @@
+---
+title: Bucket
+type: docs
+menu: components
+---
+
 # Bucket
 
 The bucket component of Thanos is a set of commands to inspect data in object storage buckets.
 It is normally run as a stand alone command to aid with troubleshooting.
 
 Example:
 
-```
+```bash
 $ thanos bucket verify --objstore.config-file=bucket.yml
 ```
 

docs/components/compact.md

@@ -1,11 +1,17 @@
+---
+title: Compact
+type: docs
+menu: components
+---
+
 # Compact
 
 The compactor component of Thanos applies the compaction procedure of the Prometheus 2.0 storage engine to block data stored in object storage.
 It is generally not semantically concurrency safe and must be deployed as a singleton against a bucket.
 
 Example:
 
-```
+```bash
 $ thanos compact --data-dir /tmp/thanos-compact --objstore.config-file=bucket.yml
 ```
 

docs/components/query.md

@@ -1,3 +1,9 @@
+---
+title: Query
+type: docs
+menu: components
+---
+
 # Query
 
 The query component implements the Prometheus HTTP v1 API to query data in a Thanos cluster via PromQL.
@@ -7,7 +13,7 @@ on how to connect querier with desired StoreAPIs.
 
 Querier currently is fully stateless and horizontally scalable.
 
-```
+```bash
 $ thanos query \
     --http-address     "0.0.0.0:9090" \
     --store            "<store-api>:<grpc-port>" \

docs/components/rule.md

@@ -1,3 +1,9 @@
+---
+title: Rule
+type: docs
+menu: components
+---
+
 # Rule
 
 _**NOTE:** The rule component is experimental since it has conceptual tradeoffs that might not be favorable for most use cases. It is recommended to keep deploying rules to the relevant Prometheus servers._
@@ -8,7 +14,7 @@ The rule component evaluates Prometheus recording and alerting rules against ran
 
 The data of each rule node can be labeled to satisfy the clusters labeling scheme. High-availability pairs can be run in parallel and should be distinguished by the designated replica label, just like regular Prometheus servers.
 
-```
+```bash
 $ thanos rule \
     --data-dir          "/path/to/data" \
     --eval-interval     "30s" \

docs/components/sidecar.md

@@ -1,3 +1,9 @@
+---
+title: Sidecar
+type: docs
+menu: components
+---
+
 # Sidecar
 
 The sidecar component of Thanos gets deployed along with a Prometheus instance. It implements Thanos' Store API on top of Prometheus' remote-read API and advertises itself as a data source to the cluster. Thereby queriers in the cluster can treat Prometheus servers as yet another source of time series data without directly talking to its APIs.
@@ -13,14 +19,14 @@ Prometheus servers connected to the Thanos cluster via the sidecar are subject t
 The retention is recommended to not be lower than three times the block duration. This achieves resilience in the face of connectivity issues
 to the object storage since all local data will remain available within the Thanos cluster. If connectivity gets restored the backlog of blocks gets uploaded to the object storage.
 
-```console
+```bash
 $ prometheus \
   --storage.tsdb.max-block-duration=2h \
   --storage.tsdb.min-block-duration=2h \
   --web.enable-lifecycle
 ```
 
-```console
+```bash
 $ thanos sidecar \
     --tsdb.path        "/path/to/prometheus/data/dir" \
     --prometheus.url   "http://localhost:9090" \

docs/components/store.md

@@ -1,9 +1,15 @@
+---
+title: Store
+type: docs
+menu: components
+---
+
 # Store
 
 The store component of Thanos implements the Store API on top of historical data in an object storage bucket. It acts primarily as an API gateway and therefore does not need significant amounts of local disk space. It joins a Thanos cluster on startup and advertises the data it can access.
 It keeps a small amount of information about all remote blocks on local disk and keeps it in sync with the bucket. This data is generally safe to delete across restarts at the cost of increased startup times.
 
-```
+```bash
 $ thanos store \
     --data-dir        "/local/state/data/dir" \
     --cluster.peers    "thanos-cluster.example.org" \

docs/troubleshooting/dev.md → docs/contributing/dev.md

@@ -1,4 +1,8 @@
-# Troubleshooting for dev workflow
+---
+title: Troubleshooting for dev workflow
+type: doc
+menu: contributing
+---
 
 ## Dep `grouped write of manifest, lock and vendor: scratch directory ... already exists, please remove it`