Merge branch 'master' of https://github.com/thanos-io/thanos

thanos-io · Sep 26, 2019 · ea7b497 · ea7b497
2 parents a9f644b + 99bc7d2
commit ea7b497
Show file tree

Hide file tree

Showing 199 changed files with 6,015 additions and 7,342 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -3,12 +3,18 @@ version: 2
 jobs:
   test:
     docker:
-      # Available from https://hub.docker.com/r/circleci/golang/
-      - image: circleci/golang:1.12.5
+     # Build by Thanos make docker-ci
+      - image: quay.io/thanos/thanos-ci:v0.1.0
     working_directory: /go/src/github.com/thanos-io/thanos
     environment:
       GO111MODULE: 'on'
-      GOBIN: "/go/bin"
+      # Run garbage collection more aggresively to avoid getting OOMed during the lint phase (4GB limit).
+      GOGC: "20"
+      # By default Go uses GOMAXPROCS but a Circle CI executor has many
+      # cores (> 30) while the CPU and RAM resources are throttled. If we
+      # don't limit this to the number of allocated cores, the job is
+      # likely to get OOMed and killed.
+      GOOPTS: "-p 2"
     steps:
       - checkout
       - setup_remote_docker:
@@ -22,28 +28,20 @@ jobs:
               echo $GOOGLE_APPLICATION_CREDENTIALS_CONTENT > $GOOGLE_APPLICATION_CREDENTIALS
               echo "Awesome! GCS integration tests are enabled."
             fi
-      - run: make check-go-mod
+      - run: make deps
       - run: make lint
       - run: make check-docs
       - run: make format
       - run:
           name: "Run all tests"
-          # TODO(bplotka): Setup some S3 tests for CI.
+          # TODO(bwplotka): Setup some S3 tests for CI.
+          # taskset sets CPU affinity to 2 (current CPU limit).
           command: |
             if [ -z ${GCP_PROJECT} ]; then
-              export THANOS_SKIP_GCS_TESTS="true"
-              echo "Skipping GCS tests."
+              taskset 2 make test-local
+              exit
             fi
-            export THANOS_SKIP_S3_AWS_TESTS="true"
-            echo "Skipping AWS tests."
-            export THANOS_SKIP_AZURE_TESTS="true"
-            echo "Skipping Azure tests."
-            export THANOS_SKIP_SWIFT_TESTS="true"
-            echo "Skipping SWIFT tests."
-            export THANOS_SKIP_TENCENT_COS_TESTS="true"
-            echo "Skipping TENCENT COS tests."
-
-            make test
+            taskset 2 make test-only-gcs
 
   # Cross build is needed for publish_release but needs to be done outside of docker.
   cross_build:
@@ -61,11 +59,9 @@ jobs:
 
   publish_master:
     docker:
-      # Available from https://hub.docker.com/r/circleci/golang/
-      - image: circleci/golang:1.12.5
+      # Build by Thanos make docker-ci
+      - image: quay.io/thanos/thanos-ci:v0.1.0
     working_directory: /go/src/github.com/thanos-io/thanos
-    environment:
-      GOBIN: "/go/bin"
     steps:
       - checkout
       - setup_remote_docker:
@@ -75,6 +71,9 @@ jobs:
       - run: ln -s .build/linux-amd64/thanos thanos
       - run: make docker
       - run: docker run thanos --help
+      # Upload to both dockerhub and quay.io.
+      - run: echo "${DOCKERHUB_PASSWORD}" | docker login -u="${DOCKERHUB_USERNAME}" --password-stdin
+      - run: make docker-push DOCKER_IMAGE_REPO=thanosio/thanos
       - run: echo "${QUAY_PASSWORD}" | docker login -u="${QUAY_USERNAME}" quay.io --password-stdin
       - run: make docker-push
 
@@ -98,6 +97,9 @@ jobs:
       - run: ln -s .build/linux-amd64/thanos thanos
       - run: make docker
       - run: docker run thanos --help
+      # Upload to both dockerhub and quay.io.
+      - run: echo "${DOCKERHUB_PASSWORD}" | docker login -u="${DOCKERHUB_USERNAME}" --password-stdin
+      - run: make docker-push DOCKER_IMAGE_REPO=thanosio/thanos DOCKER_IMAGE_TAG=$CIRCLE_TAG
       - run: echo "${QUAY_PASSWORD}" | docker login -u="${QUAY_USERNAME}" quay.io --password-stdin
       - run: make docker-push DOCKER_IMAGE_TAG=$CIRCLE_TAG
 

diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,5 @@
+.envrc
+
 /prometheus
 /thanos
 /vendor/
@@ -21,3 +23,5 @@ data/
 website/public/
 website/docs-pre-processed/
 !website/data
+
+tmp/bin
diff --git a/.promu.yml b/.promu.yml
@@ -19,3 +19,4 @@ crossbuild:
         - darwin/amd64
         - linux/arm64
         - windows/amd64
+        - freebsd/amd64
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,15 +9,65 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 
 We use *breaking* word for marking changes that are not backward compatible (relates only to v0.y.z releases.)
 
-## Unreleased.
+## Unreleased
 
-- [#1338](https://github.com/thanos-io/thanos/pull/1338) Querier still warns on store API duplicate, but allows a single one from duplicated set. This is gracefully warn about the problematic logic and not disrupt immediately.
+## Added
+- [#1540](https://github.com/thanos-io/thanos/pull/1540) Thanos Downsample added `/-/ready` and `/-/healthy` endpoints.
+- [#1538](https://github.com/thanos-io/thanos/pull/1538) Thanos Rule added `/-/ready` and `/-/healthy` endpoints.
+- [#1537](https://github.com/thanos-io/thanos/pull/1537) Thanos Receive added `/-/ready` and `/-/healthy` endpoints.
+- [#1460](https://github.com/thanos-io/thanos/pull/1460) Thanos Store Added `/-/ready` and `/-/healthy` endpoints.
+- [#1534](https://github.com/thanos-io/thanos/pull/1534) Thanos Query Added `/-/ready` and `/-/healthy` endpoints.
+- [#1533](https://github.com/thanos-io/thanos/pull/1533) Thanos inspect now supports the timeout flag.
 
 ### Fixed
 
-- [#1327](https://github.com/thanos-io/thanos/pull/1327) `/series` API end-point now properly returns an empty array just like Prometheus if there are no results
+- [#1525](https://github.com/thanos-io/thanos/pull/1525) Thanos now deletes block's file in correct order allowing to detect partial blocks without problems.
+- [#1505](https://github.com/thanos-io/thanos/pull/1505) Thanos store now removes invalid local cache blocks.
 
+## v0.7.0 - 2019.09.02
+
+Accepted into CNCF:
+- Thanos moved to new repository <https://github.com/thanos-io/thanos>
+- Docker images moved to <https://quay.io/thanos/thanos> and mirrored at <https://hub.docker.com/r/thanosio/thanos>
+- Slack moved to <https://slack.cncf.io> `#thanos`/`#thanos-dev`/`#thanos-prs`
+
+### Added
+
+- [#1478](https://github.com/thanos-io/thanos/pull/1478) Thanos components now exposes gRPC server metrics as soon as server starts, to provide more reliable data for instrumentation.
+- [#1378](https://github.com/thanos-io/thanos/pull/1378) Thanos Receive now exposes `thanos_receive_config_hash`, `thanos_receive_config_last_reload_successful` and `thanos_receive_config_last_reload_success_timestamp_seconds` metrics to track latest configuration change
+- [#1268](https://github.com/thanos-io/thanos/pull/1268) Thanos Sidecar added support for newest Prometheus streaming remote read added [here](https://github.com/prometheus/prometheus/pull/5703). This massively improves memory required by single
+  request for both Prometheus and sidecar. Single requests now should take constant amount of memory on sidecar, so resource consumption prediction is now straightforward. This will be used if you have Prometheus `2.13` or `2.12-master`.
+- [#1358](https://github.com/thanos-io/thanos/pull/1358) Added `part_size` configuration option for HTTP multipart requests minimum part size for S3 storage type
+- [#1363](https://github.com/thanos-io/thanos/pull/1363) Thanos Receive now exposes `thanos_receive_hashring_nodes` and `thanos_receive_hashring_tenants` metrics to monitor status of hash-rings
+- [#1362](https://github.com/thanos-io/thanos/pull/1362) Optional `replicaLabels` param for `/query` and `/query_range` querier endpoints. When provided overwrite the `query.replica-label` cli flags.
+- [#1395](https://github.com/thanos-io/thanos/pull/1395) Thanos Sidecar added `/-/ready` and `/-/healthy` endpoints to Thanos sidecar.
+- [#1297](https://github.com/thanos-io/thanos/pull/1297) Thanos Compact added `/-/ready` and `/-/healthy` endpoints to Thanos compact.
+- [#1431](https://github.com/thanos-io/thanos/pull/1431) Thanos Query added hidden flag to allow the use of downsampled resolution data for instant queries.
+- [#1408](https://github.com/thanos-io/thanos/pull/1408) Thanos Store Gateway can now allow the specifying of supported time ranges it will serve (time sharding). Flags: `min-time` & `max-time`
+
+### Changed
+
+- [#1362](https://github.com/thanos-io/thanos/pull/1362) `query.replica-label` configuration can be provided more than once for multiple deduplication labels like: `--query.replica-label=prometheus_replica --query.replica-label=service`.
+- [#1414](https://github.com/thanos-io/thanos/pull/1413) Upgraded important dependencies: Prometheus to 2.12-rc.0. TSDB is now part of Prometheus.
+- [#1380](https://github.com/thanos-io/thanos/pull/1380) Upgraded important dependencies: Prometheus to 2.11.1 and TSDB to 0.9.1. Some changes affecting Querier:
+  - [ENHANCEMENT] Query performance improvement: Efficient iteration and search in HashForLabels and HashWithoutLabels. #5707
+  - [ENHANCEMENT] Optimize queries using regexp for set lookups. tsdb#602
+  - [BUGFIX] prometheus_tsdb_compactions_failed_total is now incremented on any compaction failure. tsdb#613
+  - [BUGFIX] PromQL: Correctly display {__name__="a"}.
+- [#1338](https://github.com/thanos-io/thanos/pull/1338) Thanos Query still warns on store API duplicate, but allows a single one from duplicated set. This is gracefully warn about the problematic logic and not disrupt immediately.
+- [#1385](https://github.com/thanos-io/thanos/pull/1385) Thanos Compact exposes flag to disable downsampling `downsampling.disable`.
+
+### Fixed
+
+- [#1327](https://github.com/thanos-io/thanos/pull/1327) Thanos Query `/series` API end-point now properly returns an empty array just like Prometheus if there are no results
 - [#1302](https://github.com/thanos-io/thanos/pull/1302) Thanos now efficiently reuses HTTP keep-alive connections
+- [#1371](https://github.com/thanos-io/thanos/pull/1371) Thanos Receive fixed race condition in hashring
+- [#1430](https://github.com/thanos-io/thanos/pull/1430) Thanos fixed value of GOMAXPROCS inside container.
+
+### Deprecated
+
+- [#1458](https://github.com/thanos-io/thanos/pull/1458) Thanos Query and Receive now use common instrumentation middleware. As as result, for sake of `http_requests_total` and `http_request_duration_seconds_bucket`; Thanos Query no longer exposes `thanos_query_api_instant_query_duration_seconds`, `thanos_query_api_range_query_duration_second` metrics and Thanos Receive no longer exposes `thanos_http_request_duration_seconds`, `thanos_http_requests_total`, `thanos_http_response_size_bytes`.
+- [#1423](https://github.com/thanos-io/thanos/pull/1423) Thanos Bench deprecated.
 
 ## [v0.6.0](https://github.com/thanos-io/thanos/releases/tag/v0.6.0) - 2019.07.18
 
@@ -59,7 +109,7 @@ The other `type` you can use is `JAEGER` now. The `config` keys and values are J
 
 ### Changed
 
-- [#1284](https://github.com/thanos-io/thanos/pull/1284) Add support for multiple label-sets in Info gRPC service. 
+- [#1284](https://github.com/thanos-io/thanos/pull/1284) Add support for multiple label-sets in Info gRPC service.
 This deprecates the single `Labels` slice of the `InfoResponse`, in a future release backward compatible handling for the single set of Labels will be removed. Upgrading to v0.6.0 or higher is advised.
 *breaking* If you run have duplicate queries in your Querier configuration with hierarchical federation of multiple Queries this PR makes Thanos Querier to detect this case and block all duplicates. Refer to 0.6.1 which at least allows for single replica to work.
 
@@ -160,7 +210,7 @@ This version moved tarballs to Golang 1.12.5 from 1.11 as well, so same warning
 :warning: **IMPORTANT** :warning: This is the last release that supports gossip. From Thanos v0.5.0, gossip will be completely removed.
 
 This release also disables gossip mode by default for all components.
-See [this](docs/proposals/completed/201809_gossip-removal.md) for more details.
+See [this](docs/proposals/201809_gossip-removal.md) for more details.
 
 :warning: This release moves Thanos docker images (NOT artifacts by accident) to Golang 1.12. This release includes change in GC's memory release which gives following effect (source: https://golang.org/doc/go1.12):
 
@@ -260,7 +310,7 @@ Note that this is required to have SRV resolution working on [Golang 1.11+ with
       * [ENHANCEMENT] When closing the db any running compaction will be cancelled so it doesn't block.
       * [CHANGE] *breaking* Renamed flag `--sync-delay` to `--consistency-delay` [#1053](https://github.com/thanos-io/thanos/pull/1053)
 
-  For ruler essentially whole TSDB CHANGELOG applies beween v0.4.0-v0.6.1: https://github.com/prometheus/tsdb/blob/master/CHANGELOG.md
+  For ruler essentially whole TSDB CHANGELOG applies between v0.4.0-v0.6.1: https://github.com/prometheus/tsdb/blob/master/CHANGELOG.md
 
   Note that this was added on TSDB and Prometheus: [FEATURE] Time-ovelapping blocks are now allowed. #370
   Whoever due to nature of Thanos compaction (distributed systems), for safety reason this is disabled for Thanos compactor for now.
@@ -437,7 +487,7 @@ Note lots of necessary breaking changes in flags that relates to bucket configur
 
 ## [v0.1.0](https://github.com/thanos-io/thanos/releases/tag/v0.1.0) - 2018.09.14
 
-Initial version to have a stable reference before [gossip protocol removal](/docs/proposals/completed/201809_gossip-removal.md).
+Initial version to have a stable reference before [gossip protocol removal](/docs/proposals/201809_gossip-removal.md).
 
 ### Added
 

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -64,7 +64,7 @@ CI runs GCS and inmem tests only for now. Not having these variables will produc
 6. If your change affects users (adds or removes feature) consider adding the item to [CHANGELOG](CHANGELOG.md)
 7. You may merge the Pull Request in once you have the sign-off of at least one developers with write access, or if you
    do not have permission to do that, you may request the second reviewer to merge it for you.
-8. If you feel like your PR waits too long for a review, feel free to ping [`#thanos-dev`](https://slack.cncf.io/) channel on our slack for review!
+8. If you feel like your PR waits too long for a review, feel free to ping [`#thanos-prs`](https://slack.cncf.io/) channel on our slack for review!
 
 ## Dependency management
 
@@ -83,7 +83,7 @@ go get example.com/some/module/pkg@vX.Y.Z
 Tidy up the `go.mod` and `go.sum` files:
 
 ```bash
-make go-mod-tidy
+make deps
 git add go.mod go.sum
 git commit
 ```

diff --git a/Dockerfile.thanos-ci b/Dockerfile.thanos-ci
@@ -0,0 +1,6 @@
+# Available from https://hub.docker.com/r/circleci/golang/
+FROM circleci/golang:1.12.5
+
+ENV GOBIN=/go/bin
+
+COPY tmp/bin ${GOBIN}
diff --git a/MAINTAINERS.md b/MAINTAINERS.md
@@ -7,10 +7,11 @@
 | Frederic Branczyk     | fbranczyk@gmail.com    | `@brancz`                | [@brancz](https://github.com/brancz)                       |
 | Giedrius Statkevičius | giedriuswork@gmail.com | `@Giedrius Statkevičius` | [@GiedriusS](https://github.com/GiedriusS)                 |
 | Povilas Versockas     | p.versockas@gmail.com  | `@povilasv`              | [@povilasv](https://github.com/povilasv)                   |
+| Matthias Loibl        | mail@matthiasloibl.com | `@metalmatze`            | [@metalmatze](https://github.com/metalmatze)               |
 
 We are bunch of people from different companies with various interests and skills.
 We are from different parts of Europe: Germany, Lithuania, Poland and UK. 
-We have something in common though: We all share the love for OpenSource, Golang, Prometheus, :coffee: and Observability topics.
+We have something in common though: We all share the love for OpenSource, Go, Prometheus, :coffee: and Observability topics.
 
 As either Software Developers or SRE (or both!) we've chosen to maintain (mostly in our free time) Thanos, the de facto way to scale awesome [Prometheus](https://prometheus.io) project.
 
@@ -96,4 +97,5 @@ maintainer team.
 
 ## Initial authors
 
-Fabian Reinartz @fabxc and Bartłomiej Płotka @bwplotka
+Fabian Reinartz @fabxc and Bartłomiej Płotka @bwplotka
+