Merge remote-tracking branch 'origin/master' into HEAD

Signed-off-by: Giedrius Statkevičius <giedriuswork@gmail.com>

.circleci/config.yml

@@ -1,21 +1,26 @@
 # NOTE: Current plan gives 1500 build minutes per month.
-version: 2
-# https://circleci.com/blog/circleci-hacks-reuse-yaml-in-your-circleci-config-with-yaml/
-defaults: &defaults
-  docker:
-    # Built by Thanos make docker-ci
-    - image: &default-docker-image quay.io/thanos/thanos-ci:v1.2-go1.15-node
+version: 2.1
 
-jobs:
-  test:
+orbs:
+  go: circleci/go@1.5.0
+
+executors:
+  golang:
     docker:
-      - image: *default-docker-image
+      - image: cimg/go:1.15-node
+  golang-test:
+    docker:
+      - image: cimg/go:1.15-node
       - image: beaukode/docker-swift-onlyone-authv2-keystone
-    working_directory: /go/src/github.com/thanos-io/thanos
+
+jobs:
+  test:
+    executor: golang-test
     environment:
       GO111MODULE: 'on'
     steps:
       - checkout
+      - go/mod-download-cached
       - setup_remote_docker:
           version: 17.07.0-ce
       - run:
@@ -30,7 +35,6 @@ jobs:
       - run:
           name: "Run unit tests."
           environment:
-            GOBIN: "/go/bin"
             THANOS_TEST_OBJSTORE_SKIP: AZURE,COS,ALIYUNOSS
             # Variables for Swift testing.
             OS_AUTH_URL: http://127.0.0.1:5000/v2.0
@@ -65,12 +69,10 @@ jobs:
             - .build
 
   publish_master:
-    <<: *defaults
-    working_directory: /go/src/github.com/thanos-io/thanos
-    environment:
-      GOBIN: "/go/bin"
+    executor: golang
     steps:
       - checkout
+      - go/mod-download-cached
       - setup_remote_docker:
           version: 17.07.0-ce
       - attach_workspace:
@@ -85,12 +87,10 @@ jobs:
       - run: make docker-push
 
   publish_release:
-    <<: *defaults
-    working_directory: /go/src/github.com/thanos-io/thanos
-    environment:
-      GOBIN: "/go/bin"
+    executor: golang
     steps:
       - checkout
+      - go/mod-download-cached
       - setup_remote_docker:
           version: 17.07.0-ce
       - attach_workspace:

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,67 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '30 12 * * 1'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go', 'javascript' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.promu.yml

@@ -1,5 +1,5 @@
 go:
-  version: 1.15.0
+  version: 1.15
 repository:
   path: github.com/thanos-io/thanos
 build:

CHANGELOG.md

@@ -17,13 +17,38 @@ We use _breaking :warning:_ to mark changes that are not backward compatible (re
 - [#3469](https://github.com/thanos-io/thanos/pull/3469) StoreAPI: Added `hints` field to `LabelNamesRequest` and `LabelValuesRequest`. Hints in an opaque data structure that can be used to carry additional information from the store and its content is implementation specific.
 - [#3421](https://github.com/thanos-io/thanos/pull/3421) Tools: Added `thanos tools bucket rewrite` command allowing to delete series from given block.
 - [#3509](https://github.com/thanos-io/thanos/pull/3509) Store: Added touch series limit
+- [#3388](https://github.com/thanos-io/thanos/pull/3378) Tools: Bucket replicator now can specify block IDs to copy.
+
 ### Fixed
 
--
+- [#3527](https://github.com/thanos-io/thanos/pull/3527) Query Frontend: Fix query_range behavior when start/end times are the same
+- [#3560](https://github.com/thanos-io/thanos/pull/3560) query-frontend: Allow separate label cache
 
 ### Changed
 
 - [#3496](https://github.com/thanos-io/thanos/pull/3496) s3: Respect SignatureV2 flag for all credential providers.
+- [#2732](https://github.com/thanos-io/thanos/pull/2732) Swift: Switched to a new library [ncw/swift](https://github.com/ncw/swift) providing large objects support.
+   By default, segments will be uploaded to the same container directory `segments/` if the file is bigger than `1GB`.
+   To change the defaults see [the docs](./docs/storage.md#openstack-swift).
+
+
+
+## [v0.17.2](https://github.com/thanos-io/thanos/releases/tag/v0.17.2) - 2020.12.07
+
+### Fixed
+
+- [#3532](https://github.com/thanos-io/thanos/pull/3532) compact: do not cleanup blocks on boot. Reverts the behavior change introduced in [#3115](https://github.com/thanos-io/thanos/pull/3115) as in some very bad cases the boot of Thanos Compact took a very long time since there were a lot of blocks-to-be-cleaned.
+- [#3520](https://github.com/thanos-io/thanos/pull/3520) Fix index out of bound bug when comparing ZLabelSets.
+
+## [v0.17.1](https://github.com/thanos-io/thanos/releases/tag/v0.17.1) - 2020.11.24
+
+### Fixed
+
+- [#3480](https://github.com/thanos-io/thanos/pull/3480) Query-frontend: Fixed regression.
+
+### Changed
+
+- [#3498](https://github.com/thanos-io/thanos/pull/3498) Enabled debug.SetPanicOnFault(true) which allow us to recover on queries causing SEG FAULTs (e.g unmmaped memory access).
 
 ## [v0.17.0](https://github.com/thanos-io/thanos/releases/tag/v0.17.0) - 2020.11.18
 

Dockerfile

@@ -1,4 +1,5 @@
-ARG SHA="0c38f63cbe19e40123668a48c36466ef72b195e723cbfcbe01e9657a5f14cec6"
+# By default we pin to amd64 sha. Use make docker to automatically adjust for arm64 versions.
+ARG SHA="fca3819d670cdaee0d785499fda202ea01c0640ca0803d26ae6dbf2a1c8c041c"
 FROM quay.io/prometheus/busybox@sha256:${SHA}
 LABEL maintainer="The Thanos Authors"
 

Dockerfile.multi-stage

@@ -14,7 +14,8 @@ RUN git update-index --refresh; make build
 
 # -----------------------------------------------------------------------------
 
-ARG SHA="0c38f63cbe19e40123668a48c36466ef72b195e723cbfcbe01e9657a5f14cec6"
+# By default we pin to amd64 sha. Use make docker to automatically adjust for arm64 versions.
+ARG SHA="fca3819d670cdaee0d785499fda202ea01c0640ca0803d26ae6dbf2a1c8c041c"
 FROM quay.io/prometheus/busybox@sha256:${SHA}
 LABEL maintainer="The Thanos Authors"
 

MAINTAINERS.md

@@ -3,17 +3,18 @@
 | Name                  | Email                  | Slack                    | GitHub                                      | Company           |
 |-----------------------|------------------------|--------------------------|---------------------------------------------|-------------------|
 | Bartłomiej Płotka     | bwplotka@gmail.com     | `@bwplotka`              | [@bwplotka](https://github.com/bwplotka)    | Red Hat           |
-| Frederic Branczyk     | fbranczyk@gmail.com    | `@brancz`                | [@brancz](https://github.com/brancz)        | Red Hat           |
+| Frederic Branczyk     | fbranczyk@gmail.com    | `@brancz`                | [@brancz](https://github.com/brancz)        | Polar Signals     |
 | Giedrius Statkevičius | giedriuswork@gmail.com | `@Giedrius Statkevičius` | [@GiedriusS](https://github.com/GiedriusS)  | AdForm            |
 | Kemal Akkoyun         | kakkoyun@gmail.com     | `@kakkoyun`              | [@kakkoyun](https://github.com/kakkoyun)    | Red Hat           |
 | Lucas Servén Marín    | lserven@gmail.com      | `@squat`                 | [@squat](https://github.com/squat)          | Red Hat           |
 | Povilas Versockas     | p.versockas@gmail.com  | `@povilasv`              | [@povilasv](https://github.com/povilasv)    | Utility Warehouse |
 | Prem Saraswat         | prmsrswt@gmail.com     | `@Prem Saraswat`         | [@prmsrswt](https://github.com/prmsrswt)    | Atlan             |
 | Marco Pracucci        | marco@pracucci.com     | `@pracucci`              | [@pracucci](https://github.com/pracucci)    | Grafana Labs      |
 | Matthias Loibl        | mail@matthiasloibl.com | `@metalmatze`            | [@metalmatze](https://github.com/metalmatze)| Red Hat           |
+| Ben Ye                | yb532204897@gmail.com  | `@yeya24`                | [@yeya24](https://github.com/yeya24)        | Red Hat           |
 
 We are bunch of people from different companies with various interests and skills.
-We are from different parts of Europe: Germany, Italy, Lithuania, Poland, and UK and India.
+We are from different parts of the world: Germany, Italy, Lithuania, Poland, UK, India and China.
 We have something in common though: We all share the love for OpenSource, Go, Prometheus, :coffee: and Observability topics.
 
 As either Software Developers or SRE (or both!) we've chosen to maintain (mostly in our free time) Thanos, the de facto way to scale awesome [Prometheus](https://prometheus.io) project.
@@ -34,7 +35,6 @@ Full list of triage persons is displayed below:
 | Name                  | Slack                    | GitHub                                                     |
 |-----------------------|--------------------------|------------------------------------------------------------|
 | Adrien Fillon         | `@Adrien F`              | [@adrien-f](https://github.com/adrien-f)                   |
-| Ben Ye                | `@yeya24`                | [@yeya24](https://github.com/yeya24)                       |
 | Martin Chodur         | `@FUSAKLA`               | [@fusakla](https://github.com/fusakla)                     |
 | Michael Dai           | `@jojohappy`             | [@jojohappy](https://github.com/jojohappy)                 |
 | Xiang Dai             | `@daixiang0`             | [@daixiang0](https://github.com/daixiang0)                 |

Makefile

@@ -8,13 +8,13 @@ DOCKER_CI_TAG     ?= test
 SHA=''
 arch = $(shell uname -m)
 # Run `DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect quay.io/prometheus/busybox:latest` to get SHA
-# Update at 2020.07.06
+# Update at 2020.12.11
 ifeq ($(arch), x86_64)
     # amd64
-    SHA="248b7ec76e03e6b4fbb796fc3cdd2f91dad45546a6d7dee61c322475e0e8a08f"
+    SHA="fca3819d670cdaee0d785499fda202ea01c0640ca0803d26ae6dbf2a1c8c041c"
 else ifeq ($(arch), armv8)
     # arm64
-    SHA="69508e8fdc516eacbacc0379c03c971e3043706cc8211e6bddb35d903edc3628"
+    SHA="5478a46f1eb37ebe414c399766f8088bc8353345602053485dd429b9a87097e5"
 else
     echo >&2 "only support amd64 or arm64 arch" && exit 1
 endif
@@ -257,14 +257,6 @@ install-deps: ## Installs dependencies for integration tests. It installs suppor
 install-deps: $(ALERTMANAGER) $(MINIO) $(PROMETHEUS_ARRAY)
 	@echo ">>GOBIN=$(GOBIN)"
 
-.PHONY: docker-ci
-docker-ci: ## Builds and pushes docker image used by our CI. This is done to cache our tools and dependencies. To be run by Thanos maintainer.
-docker-ci: install-deps
-	@docker build -t thanos-ci -f Dockerfile.thanos-ci .
-	@echo ">> pushing thanos-ci image"
-	@docker tag "thanos-ci" "quay.io/thanos/thanos-ci:$(DOCKER_CI_TAG)"
-	@docker push "quay.io/thanos/thanos-ci:$(DOCKER_CI_TAG)"
-
 .PHONY: check-git
 check-git:
 ifneq ($(GIT),)

cmd/thanos/compact.go

@@ -113,6 +113,10 @@ func runCompact(
 		Name: "thanos_compact_iterations_total",
 		Help: "Total number of iterations that were executed successfully.",
 	})
+	cleanups := promauto.With(reg).NewCounter(prometheus.CounterOpts{
+		Name: "thanos_compact_block_cleanup_loops_total",
+		Help: "Total number of concurrent cleanup loops of partially uploaded blocks and marked blocks that were executed successfully.",
+	})
 	partialUploadDeleteAttempts := promauto.With(reg).NewCounter(prometheus.CounterOpts{
 		Name: "thanos_compact_aborted_partial_uploads_deletion_attempts_total",
 		Help: "Total number of started deletions of blocks that are assumed aborted and only partially uploaded.",
@@ -339,29 +343,20 @@ func runCompact(
 		cleanMtx.Lock()
 		defer cleanMtx.Unlock()
 
-		// No need to resync before partial uploads and delete marked blocks. Last sync should be valid.
+		if err := sy.SyncMetas(ctx); err != nil {
+			cancel()
+			return errors.Wrap(err, "syncing metas")
+		}
+
 		compact.BestEffortCleanAbortedPartialUploads(ctx, logger, sy.Partial(), bkt, partialUploadDeleteAttempts, blocksCleaned, blockCleanupFailures)
 		if err := blocksCleaner.DeleteMarkedBlocks(ctx); err != nil {
 			return errors.Wrap(err, "cleaning marked blocks")
 		}
+		cleanups.Inc()
 
-		if err := sy.SyncMetas(ctx); err != nil {
-			level.Error(logger).Log("msg", "failed to sync metas", "err", err)
-		}
 		return nil
 	}
 
-	// Do it once at the beginning to ensure that it runs at least once before
-	// the main loop.
-	if err := sy.SyncMetas(ctx); err != nil {
-		cancel()
-		return errors.Wrap(err, "syncing metas")
-	}
-	if err := cleanPartialMarked(); err != nil {
-		cancel()
-		return errors.Wrap(err, "cleaning partial and marked blocks")
-	}
-
 	compactMainFn := func() error {
 		if err := compactor.Compact(ctx); err != nil {
 			return errors.Wrap(err, "compaction")
@@ -481,11 +476,6 @@ func runCompact(
 		// since one iteration potentially could take a long time.
 		if conf.cleanupBlocksInterval > 0 {
 			g.Add(func() error {
-				// Wait the whole period at the beginning because we've executed this on boot.
-				select {
-				case <-time.After(conf.cleanupBlocksInterval):
-				case <-ctx.Done():
-				}
 				return runutil.Repeat(conf.cleanupBlocksInterval, ctx.Done(), cleanPartialMarked)
 			}, func(error) {
 				cancel()

cmd/thanos/main.go

@@ -11,6 +11,7 @@ import (
 	"os/signal"
 	"path/filepath"
 	"runtime"
+	"runtime/debug"
 	"syscall"
 
 	"github.com/go-kit/kit/log"
@@ -28,6 +29,10 @@ import (
 )
 
 func main() {
+	// We use mmaped resources in most of the components so hardcode PanicOnFault to true. This allows us to recover (if we can e.g if queries
+	// are temporarily accessing unmapped memory).
+	debug.SetPanicOnFault(true)
+
 	if os.Getenv("DEBUG") != "" {
 		runtime.SetMutexProfileFraction(10)
 		runtime.SetBlockProfileRate(10)

cmd/thanos/query.go

@@ -391,12 +391,12 @@ func runQuery(
 		ctx, cancel := context.WithCancel(context.Background())
 		g.Add(func() error {
 			return runutil.Repeat(dnsSDInterval, ctx.Done(), func() error {
-				ctx, cancel = context.WithTimeout(ctx, dnsSDInterval)
-				defer cancel()
-				if err := dnsStoreProvider.Resolve(ctx, append(fileSDCache.Addresses(), storeAddrs...)); err != nil {
+				resolveCtx, resolveCancel := context.WithTimeout(ctx, dnsSDInterval)
+				defer resolveCancel()
+				if err := dnsStoreProvider.Resolve(resolveCtx, append(fileSDCache.Addresses(), storeAddrs...)); err != nil {
 					level.Error(logger).Log("msg", "failed to resolve addresses for storeAPIs", "err", err)
 				}
-				if err := dnsRuleProvider.Resolve(ctx, ruleAddrs); err != nil {
+				if err := dnsRuleProvider.Resolve(resolveCtx, ruleAddrs); err != nil {
 					level.Error(logger).Log("msg", "failed to resolve addresses for rulesAPIs", "err", err)
 				}
 				return nil