[coordinator] Add mapping rule config support for keeping metrics at different resolutions

.buildkite/pipeline.yml

@@ -47,7 +47,7 @@ steps:
     parallelism: 2
     env:
       CGO_ENABLED: 0
-      GIMME_GO_VERSION: 1.12.x
+      GIMME_GO_VERSION: 1.13.x
     plugins:
         gopath-checkout#v1.0.1:
           import: github.com/m3db/m3
@@ -57,7 +57,7 @@ steps:
     parallelism: 1
     env:
       CGO_ENABLED: 0
-      GIMME_GO_VERSION: 1.12.x
+      GIMME_GO_VERSION: 1.13.x
     plugins:
         gopath-checkout#v1.0.1:
           import: github.com/m3db/m3

scripts/docker-integration-tests/common.sh

@@ -43,6 +43,59 @@ function retry_with_backoff {
   return $exitCode
 }
 
+function setup_single_m3db_node_long_namespaces {
+  local dbnode_host=${DBNODE_HOST:-dbnode01}
+  local dbnode_port=${DBNODE_PORT:-9000}
+  local dbnode_health_port=${DBNODE_HEALTH_PORT:-9002}
+  local dbnode_id=${DBNODE_ID:-m3db_local}
+  local coordinator_port=${COORDINATOR_PORT:-7201}
+  local zone=${ZONE:-embedded}
+
+  echo "Wait for API to be available"
+  ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
+    '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/namespace | jq ".namespaces | length")" == "0" ]'
+
+  echo "Adding placement and agg namespace"
+  curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/create -d '{
+    "type": "cluster",
+    "namespaceName": "agg",
+    "retentionTime": "24h",
+    "num_shards": 4,
+    "replicationFactor": 1,
+    "hosts": [
+      {
+          "id": "'${dbnode_id}'",
+          "isolation_group": "rack-a",
+          "zone": "'${zone}'",
+          "weight": 1024,
+          "address": "'"${dbnode_host}"'",
+          "port": '"${dbnode_port}"'
+      }
+    ]
+  }'
+
+  echo "Wait until placement is init'd"
+  ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
+    '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/placement | jq .placement.instances.'${dbnode_id}'.id)" == \"'${dbnode_id}'\" ]'
+
+  wait_for_namespaces
+
+  echo "Adding agg2d namespace"
+  curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/namespace/create -d '{
+    "namespaceName": "agg2d",
+    "retentionTime": "48h"
+  }'
+
+  echo "Wait until agg2d namespace is init'd"
+  ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
+    '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/namespace | jq .registry.namespaces.agg2d.indexOptions.enabled)" == true ]'
+
+
+  echo "Wait until bootstrapped"
+  ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
+    '[ "$(curl -sSf 0.0.0.0:'"${dbnode_health_port}"'/health | jq .bootstrapped)" == true ]'
+}
+
 function setup_single_m3db_node {
   local dbnode_host=${DBNODE_HOST:-dbnode01}
   local dbnode_port=${DBNODE_PORT:-9000}

scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml

@@ -0,0 +1,28 @@
+version: "3.5"
+services:
+  dbnode01:
+    expose:
+      - "9000-9004"
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:9000-9004:9000-9004"
+      - "0.0.0.0:2379-2380:2379-2380"
+    networks:
+      - backend
+    image: "m3dbnode_integration:${REVISION}"
+  coordinator01:
+    expose:
+      - "7201"
+      - "7203"
+      - "7204"
+    ports:
+      - "0.0.0.0:7201:7201"
+      - "0.0.0.0:7203:7203"
+      - "0.0.0.0:7204:7204"
+    networks:
+      - backend
+    image: "m3coordinator_integration:${REVISION}"
+    volumes:
+      - "./:/etc/m3coordinator/"
+networks:
+  backend:

scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml

@@ -0,0 +1,83 @@
+listenAddress:
+  value: "0.0.0.0:7201"
+
+logging:
+  level: info
+
+metrics:
+  scope:
+    prefix: "coordinator"
+  prometheus:
+    handlerPath: /metrics
+    listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved
+  sanitization: prometheus
+  samplingRate: 1.0
+  extended: none
+
+clusters:
+  - namespaces:
+      - namespace: agg
+        type: aggregated
+        retention: 24h
+        resolution: 30s
+        downsample:
+          all: false
+      - namespace: agg2d
+        type: aggregated
+        retention: 48h
+        resolution: 1m
+        downsample:
+          all: false
+      - namespace: unagg
+        type: unaggregated
+        retention: 10m
+    client:
+      config:
+        service:
+          env: default_env
+          zone: embedded
+          service: m3db
+          cacheDir: /var/lib/m3kv
+          etcdClusters:
+            - zone: embedded
+              endpoints:
+                - dbnode01:2379
+      writeConsistencyLevel: majority
+      readConsistencyLevel: unstrict_majority
+
+downsample:
+  rules:
+    mappingRules:
+      - name: "mysql metrics"
+        filter: "app:mysql*"
+        aggregations: ["Last"]
+        storagePolicies:
+          - resolution: 1m
+            retention: 48h
+      - name: "nginx metrics"
+        filter: "app:nginx*"
+        aggregations: ["Last"]
+        storagePolicies:
+          - resolution: 30s
+            retention: 24h
+    rollupRules:
+      - name: "requests per second by status code"
+        filter: "__name__:http_requests app:* status_code:* endpoint:*"
+        transforms:
+          - transform:
+              type: "PerSecond"
+          - rollup:
+              metricName: "http_requests_by_status_code"
+              groupBy: ["app", "status_code", "endpoint"]
+              aggregations: ["Sum"]
+        storagePolicies:
+          - resolution: 5s
+            retention: 10h
+  # NB(r): Use high buffer past limits for test since we are using
+  # resolution of 5s which races against the timestamps being ingested.
+  bufferPastLimits:
+    - resolution: 0s
+      bufferPast: 90s
+
+tagOptions:
+  idScheme: quoted