Merge pull request open-telemetry#11 from damemi/main

Add GCP collector config, readme, samples

.licenserc.json

@@ -41,6 +41,7 @@
         "src/featureflagservice/assets/vendor/",
         "src/featureflagservice/priv/",
         "src/productcatalogservice/genproto/",
-        "internal/tools/"
+        "internal/tools/",
+        "gcp-config-values.yml"
     ]
-  }
+  }

CONTRIBUTING.md

@@ -4,7 +4,7 @@ Welcome to OpenTelemetry Demo Webstore repository Google Cloud integration!
 
 This is the fork for the Google Cloud integration. To contribute to the demo
 itself, see
-https://github.com/open-telemetry/opentelemetry-demo/blob/main/CONTRIBUTING.md.
+[the OpenTelemetry `CONTRIBUTING.md`](https://github.com/open-telemetry/opentelemetry-demo/blob/main/CONTRIBUTING.md).
 
 If you would like to contribute to the Google Cloud integration, see
 instructions below.
@@ -184,3 +184,4 @@ on each other), the owner should try to get people aligned by:
   the owner should bring it to the OpenTelemetry Community Demo SIG
   [meeting](README.md#contributing).
 
+[docs]: https://opentelemetry.io/docs/demo/

README.md

@@ -9,11 +9,13 @@
 
 ## Welcome to the OpenTelemetry Astronomy Shop Demo with Google Cloud
 
-This repository containes a fork of https://github.com/open-telemetry/opentelemetry-demo,
+This repository contains a fork of [open-telemetry/opentelemetry-demo](https://github.com/open-telemetry/opentelemetry-demo),
 which integrates with Google Cloud Monitoring, Logging, and Trace.
 
 **This is not an officially supported Google product.**
 
+For details on how to use this with GCP, see [`README_GCP.md`](README_GCP.md).
+
 This repository contains the OpenTelemetry Astronomy Shop, a microservice-based
 distributed system intended to illustrate the implementation of OpenTelemetry in
 a near real-world environment.

README_GCP.md

@@ -0,0 +1,87 @@
+# Running the demo on Google Cloud
+
+The demo can send logs, traces, and metrics to Google Cloud. The easiest way to
+do this is with the [`gcp-config-values.yml`](gcp-config-values.yml) file (when
+deploying via Helm on GKE) or
+[`src/otelcollector/otelcol-config-extras.yml`](src/otelcollector/otelcol-config-extras.yml)
+when running with `docker-compose` on GCE.
+
+## Running on GKE
+
+The recommended way to run the demo on GKE is with the official Helm chart. If
+running on a GKE Autopilot cluster (or any cluster with Workload Identity), you
+must follow the prerequisite steps to set up a Workload Identity-enabled service
+account below. Otherwise, you can skip to the next section.
+
+### Workload Identity prequisites
+
+Follow the [Workload Identity
+docs](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity#authenticating_to)
+to set up an IAM service account in your GCP project with permission to use
+Workload Identity and write logs, traces, and metrics:
+
+```console
+export GCLOUD_PROJECT=<your project id>
+```
+
+```console
+gcloud iam service-accounts create opentelemetry-demo \
+    --project=${GCLOUD_PROJECT}
+gcloud projects add-iam-policy-binding ${GCLOUD_PROJECT} \
+    --member "serviceAccount:opentelemetry-demo@${GCLOUD_PROJECT}.iam.gserviceaccount.com" \
+    --role "roles/logging.logWriter"
+gcloud projects add-iam-policy-binding ${GCLOUD_PROJECT} \
+    --member "serviceAccount:opentelemetry-demo@${GCLOUD_PROJECT}.iam.gserviceaccount.com" \
+    --role "roles/monitoring.metricWriter"
+gcloud projects add-iam-policy-binding ${GCLOUD_PROJECT} \
+    --member "serviceAccount:opentelemetry-demo@${GCLOUD_PROJECT}.iam.gserviceaccount.com" \
+    --role "roles/cloudtrace.agent"
+gcloud iam service-accounts add-iam-policy-binding opentelemetry-demo@${GCLOUD_PROJECT}.iam.gserviceaccount.com \
+    --role roles/iam.workloadIdentityUser \
+    --member "serviceAccount:${GCLOUD_PROJECT}.svc.id.goog[default/opentelemetry-demo]"
+```
+
+Update [`gcp-config-valus.yml`](gcp-config-values.yml) to annotate the
+Kubernetes service account with your project:
+
+```console
+sed -i "s/%GCLOUD_PROJECT%/${GCLOUD_PROJECT}/g" gcp-config-values.yml
+```
+
+Next, when you deploy the Helm chart, the
+[`gcp-config-values.yml`](gcp-config-values.yml) file will create the Kubernetes
+service account and annotate it to use Workload Identity.
+
+### Deploying the Helm chart
+
+Follow the [OpenTelemetry docs to run the Helm chart](https://opentelemetry.io/docs/demo/kubernetes-deployment):
+
+```console
+helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
+helm repo update
+helm install my-otel-demo open-telemetry/opentelemetry-demo --values gcp-config-values.yml
+```
+
+## Running on GCE
+
+Follow the [OpenTelemetry docs to run with Docker](https://opentelemetry.io/docs/demo/docker-deployment/):
+
+```console
+make start
+```
+
+## Seeing telemetry
+
+With the demo running, you should see telemetry automatically created by the
+demo's load generator. You can see metrics under "Prometheus Target" in Cloud
+Monitoring:
+
+![metrics](gcp_metrics.png)
+
+Traces in the Trace explorer:
+
+![traces](gcp_traces.png)
+
+And logs in the Logs explorer organized by service:
+
+![logs](gcp_logs.png)

gcp-config-values.yml

@@ -0,0 +1,83 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# this file is used by Helm chart launch steps.
+# when merging YAML with the default values, objects are merged but lists are overwritten.
+# so, where a list is used, we have copied the defaults from the upstream values to preserve them.
+
+opentelemetry-collector:
+  # Service account for workload identity
+  serviceAccount:
+    create: true
+    annotations: {"iam.gke.io/gcp-service-account":"opentelemetry-demo@%GCLOUD_PROJECT%.iam.gserviceaccount.com"}
+    name: "opentelemetry-demo"
+
+  config:
+    processors:
+      memory_limiter:
+        check_interval: 1s
+        limit_percentage: 65
+        spike_limit_percentage: 20
+
+      batch:
+        send_batch_max_size: 200
+        send_batch_size: 200
+        timeout: 5s
+
+      resourcedetection:
+        detectors: [gcp]
+        timeout: 10s
+
+      transform/collision:
+        metric_statements:
+        - context: datapoint
+          statements:
+          - set(attributes["exported_location"], attributes["location"])
+          - delete_key(attributes, "location")
+          - set(attributes["exported_cluster"], attributes["cluster"])
+          - delete_key(attributes, "cluster")
+          - set(attributes["exported_namespace"], attributes["namespace"])
+          - delete_key(attributes, "namespace")
+          - set(attributes["exported_job"], attributes["job"])
+          - delete_key(attributes, "job")
+          - set(attributes["exported_instance"], attributes["instance"])
+          - delete_key(attributes, "instance")
+          - set(attributes["exported_project_id"], attributes["project_id"])
+          - delete_key(attributes, "project_id")
+
+      # See https://github.com/open-telemetry/opentelemetry-demo/issues/1330
+      filter/currency:
+        metrics:
+          metric:
+          - 'IsMatch(name, "(.*)app_currency(.*)")'
+
+    exporters:
+      googlecloud:
+        log:
+          default_log_name: opentelemetry.io/opentelemetry-demo
+
+      googlemanagedprometheus:
+
+    service:
+      pipelines:
+        logs:
+          processors: [memory_limiter, resourcedetection, resource, batch]
+          exporters: [googlecloud]
+        traces:
+          processors: [memory_limiter, resourcedetection, resource, batch]
+          exporters: [googlecloud] #spanmetrics disabled
+        metrics:
+          receivers: [otlp] #spanmetrics disabled
+          processors: [filter/ottl, filter/currency, memory_limiter, resourcedetection, transform, transform/collision, resource, batch]
+          exporters: [googlemanagedprometheus]

src/otelcollector/otelcol-config-extras.yml

@@ -4,6 +4,9 @@
 # extra settings to be merged into OpenTelemetry Collector configuration
 # do not delete this file
 
+# this file is used by docker-compose launch steps. it is merged with the default Collector
+# config in otelcol-config.yaml.
+
 ## Example configuration for sending data to your own OTLP HTTP backend
 ## Note: the spanmetrics exporter must be included in the exporters array
 ## if overriding the traces pipeline.
@@ -16,3 +19,59 @@
 #    pipelines:
 #      traces:
 #        exporters: [spanmetrics, otlphttp/example]
+
+processors:
+  memory_limiter:
+    check_interval: 1s
+    limit_percentage: 65
+    spike_limit_percentage: 20
+
+  batch:
+    send_batch_max_size: 200
+    send_batch_size: 200
+    timeout: 5s
+
+  resourcedetection:
+    detectors: [gcp]
+    timeout: 10s
+
+  transform/collision:
+    metric_statements:
+    - context: datapoint
+      statements:
+      - set(attributes["exported_location"], attributes["location"])
+      - delete_key(attributes, "location")
+      - set(attributes["exported_cluster"], attributes["cluster"])
+      - delete_key(attributes, "cluster")
+      - set(attributes["exported_namespace"], attributes["namespace"])
+      - delete_key(attributes, "namespace")
+      - set(attributes["exported_job"], attributes["job"])
+      - delete_key(attributes, "job")
+      - set(attributes["exported_instance"], attributes["instance"])
+      - delete_key(attributes, "instance")
+      - set(attributes["exported_project_id"], attributes["project_id"])
+      - delete_key(attributes, "project_id")
+
+  # See https://github.com/open-telemetry/opentelemetry-demo/issues/1330
+  filter/currency:
+    metrics:
+      metric:
+      - 'IsMatch(name, "(.*)app_currency(.*)")'
+
+exporters:
+  googlecloud:
+
+  googlemanagedprometheus:
+
+service:
+  pipelines:
+    logs:
+      processors: [memory_limiter, resourcedetection, resource, batch]
+      exporters: [googlecloud]
+    traces:
+      processors: [memory_limiter, resourcedetection, resource, batch]
+      exporters: [googlecloud] #spanmetrics disabled
+    metrics:
+      receivers: [otlp] #spanmetrics disabled
+      processors: [filter/ottl, filter/currency, memory_limiter, resourcedetection, transform, transform/collision, resource, batch]
+      exporters: [googlemanagedprometheus]