From acf1cdc62c8523797d524346421e210b533c12f3 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Wed, 26 Apr 2023 11:49:50 -0700 Subject: [PATCH 001/142] Initial setup of Observability OTEL demo using Opensearch as store --- .github/README.md | 26 ++++++++++++++++++ README.md | 14 +++++----- docker-compose.yml | 66 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 .github/README.md diff --git a/.github/README.md b/.github/README.md new file mode 100644 index 0000000000..377aec4cd8 --- /dev/null +++ b/.github/README.md @@ -0,0 +1,26 @@ +# OTel logo + OTel logo OpenTelemetry Demo with OpenSearch + +The following guide describes how to setup the OpenTelemetry demo with OpenSearch Observability using [Docker compose](#docker-compose) or [Kubernetes](#kubernetes). + +## Docker compose + +OpenSearch has [documented](https://opensearch.org/docs/latest/observing-your-data/trace/trace-analytics-jaeger/#setting-up-opensearch-to-use-jaeger-data) the usage of the Observability plugin with jaeger as a trace signal source. +The next instructions are similar and use the same docker compose file. +1. Start the demo with the following command from the repository's root directory: + ``` + docker-compose up -d + ``` + +## Explore and analyze the data With OpenSearch Observability + +### Service map +![Service map](https://docs.aws.amazon.com/images/opensearch-service/latest/developerguide/images/ta-dashboards-services.png) + +### Traces +![Traces](https://opensearch.org/docs/2.6/images/ta-trace.png) + +### Correlation +![Correlation](https://opensearch.org/docs/latest/images/observability-trace.png) + +### Logs +![Logs](https://opensearch.org/docs/latest/images/trace_log_correlation.gif) \ No newline at end of file diff --git a/README.md b/README.md index 77f509cef2..1558935de1 100644 --- a/README.md +++ b/README.md @@ -54,14 +54,14 @@ adding a link below. The community is committed to maintaining the project and keeping it up to date for you. | | | | -| ----------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | -| [AlibabaCloud LogService](https://github.com/aliyun-sls/opentelemetry-demo) | [Grafana Labs](https://github.com/grafana/opentelemetry-demo) | [Sentry](https://github.com/getsentry/opentelemetry-demo) | -| [AppDynamics](https://www.appdynamics.com/blog/cloud/how-to-observe-opentelemetry-demo-app-in-appdynamics-cloud/) | [Helios](https://otelsandbox.gethelios.dev) | [Splunk](https://github.com/signalfx/opentelemetry-demo) | -| [Aspecto](https://github.com/aspecto-io/opentelemetry-demo) | [Honeycomb.io](https://github.com/honeycombio/opentelemetry-demo) | [Sumo Logic](https://www.sumologic.com/blog/common-opentelemetry-demo-application/) | -| [Coralogix](https://coralogix.com/blog/configure-otel-demo-send-telemetry-data-coralogix) | [Instana](https://github.com/instana/opentelemetry-demo) | [TelemetryHub](https://github.com/TelemetryHub/opentelemetry-demo/tree/telemetryhub-backend) | -| [Datadog](https://github.com/DataDog/opentelemetry-demo) | [Kloudfuse](https://github.com/kloudfuse/opentelemetry-demo) | [Teletrace](https://github.com/teletrace/opentelemetry-demo) | +| ----------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- |----------------------------------------------------------------------------------------------| +| [AlibabaCloud LogService](https://github.com/aliyun-sls/opentelemetry-demo) | [Grafana Labs](https://github.com/grafana/opentelemetry-demo) | [Sentry](https://github.com/getsentry/opentelemetry-demo) | +| [AppDynamics](https://www.appdynamics.com/blog/cloud/how-to-observe-opentelemetry-demo-app-in-appdynamics-cloud/) | [Helios](https://otelsandbox.gethelios.dev) | [Splunk](https://github.com/signalfx/opentelemetry-demo) | +| [Aspecto](https://github.com/aspecto-io/opentelemetry-demo) | [Honeycomb.io](https://github.com/honeycombio/opentelemetry-demo) | [Sumo Logic](https://www.sumologic.com/blog/common-opentelemetry-demo-application/) | +| [Coralogix](https://coralogix.com/blog/configure-otel-demo-send-telemetry-data-coralogix) | [Instana](https://github.com/instana/opentelemetry-demo) | [TelemetryHub](https://github.com/TelemetryHub/opentelemetry-demo/tree/telemetryhub-backend) | +| [Datadog](https://github.com/DataDog/opentelemetry-demo) | [Kloudfuse](https://github.com/kloudfuse/opentelemetry-demo) | [Teletrace](https://github.com/teletrace/opentelemetry-demo) | | [Dynatrace](https://www.dynatrace.com/news/blog/opentelemetry-demo-application-with-dynatrace/) | [Lightstep](https://github.com/lightstep/opentelemetry-demo) | [Uptrace](https://github.com/uptrace/uptrace/tree/master/example/opentelemetry-demo) | -| [Elastic](https://github.com/elastic/opentelemetry-demo) | [New Relic](https://github.com/newrelic/opentelemetry-demo) | | +| [Elastic](https://github.com/elastic/opentelemetry-demo) | [New Relic](https://github.com/newrelic/opentelemetry-demo) | [OpenSearch](https://github.com/opensearch-project/opentelemetry-demo) | | | | | ## Contributing diff --git a/docker-compose.yml b/docker-compose.yml index e21afb41f1..112ff2bbc9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,10 @@ x-default-logging: &logging max-size: "5m" max-file: "2" +volumes: + opensearch-data1: + opensearch-data2: + networks: default: name: opentelemetry-demo @@ -659,6 +663,68 @@ services: - "${PROMETHEUS_SERVICE_PORT}:${PROMETHEUS_SERVICE_PORT}" logging: *logging + # OpenSearch store - node1 + opensearch-node1: # This is also the hostname of the container within the Docker network (i.e. https://opensearch-node1/) + image: opensearchproject/opensearch:latest # Specifying the latest available image - modify if you want a specific version + container_name: opensearch-node1 + environment: + - cluster.name=opensearch-cluster # Name the cluster + - node.name=opensearch-node1 # Name the node that will run in this container + - discovery.seed_hosts=opensearch-node1,opensearch-node2 # Nodes to look for when discovering the cluster + - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 # Nodes eligible to serve as cluster manager + - bootstrap.memory_lock=true # Disable JVM heap memory swapping + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # Set min and max JVM heap sizes to at least 50% of system RAM + ulimits: + memlock: + soft: -1 # Set memlock to unlimited (no soft or hard limit) + hard: -1 + nofile: + soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536 + hard: 65536 + volumes: + - opensearch-data1:/usr/share/opensearch/data # Creates volume called opensearch-data1 and mounts it to the container + ports: + - "9200:9200" + - "9600:9600" + networks: + - opentelemetry-demo # All of the containers will join the same Docker bridge network + + # OpenSearch store - node2 + opensearch-node2: + image: opensearchproject/opensearch:latest # This should be the same image used for opensearch-node1 to avoid issues + container_name: opensearch-node2 + environment: + - cluster.name=opensearch-cluster + - node.name=opensearch-node2 + - discovery.seed_hosts=opensearch-node1,opensearch-node2 + - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - opensearch-data2:/usr/share/opensearch/data + networks: + - opentelemetry-demo + + # OpenSearch store - dashboard + opensearch-dashboards: + image: opensearchproject/opensearch-dashboards:latest # Make sure the version of opensearch-dashboards matches the version of opensearch installed on other nodes + container_name: opensearch-dashboards + ports: + - 5601:5601 # Map host port 5601 to container port 5601 + expose: + - "5601" # Expose port 5601 for web access to OpenSearch Dashboards + environment: + OPENSEARCH_HOSTS: '["https://opensearch-node1:9200","https://opensearch-node2:9200"]' # Define the OpenSearch nodes that OpenSearch Dashboards will query + networks: + - opentelemetry-demo + # ***** # Tests # ***** From c8421502a85a95ab0eabe8f665c4ff3d9c52179f Mon Sep 17 00:00:00 2001 From: YANGDB Date: Wed, 26 Apr 2023 11:49:50 -0700 Subject: [PATCH 002/142] Initial setup of Observability OTEL demo using Opensearch as store Signed-off-by: YANGDB --- .github/README.md | 26 ++++++++++++++++++++ README.md | 14 +++++------ docker-compose.yml | 60 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 7 deletions(-) create mode 100644 .github/README.md diff --git a/.github/README.md b/.github/README.md new file mode 100644 index 0000000000..377aec4cd8 --- /dev/null +++ b/.github/README.md @@ -0,0 +1,26 @@ +# OTel logo + OTel logo OpenTelemetry Demo with OpenSearch + +The following guide describes how to setup the OpenTelemetry demo with OpenSearch Observability using [Docker compose](#docker-compose) or [Kubernetes](#kubernetes). + +## Docker compose + +OpenSearch has [documented](https://opensearch.org/docs/latest/observing-your-data/trace/trace-analytics-jaeger/#setting-up-opensearch-to-use-jaeger-data) the usage of the Observability plugin with jaeger as a trace signal source. +The next instructions are similar and use the same docker compose file. +1. Start the demo with the following command from the repository's root directory: + ``` + docker-compose up -d + ``` + +## Explore and analyze the data With OpenSearch Observability + +### Service map +![Service map](https://docs.aws.amazon.com/images/opensearch-service/latest/developerguide/images/ta-dashboards-services.png) + +### Traces +![Traces](https://opensearch.org/docs/2.6/images/ta-trace.png) + +### Correlation +![Correlation](https://opensearch.org/docs/latest/images/observability-trace.png) + +### Logs +![Logs](https://opensearch.org/docs/latest/images/trace_log_correlation.gif) \ No newline at end of file diff --git a/README.md b/README.md index 77f509cef2..1558935de1 100644 --- a/README.md +++ b/README.md @@ -54,14 +54,14 @@ adding a link below. The community is committed to maintaining the project and keeping it up to date for you. | | | | -| ----------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | -| [AlibabaCloud LogService](https://github.com/aliyun-sls/opentelemetry-demo) | [Grafana Labs](https://github.com/grafana/opentelemetry-demo) | [Sentry](https://github.com/getsentry/opentelemetry-demo) | -| [AppDynamics](https://www.appdynamics.com/blog/cloud/how-to-observe-opentelemetry-demo-app-in-appdynamics-cloud/) | [Helios](https://otelsandbox.gethelios.dev) | [Splunk](https://github.com/signalfx/opentelemetry-demo) | -| [Aspecto](https://github.com/aspecto-io/opentelemetry-demo) | [Honeycomb.io](https://github.com/honeycombio/opentelemetry-demo) | [Sumo Logic](https://www.sumologic.com/blog/common-opentelemetry-demo-application/) | -| [Coralogix](https://coralogix.com/blog/configure-otel-demo-send-telemetry-data-coralogix) | [Instana](https://github.com/instana/opentelemetry-demo) | [TelemetryHub](https://github.com/TelemetryHub/opentelemetry-demo/tree/telemetryhub-backend) | -| [Datadog](https://github.com/DataDog/opentelemetry-demo) | [Kloudfuse](https://github.com/kloudfuse/opentelemetry-demo) | [Teletrace](https://github.com/teletrace/opentelemetry-demo) | +| ----------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- |----------------------------------------------------------------------------------------------| +| [AlibabaCloud LogService](https://github.com/aliyun-sls/opentelemetry-demo) | [Grafana Labs](https://github.com/grafana/opentelemetry-demo) | [Sentry](https://github.com/getsentry/opentelemetry-demo) | +| [AppDynamics](https://www.appdynamics.com/blog/cloud/how-to-observe-opentelemetry-demo-app-in-appdynamics-cloud/) | [Helios](https://otelsandbox.gethelios.dev) | [Splunk](https://github.com/signalfx/opentelemetry-demo) | +| [Aspecto](https://github.com/aspecto-io/opentelemetry-demo) | [Honeycomb.io](https://github.com/honeycombio/opentelemetry-demo) | [Sumo Logic](https://www.sumologic.com/blog/common-opentelemetry-demo-application/) | +| [Coralogix](https://coralogix.com/blog/configure-otel-demo-send-telemetry-data-coralogix) | [Instana](https://github.com/instana/opentelemetry-demo) | [TelemetryHub](https://github.com/TelemetryHub/opentelemetry-demo/tree/telemetryhub-backend) | +| [Datadog](https://github.com/DataDog/opentelemetry-demo) | [Kloudfuse](https://github.com/kloudfuse/opentelemetry-demo) | [Teletrace](https://github.com/teletrace/opentelemetry-demo) | | [Dynatrace](https://www.dynatrace.com/news/blog/opentelemetry-demo-application-with-dynatrace/) | [Lightstep](https://github.com/lightstep/opentelemetry-demo) | [Uptrace](https://github.com/uptrace/uptrace/tree/master/example/opentelemetry-demo) | -| [Elastic](https://github.com/elastic/opentelemetry-demo) | [New Relic](https://github.com/newrelic/opentelemetry-demo) | | +| [Elastic](https://github.com/elastic/opentelemetry-demo) | [New Relic](https://github.com/newrelic/opentelemetry-demo) | [OpenSearch](https://github.com/opensearch-project/opentelemetry-demo) | | | | | ## Contributing diff --git a/docker-compose.yml b/docker-compose.yml index e21afb41f1..4b463eb492 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,10 @@ x-default-logging: &logging max-size: "5m" max-file: "2" +volumes: + opensearch-data1: + opensearch-data2: + networks: default: name: opentelemetry-demo @@ -659,6 +663,62 @@ services: - "${PROMETHEUS_SERVICE_PORT}:${PROMETHEUS_SERVICE_PORT}" logging: *logging + # OpenSearch store - node1 + opensearch-node1: # This is also the hostname of the container within the Docker network (i.e. https://opensearch-node1/) + image: opensearchproject/opensearch:latest # Specifying the latest available image - modify if you want a specific version + container_name: opensearch-node1 + environment: + - cluster.name=opensearch-cluster # Name the cluster + - node.name=opensearch-node1 # Name the node that will run in this container + - discovery.seed_hosts=opensearch-node1,opensearch-node2 # Nodes to look for when discovering the cluster + - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 # Nodes eligible to serve as cluster manager + - bootstrap.memory_lock=true # Disable JVM heap memory swapping + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # Set min and max JVM heap sizes to at least 50% of system RAM + ulimits: + memlock: + soft: -1 # Set memlock to unlimited (no soft or hard limit) + hard: -1 + nofile: + soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536 + hard: 65536 + volumes: + - opensearch-data1:/usr/share/opensearch/data # Creates volume called opensearch-data1 and mounts it to the container + ports: + - "9200:9200" + - "9600:9600" + + # OpenSearch store - node2 + opensearch-node2: + image: opensearchproject/opensearch:latest # This should be the same image used for opensearch-node1 to avoid issues + container_name: opensearch-node2 + environment: + - cluster.name=opensearch-cluster + - node.name=opensearch-node2 + - discovery.seed_hosts=opensearch-node1,opensearch-node2 + - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - opensearch-data2:/usr/share/opensearch/data + + # OpenSearch store - dashboard + opensearch-dashboards: + image: opensearchproject/opensearch-dashboards:latest # Make sure the version of opensearch-dashboards matches the version of opensearch installed on other nodes + container_name: opensearch-dashboards + ports: + - 5601:5601 # Map host port 5601 to container port 5601 + expose: + - "5601" # Expose port 5601 for web access to OpenSearch Dashboards + environment: + OPENSEARCH_HOSTS: '["https://opensearch-node1:9200","https://opensearch-node2:9200"]' # Define the OpenSearch nodes that OpenSearch Dashboards will query + # ***** # Tests # ***** From 9850688e4e52a40b55ce8029cab3d795d12be81b Mon Sep 17 00:00:00 2001 From: YANGDB Date: Mon, 1 May 2023 16:59:20 -0700 Subject: [PATCH 003/142] Initial setup of Observability OTEL demo using Opensearch as store Signed-off-by: YANGDB --- docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 4b463eb492..08055bebe8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -398,6 +398,9 @@ services: logging: *logging # Product Catalog service + # if proxy.golang.org is experiencing i/o timeout use the next: + # - go env -w GOPROXY=direct + # - go env -w GOSUMDB=off productcatalogservice: image: ${IMAGE_NAME}:${IMAGE_VERSION}-productcatalogservice container_name: product-catalog-service From bf41276676899be18c350b7643e69a789ffd7d01 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Wed, 3 May 2023 09:57:07 -0700 Subject: [PATCH 004/142] add curl to the basic docker O/S Signed-off-by: YANGDB --- src/accountingservice/Dockerfile | 1 + src/checkoutservice/Dockerfile | 1 + src/currencyservice/Dockerfile | 1 + src/frontend/Dockerfile | 1 + src/paymentservice/Dockerfile | 2 +- src/productcatalogservice/Dockerfile | 1 + 6 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/accountingservice/Dockerfile b/src/accountingservice/Dockerfile index 8936ab07db..c29cfff920 100644 --- a/src/accountingservice/Dockerfile +++ b/src/accountingservice/Dockerfile @@ -5,6 +5,7 @@ FROM golang:1.19.2-alpine AS builder RUN apk add build-base protobuf-dev protoc RUN apk add --no-cache git +RUN apk add --no-cache curl WORKDIR /usr/src/app/ diff --git a/src/checkoutservice/Dockerfile b/src/checkoutservice/Dockerfile index a29aa8a81f..0a113cbc2b 100644 --- a/src/checkoutservice/Dockerfile +++ b/src/checkoutservice/Dockerfile @@ -4,6 +4,7 @@ FROM golang:1.19.2-alpine AS builder RUN apk add --no-cache git +RUN apk add --no-cache curl RUN apk add build-base protobuf-dev protoc WORKDIR /usr/src/app/ diff --git a/src/currencyservice/Dockerfile b/src/currencyservice/Dockerfile index cda395e18b..d54a6d05d5 100644 --- a/src/currencyservice/Dockerfile +++ b/src/currencyservice/Dockerfile @@ -17,6 +17,7 @@ FROM alpine as builder RUN apk update +RUN apk add --no-cache curl RUN apk add git cmake make g++ grpc-dev re2-dev protobuf-dev c-ares-dev ARG OPENTELEMETRY_CPP_VERSION=1.8.2 diff --git a/src/frontend/Dockerfile b/src/frontend/Dockerfile index ca4ea57c1d..0bbb337b40 100644 --- a/src/frontend/Dockerfile +++ b/src/frontend/Dockerfile @@ -23,6 +23,7 @@ RUN npm run build FROM node:18-alpine AS runner WORKDIR /app RUN apk add --no-cache protobuf-dev protoc +RUN apk add --no-cache curl ENV NODE_ENV=production diff --git a/src/paymentservice/Dockerfile b/src/paymentservice/Dockerfile index d77d1f24c3..fe3d1972ed 100644 --- a/src/paymentservice/Dockerfile +++ b/src/paymentservice/Dockerfile @@ -9,7 +9,7 @@ WORKDIR /usr/src/app/ COPY ./src/paymentservice/package*.json ./ RUN apk add --no-cache python3 make g++ && npm ci --omit=dev - +RUN apk add --no-cache curl # ----------------------------------------------------------------------------- FROM node:16-alpine diff --git a/src/productcatalogservice/Dockerfile b/src/productcatalogservice/Dockerfile index 435886d06e..1573b56ec4 100644 --- a/src/productcatalogservice/Dockerfile +++ b/src/productcatalogservice/Dockerfile @@ -7,6 +7,7 @@ WORKDIR /usr/src/app/ RUN apk add --no-cache git RUN apk add build-base protobuf-dev protoc +RUN apk add --no-cache curl # Restore dependencies COPY ./src/productcatalogservice/ ./ From f0600becfb0f619a1999ce55abe410be7b1ff389 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Thu, 4 May 2023 13:53:45 -0700 Subject: [PATCH 005/142] add monitoring & opinionated assets for this demo Signed-off-by: YANGDB --- .env | 15 +++++++ docker-compose.yml | 55 ++++++++++++++++++----- src/integrations/Dockerfile | 8 ++++ src/integrations/README.md | 34 ++++++++++++++ src/integrations/install-observability.sh | 38 ++++++++++++++++ 5 files changed, 139 insertions(+), 11 deletions(-) create mode 100644 src/integrations/Dockerfile create mode 100644 src/integrations/README.md create mode 100644 src/integrations/install-observability.sh diff --git a/.env b/.env index 0b7b7614a3..b518a4b175 100644 --- a/.env +++ b/.env @@ -111,3 +111,18 @@ JAEGER_SERVICE_HOST=jaeger PROMETHEUS_SERVICE_PORT=9090 PROMETHEUS_SERVICE_HOST=prometheus PROMETHEUS_ADDR=${PROMETHEUS_SERVICE_HOST}:${PROMETHEUS_SERVICE_PORT} + +# OpenSearch Node1 +OPENSEARCH1_PORT=9200 +OPENSEARCH1_HOST=opensearch-node1 +OPENSEARCH1_ADDR=${OPENSEARCH1_HOST}:${OPENSEARCH1_PORT} + +# OpenSearch Node2 +OPENSEARCH2_PORT=9200 +OPENSEARCH2_HOST=opensearch-node2 +OPENSEARCH2_ADDR=${OPENSEARCH2_HOST}:${OPENSEARCH2_PORT} + +# OpenSearch Dashboard +OPENSEARCH_DASHBOARD_PORT=5601 +OPENSEARCH_DASHBOARD_HOST=opensearch-dashboards +OPENSEARCH_DASHBOARD_ADDR=${OPENSEARCH_DASHBOARD_HOST}:${OPENSEARCH_DASHBOARD_PORT} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 376d660bc8..c39d3cdcc7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -583,16 +583,13 @@ services: # Telemetry Components # ******************** # Jaeger - jaeger: - image: jaegertracing/all-in-one - container_name: jaeger + jaeger-collector: + image: jaegertracing/jaeger-collector:latest + container_name: jaeger-collector command: - - "--memory.max-traces" - - "10000" - - "--query.base-path" - - "/jaeger/ui" - - "--prometheus.server-url" - - "http://${PROMETHEUS_ADDR}" + - "--metrics-backend=prometheus" + - "--es.server-urls=https://opensearch-node1:9200" + - "--es.tls.enabled=true" deploy: resources: limits: @@ -600,12 +597,39 @@ services: restart: unless-stopped ports: - "${JAEGER_SERVICE_PORT}" # Jaeger UI - - "4317" # OTLP gRPC default port + - "4317" # OTLP gRPC default port + - "14269:14269" + - "14268:14268" + - "14267:14267" + - "14250:14250" + - "9411:9411" environment: - COLLECTOR_OTLP_ENABLED=true - METRICS_STORAGE_TYPE=prometheus + - SPAN_STORAGE_TYPE=opensearch + - ES_TAGS_AS_FIELDS_ALL=true + - ES_USERNAME=admin + - ES_PASSWORD=admin + - ES_TLS_SKIP_HOST_VERIFY=true + depends_on: + - opensearch-node1 logging: *logging + jaeger-agent: + image: jaegertracing/jaeger-agent:latest + hostname: jaeger-agent + command: ["--reporter.grpc.host-port=jaeger-collector:14250"] + ports: + - "5775:5775/udp" + - "6831:6831/udp" + - "6832:6832/udp" + - "5778:5778" + restart: on-failure + environment: + - SPAN_STORAGE_TYPE=opensearch + depends_on: + - jaeger-collector + # Grafana grafana: image: grafana/grafana:9.4.7 @@ -640,7 +664,7 @@ services: - "9464" # Prometheus exporter - "8888" # metrics endpoint depends_on: - - jaeger + - jaeger-agent logging: *logging # Prometheus @@ -721,6 +745,15 @@ services: - "5601" # Expose port 5601 for web access to OpenSearch Dashboards environment: OPENSEARCH_HOSTS: '["https://opensearch-node1:9200","https://opensearch-node2:9200"]' # Define the OpenSearch nodes that OpenSearch Dashboards will query + depends_on: + - opensearch-node1 + - opensearch-node2 + +# Observability OSD Integrations +# observability-integration: +# build: ./src/integrations/Dockerfile +# depends_on: +# - opensearch-dashboards # ***** # Tests diff --git a/src/integrations/Dockerfile b/src/integrations/Dockerfile new file mode 100644 index 0000000000..eb62b8a7ff --- /dev/null +++ b/src/integrations/Dockerfile @@ -0,0 +1,8 @@ +FROM alpine:latest + +RUN apk update && apk add curl netcat-openbsd + +COPY install-observability.sh /install-observability.sh +RUN chmod +x /install-observability.sh + +CMD /install-observability.sh diff --git a/src/integrations/README.md b/src/integrations/README.md new file mode 100644 index 0000000000..acecf6db29 --- /dev/null +++ b/src/integrations/README.md @@ -0,0 +1,34 @@ +# OpenSearch Integrations + +This service runs after the OpenSearch cluster and dashboard services have started. +Its purpose is to setup the necessary assets for the Observability plugin to be able to function as an Observability store and visualizations. + +For additional information about how OpenSearch connects to OTEL exporter see [here](https://opensearch.org/docs/latest/observing-your-data/trace/trace-analytics-jaeger/#setting-up-opensearch-to-use-jaeger-data) + +## Observability supported features + - Traces analytics + - Metrics analytics + - Logs analytics + - Monitoring Metrics KPI + - Loading relevant Integrations (WEB related) for specific pre-canned dashboards + + +## Assets that are installed + +### Index & mappings + - create the Logs / Traces / Metrics index template + - create the Logs / Traces / Metrics data-stream based on these index template + +### Datasource + - setup Prometheus datasource + +### Config + - setup alerts channels (Email/Slack) + +### Monitor + - setup alerts monitors + +### Dashboards + - setup dashboards for WEB server logs and metrics + + diff --git a/src/integrations/install-observability.sh b/src/integrations/install-observability.sh new file mode 100644 index 0000000000..dc18be4532 --- /dev/null +++ b/src/integrations/install-observability.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +host="$1" +port="$2" +shift +shift +cmd="$@" + +until nc -z -v -w30 "$host" "$port"; do + echo "Waiting for service at $host:$port..." + sleep 2 +done + +echo "Service is up and ready! Executing command:" +# create datasource for prometheus +echo "create datasource for prometheus" +exec $cmd + +# create index template for logs,traces,metrics +echo "create index template for logs,traces,metrics,serviceMaps" +exec $cmd + +# create data-streams for logs,traces,metrics +echo "create data-streams for logs,traces,metrics" +exec $cmd + +# create email channel for alerting +echo " create email channel for alerting" +exec $cmd + +# create saves searches for services KPI +echo " create saves searches for services KPI" +exec $cmd + +# create Alerts queries for monitoring services +echo " create Alerts queries for monitoring services +exec $cmd + From b0db9fc46f070646663e7d3bc9c55a06cada3e39 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Mon, 8 May 2023 10:55:13 -0700 Subject: [PATCH 006/142] export F/E & Load-Generator port services Signed-off-by: YANGDB --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c39d3cdcc7..dee908c647 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -277,7 +277,7 @@ services: memory: 200M restart: unless-stopped ports: - - "${FRONTEND_PORT}" + - "${FRONTEND_PORT}:${FRONTEND_PORT}" environment: - PORT=${FRONTEND_PORT} - FRONTEND_ADDR @@ -356,7 +356,7 @@ services: memory: 120M restart: unless-stopped ports: - - "${LOCUST_WEB_PORT}" + - "${LOCUST_WEB_PORT}:${LOCUST_WEB_PORT}" environment: - LOCUST_WEB_PORT - LOCUST_USERS @@ -642,7 +642,7 @@ services: - ./src/grafana/grafana.ini:/etc/grafana/grafana.ini - ./src/grafana/provisioning/:/etc/grafana/provisioning/ ports: - - "${GRAFANA_SERVICE_PORT}" + - "${GRAFANA_SERVICE_PORT}:${GRAFANA_SERVICE_PORT}" logging: *logging # OpenTelemetry Collector From ba709f19f9c257bafaac4663f188904ee194c38f Mon Sep 17 00:00:00 2001 From: YANGDB Date: Mon, 8 May 2023 17:14:58 -0700 Subject: [PATCH 007/142] fix jaeger service name Signed-off-by: YANGDB --- .env | 5 ----- docker-compose.yml | 9 +++++---- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.env b/.env index b518a4b175..1faf61de80 100644 --- a/.env +++ b/.env @@ -117,11 +117,6 @@ OPENSEARCH1_PORT=9200 OPENSEARCH1_HOST=opensearch-node1 OPENSEARCH1_ADDR=${OPENSEARCH1_HOST}:${OPENSEARCH1_PORT} -# OpenSearch Node2 -OPENSEARCH2_PORT=9200 -OPENSEARCH2_HOST=opensearch-node2 -OPENSEARCH2_ADDR=${OPENSEARCH2_HOST}:${OPENSEARCH2_PORT} - # OpenSearch Dashboard OPENSEARCH_DASHBOARD_PORT=5601 OPENSEARCH_DASHBOARD_HOST=opensearch-dashboards diff --git a/docker-compose.yml b/docker-compose.yml index 2bdbcd6397..d28de05eda 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -584,9 +584,9 @@ services: # Telemetry Components # ******************** # Jaeger - jaeger-collector: + jaeger: image: jaegertracing/jaeger-collector:latest - container_name: jaeger-collector + container_name: jaeger command: - "--metrics-backend=prometheus" - "--es.server-urls=https://opensearch-node1:9200" @@ -619,8 +619,9 @@ services: jaeger-agent: image: jaegertracing/jaeger-agent:latest hostname: jaeger-agent - command: ["--reporter.grpc.host-port=jaeger-collector:14250"] + command: ["--reporter.grpc.host-port=jaeger:14250"] ports: + - "${GRAFANA_SERVICE_PORT}" - "5775:5775/udp" - "6831:6831/udp" - "6832:6832/udp" @@ -629,7 +630,7 @@ services: environment: - SPAN_STORAGE_TYPE=opensearch depends_on: - - jaeger-collector + - jaeger # Grafana grafana: From daedfab2db1bf6fd13bc5e1f7b1fa859319cd32b Mon Sep 17 00:00:00 2001 From: YANGDB Date: Tue, 9 May 2023 20:33:44 -0700 Subject: [PATCH 008/142] adding nginx as frontend proxy, fluent-bit.conf as log emitter Signed-off-by: YANGDB --- docker-compose.yml | 73 ++++++++++++------------ src/fluent-bit/fluent-bit.conf | 30 ++++++++++ src/fluent-bit/parsers.conf | 6 ++ src/nginx-otel/default.conf | 11 ++++ src/nginx-otel/opentelemetry_module.conf | 8 +++ 5 files changed, 93 insertions(+), 35 deletions(-) create mode 100644 src/fluent-bit/fluent-bit.conf create mode 100644 src/fluent-bit/parsers.conf create mode 100644 src/nginx-otel/default.conf create mode 100644 src/nginx-otel/opentelemetry_module.conf diff --git a/docker-compose.yml b/docker-compose.yml index d28de05eda..e828fedc97 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -308,40 +308,6 @@ services: - shippingservice logging: *logging - # Frontend Proxy (Envoy) - frontendproxy: - image: ${IMAGE_NAME}:${IMAGE_VERSION}-frontendproxy - container_name: frontend-proxy - build: - context: ./ - dockerfile: src/frontendproxy/Dockerfile - deploy: - resources: - limits: - memory: 50M - ports: - - "${ENVOY_PORT}:${ENVOY_PORT}" - - 10000:10000 - environment: - - FRONTEND_PORT - - FRONTEND_HOST - - FEATURE_FLAG_SERVICE_PORT - - FEATURE_FLAG_SERVICE_HOST - - LOCUST_WEB_HOST - - LOCUST_WEB_PORT - - GRAFANA_SERVICE_PORT - - GRAFANA_SERVICE_HOST - - JAEGER_SERVICE_PORT - - JAEGER_SERVICE_HOST - - OTEL_COLLECTOR_HOST - - OTEL_COLLECTOR_PORT - - ENVOY_PORT - depends_on: - - frontend - - featureflagservice - - loadgenerator - - grafana - # Load Generator loadgenerator: image: ${IMAGE_NAME}:${IMAGE_VERSION}-loadgenerator @@ -486,6 +452,37 @@ services: - featureflagservice logging: *logging + # Frontend Proxy service + nginx: + image: nginx:latest + volumes: + - ./src/nginx-otel/default.conf:/etc/nginx/conf.d/default.conf + ports: + - 90:90 + depends_on: + - frontend + - fluentbit + - otelcol + links: + - fluentbit + logging: + driver: "fluentd" + options: + fluentd-address: 127.0.0.1:24224 + tag: nginx.access + + # Fluent-bit logs shipper service + fluentbit: + container_name: fluentbit + image: fluent/fluent-bit:latest + volumes: + - ./src/fluent-bit:/fluent-bit/etc + ports: + - "24224:24224" + - "24224:24224/udp" + depends_on: + - opensearch-node1 + # Shipping service shippingservice: image: ${IMAGE_NAME}:${IMAGE_VERSION}-shippingservice @@ -614,6 +611,8 @@ services: - ES_TLS_SKIP_HOST_VERIFY=true depends_on: - opensearch-node1 + - opensearch-node2 + logging: *logging jaeger-agent: @@ -712,6 +711,11 @@ services: hard: 65536 volumes: - opensearch-data1:/usr/share/opensearch/data # Creates volume called opensearch-data1 and mounts it to the container + healthcheck: + test: ["CMD", "curl", "-f", "http://opensearch-node1:9200/_cluster/health?wait_for_status=yellow"] + interval: 5s + timeout: 25s + retries: 4 ports: - "9200:9200" - "9600:9600" @@ -749,7 +753,6 @@ services: OPENSEARCH_HOSTS: '["https://opensearch-node1:9200","https://opensearch-node2:9200"]' # Define the OpenSearch nodes that OpenSearch Dashboards will query depends_on: - opensearch-node1 - - opensearch-node2 # Observability OSD Integrations # observability-integration: diff --git a/src/fluent-bit/fluent-bit.conf b/src/fluent-bit/fluent-bit.conf new file mode 100644 index 0000000000..0ea4787a9f --- /dev/null +++ b/src/fluent-bit/fluent-bit.conf @@ -0,0 +1,30 @@ +[SERVICE] + Parsers_File parsers.conf + +[INPUT] + Name forward + Port 24224 + +[FILTER] + Name parser + Match nginx.access + Key_Name log + Parser nginx + +[FILTER] + Name lua + Match nginx.access + code function cb_filter(a,b,c)local d={}local e=os.date("!%Y-%m-%dT%H:%M:%S.000Z")d["observerTime"]=e;d["body"]=c.remote.." "..c.host.." "..c.user.." ["..os.date("%d/%b/%Y:%H:%M:%S %z").."] \""..c.method.." "..c.path.." HTTP/1.1\" "..c.code.." "..c.size.." \""..c.referer.."\" \""..c.agent.."\""d["trace_id"]="102981ABCD2901"d["span_id"]="abcdef1010"d["attributes"]={}d["attributes"]["data_stream"]={}d["attributes"]["data_stream"]["dataset"]="nginx.access"d["attributes"]["data_stream"]["namespace"]="production"d["attributes"]["data_stream"]["type"]="logs"d["event"]={}d["event"]["category"]={"web"}d["event"]["name"]="access"d["event"]["domain"]="nginx.access"d["event"]["kind"]="event"d["event"]["result"]="success"d["event"]["type"]={"access"}d["http"]={}d["http"]["request"]={}d["http"]["request"]["method"]=c.method;d["http"]["response"]={}d["http"]["response"]["bytes"]=tonumber(c.size)d["http"]["response"]["status_code"]=c.code;d["http"]["flavor"]="1.1"d["http"]["url"]=c.path;d["communication"]={}d["communication"]["source"]={}d["communication"]["source"]["address"]="127.0.0.1"d["communication"]["source"]["ip"]=c.remote;return 1,b,d end + call cb_filter + +[OUTPUT] + Name opensearch + Match nginx.access + Host opensearch + Port 9200 + Index sso_logs-nginx-prod + Suppress_Type_Name On + +[OUTPUT] + Name stdout + Match nginx.access \ No newline at end of file diff --git a/src/fluent-bit/parsers.conf b/src/fluent-bit/parsers.conf new file mode 100644 index 0000000000..d9015f210a --- /dev/null +++ b/src/fluent-bit/parsers.conf @@ -0,0 +1,6 @@ +[PARSER] + Name nginx + Format regex + Regex ^(?[^ ]*) (?[^ ]*) (?[^ ]*) \[(?