From 7e3fc985618e49d868529c6aaa6e01a8ed63cd09 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Wed, 17 Jul 2024 11:07:31 -0700 Subject: [PATCH] Add internal/otelarrow package for shared code between otelarrow exporter, receiver (#33579) **Description:** The otelarrowexporter and otelarrowreceiver components depend on several libraries in the open-telemetry/otel-arrow repository, which would complicate release management in both repositories. This change eliminates the dependency on `otel-arrow/collector` by moving each of those packages into a subdirectory of `internal/otelarrow`. **Link to tracking Issue:** #33567 **Testing:** This brings new integration tests into this repository for the two otelarrow components. **Documentation:** Each of the three non-test packages exposed here has a brief README.md explaining why it exists. Each of these could be useful to non-Arrow components if helpful. --------- Co-authored-by: Alex Boten <223565+codeboten@users.noreply.github.com> Co-authored-by: Matthew Wear --- .chloggen/otelarrow-internals.yaml | 27 ++ .github/CODEOWNERS | 1 + .github/ISSUE_TEMPLATE/bug_report.yaml | 1 + .github/ISSUE_TEMPLATE/feature_request.yaml | 1 + .github/ISSUE_TEMPLATE/other.yaml | 1 + .github/ISSUE_TEMPLATE/unmaintained.yaml | 1 + internal/otelarrow/Makefile | 1 + internal/otelarrow/admission/README.md | 20 + internal/otelarrow/admission/boundedqueue.go | 152 +++++++ .../otelarrow/admission/boundedqueue_test.go | 189 +++++++++ internal/otelarrow/compression/README.md | 18 + internal/otelarrow/compression/zstd/mru.go | 106 +++++ .../otelarrow/compression/zstd/mru_test.go | 84 ++++ internal/otelarrow/compression/zstd/zstd.go | 324 +++++++++++++++ .../otelarrow/compression/zstd/zstd_test.go | 181 +++++++++ internal/otelarrow/go.mod | 103 +++++ internal/otelarrow/go.sum | 271 +++++++++++++ internal/otelarrow/metadata.yaml | 3 + internal/otelarrow/netstats/README.md | 37 ++ internal/otelarrow/netstats/grpc.go | 20 + internal/otelarrow/netstats/handler.go | 83 ++++ internal/otelarrow/netstats/netstats.go | 263 ++++++++++++ internal/otelarrow/netstats/netstats_test.go | 326 +++++++++++++++ internal/otelarrow/test/e2e_test.go | 378 ++++++++++++++++++ internal/otelarrow/testdata/common.go | 30 ++ internal/otelarrow/testdata/log.go | 59 +++ internal/otelarrow/testdata/metric.go | 293 ++++++++++++++ internal/otelarrow/testdata/resource.go | 10 + internal/otelarrow/testdata/trace.go | 71 ++++ internal/otelarrow/testutil/testutil.go | 117 ++++++ internal/otelarrow/testutil/testutil_test.go | 57 +++ versions.yaml | 1 + 32 files changed, 3229 insertions(+) create mode 100644 .chloggen/otelarrow-internals.yaml create mode 100644 internal/otelarrow/Makefile create mode 100644 internal/otelarrow/admission/README.md create mode 100644 internal/otelarrow/admission/boundedqueue.go create mode 100644 internal/otelarrow/admission/boundedqueue_test.go create mode 100644 internal/otelarrow/compression/README.md create mode 100644 internal/otelarrow/compression/zstd/mru.go create mode 100644 internal/otelarrow/compression/zstd/mru_test.go create mode 100644 internal/otelarrow/compression/zstd/zstd.go create mode 100644 internal/otelarrow/compression/zstd/zstd_test.go create mode 100644 internal/otelarrow/go.mod create mode 100644 internal/otelarrow/go.sum create mode 100644 internal/otelarrow/metadata.yaml create mode 100644 internal/otelarrow/netstats/README.md create mode 100644 internal/otelarrow/netstats/grpc.go create mode 100644 internal/otelarrow/netstats/handler.go create mode 100644 internal/otelarrow/netstats/netstats.go create mode 100644 internal/otelarrow/netstats/netstats_test.go create mode 100644 internal/otelarrow/test/e2e_test.go create mode 100644 internal/otelarrow/testdata/common.go create mode 100644 internal/otelarrow/testdata/log.go create mode 100644 internal/otelarrow/testdata/metric.go create mode 100644 internal/otelarrow/testdata/resource.go create mode 100644 internal/otelarrow/testdata/trace.go create mode 100644 internal/otelarrow/testutil/testutil.go create mode 100644 internal/otelarrow/testutil/testutil_test.go diff --git a/.chloggen/otelarrow-internals.yaml b/.chloggen/otelarrow-internals.yaml new file mode 100644 index 000000000000..2a69f78caacc --- /dev/null +++ b/.chloggen/otelarrow-internals.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: otelarrowreceiver, otelarrowexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: OTel-Arrow internal packages moved into this repository. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [33567] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: New integration testing between otelarrowexporter and otelarrowreceiver. + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 65bd3dd515b6..1e231d0c7584 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -132,6 +132,7 @@ internal/k8stest/ @open-teleme internal/kafka/ @open-telemetry/collector-contrib-approvers @pavolloffay @MovieStoreGuy internal/kubelet/ @open-telemetry/collector-contrib-approvers @dmitryax internal/metadataproviders/ @open-telemetry/collector-contrib-approvers @Aneurysm9 @dashpole +internal/otelarrow/ @open-telemetry/collector-contrib-approvers @jmacd @moh-osman3 internal/pdatautil/ @open-telemetry/collector-contrib-approvers @djaglowski internal/sharedcomponent/ @open-telemetry/collector-contrib-approvers @open-telemetry/collector-approvers internal/splunk/ @open-telemetry/collector-contrib-approvers @dmitryax diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index b0831457c945..2fb804a0d806 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -129,6 +129,7 @@ body: - internal/kafka - internal/kubelet - internal/metadataproviders + - internal/otelarrow - internal/pdatautil - internal/sharedcomponent - internal/splunk diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml index efa99c9f6857..d15af6e51d5f 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -123,6 +123,7 @@ body: - internal/kafka - internal/kubelet - internal/metadataproviders + - internal/otelarrow - internal/pdatautil - internal/sharedcomponent - internal/splunk diff --git a/.github/ISSUE_TEMPLATE/other.yaml b/.github/ISSUE_TEMPLATE/other.yaml index 67b65b7fb9d3..0a0833055cbb 100644 --- a/.github/ISSUE_TEMPLATE/other.yaml +++ b/.github/ISSUE_TEMPLATE/other.yaml @@ -123,6 +123,7 @@ body: - internal/kafka - internal/kubelet - internal/metadataproviders + - internal/otelarrow - internal/pdatautil - internal/sharedcomponent - internal/splunk diff --git a/.github/ISSUE_TEMPLATE/unmaintained.yaml b/.github/ISSUE_TEMPLATE/unmaintained.yaml index 54887cfc3b66..5b29c2b79c08 100644 --- a/.github/ISSUE_TEMPLATE/unmaintained.yaml +++ b/.github/ISSUE_TEMPLATE/unmaintained.yaml @@ -128,6 +128,7 @@ body: - internal/kafka - internal/kubelet - internal/metadataproviders + - internal/otelarrow - internal/pdatautil - internal/sharedcomponent - internal/splunk diff --git a/internal/otelarrow/Makefile b/internal/otelarrow/Makefile new file mode 100644 index 000000000000..ded7a36092dc --- /dev/null +++ b/internal/otelarrow/Makefile @@ -0,0 +1 @@ +include ../../Makefile.Common diff --git a/internal/otelarrow/admission/README.md b/internal/otelarrow/admission/README.md new file mode 100644 index 000000000000..053ad05a8fcf --- /dev/null +++ b/internal/otelarrow/admission/README.md @@ -0,0 +1,20 @@ +# Admission Package + +## Overview + +The admission package provides a BoundedQueue object which is a semaphore implementation that limits the number of bytes admitted into a collector pipeline. Additionally the BoundedQueue limits the number of waiters that can block on a call to `bq.Acquire(sz int64)`. + +This package is an experiment to improve the behavior of Collector pipelines having their `exporterhelper` configured to apply backpressure. This package is meant to be used in receivers, via an interceptor or custom logic. Therefore, the BoundedQueue helps limit memory within the entire collector pipeline by limiting two dimensions that cause memory issues: +1. bytes: large requests that enter the collector pipeline can require large allocations even if downstream components will eventually limit or ratelimit the request. +2. waiters: limiting on bytes alone is not enough because requests that enter the pipeline and block on `bq.Acquire()` can still consume memory within the receiver. If there are enough waiters this can be a significant contribution to memory usage. + +## Usage + +Create a new BoundedQueue by calling `bq := admission.NewBoundedQueue(maxLimitBytes, maxLimitWaiters)` + +Within the component call `bq.Acquire(ctx, requestSize)` which will either +1. succeed immediately if there is enough available memory +2. fail immediately if there are too many waiters +3. block until context cancelation or enough bytes becomes available + +Once a request has finished processing and is sent downstream call `bq.Release(requestSize)` to allow waiters to be admitted for processing. Release should only fail if releasing more bytes than previously acquired. \ No newline at end of file diff --git a/internal/otelarrow/admission/boundedqueue.go b/internal/otelarrow/admission/boundedqueue.go new file mode 100644 index 000000000000..388cd93285bb --- /dev/null +++ b/internal/otelarrow/admission/boundedqueue.go @@ -0,0 +1,152 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package admission // import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/otelarrow/admission" + +import ( + "context" + "fmt" + "sync" + + "github.com/google/uuid" + orderedmap "github.com/wk8/go-ordered-map/v2" +) + +var ErrTooManyWaiters = fmt.Errorf("rejecting request, too many waiters") + +type BoundedQueue struct { + maxLimitBytes int64 + maxLimitWaiters int64 + currentBytes int64 + currentWaiters int64 + lock sync.Mutex + waiters *orderedmap.OrderedMap[uuid.UUID, waiter] +} + +type waiter struct { + readyCh chan struct{} + pendingBytes int64 + ID uuid.UUID +} + +func NewBoundedQueue(maxLimitBytes, maxLimitWaiters int64) *BoundedQueue { + return &BoundedQueue{ + maxLimitBytes: maxLimitBytes, + maxLimitWaiters: maxLimitWaiters, + waiters: orderedmap.New[uuid.UUID, waiter](), + } +} + +func (bq *BoundedQueue) admit(pendingBytes int64) (bool, error) { + bq.lock.Lock() + defer bq.lock.Unlock() + + if pendingBytes > bq.maxLimitBytes { // will never succeed + return false, fmt.Errorf("rejecting request, request size larger than configured limit") + } + + if bq.currentBytes+pendingBytes <= bq.maxLimitBytes { // no need to wait to admit + bq.currentBytes += pendingBytes + return true, nil + } + + // since we were unable to admit, check if we can wait. + if bq.currentWaiters+1 > bq.maxLimitWaiters { // too many waiters + return false, ErrTooManyWaiters + } + + // if we got to this point we need to wait to acquire bytes, so update currentWaiters before releasing mutex. + bq.currentWaiters++ + return false, nil +} + +func (bq *BoundedQueue) Acquire(ctx context.Context, pendingBytes int64) error { + success, err := bq.admit(pendingBytes) + if err != nil || success { + return err + } + + // otherwise we need to wait for bytes to be released + curWaiter := waiter{ + pendingBytes: pendingBytes, + readyCh: make(chan struct{}), + } + + bq.lock.Lock() + + // generate unique key + for { + id := uuid.New() + _, keyExists := bq.waiters.Get(id) + if keyExists { + continue + } + bq.waiters.Set(id, curWaiter) + curWaiter.ID = id + break + } + + bq.lock.Unlock() + // @@@ instrument this code path + + select { + case <-curWaiter.readyCh: + return nil + case <-ctx.Done(): + // canceled before acquired so remove waiter. + bq.lock.Lock() + defer bq.lock.Unlock() + err = fmt.Errorf("context canceled: %w ", ctx.Err()) + + _, found := bq.waiters.Delete(curWaiter.ID) + if !found { + return err + } + + bq.currentWaiters-- + return err + } +} + +func (bq *BoundedQueue) Release(pendingBytes int64) error { + bq.lock.Lock() + defer bq.lock.Unlock() + + bq.currentBytes -= pendingBytes + + if bq.currentBytes < 0 { + return fmt.Errorf("released more bytes than acquired") + } + + for { + if bq.waiters.Len() == 0 { + return nil + } + next := bq.waiters.Oldest() + nextWaiter := next.Value + nextKey := next.Key + if bq.currentBytes+nextWaiter.pendingBytes <= bq.maxLimitBytes { + bq.currentBytes += nextWaiter.pendingBytes + bq.currentWaiters-- + close(nextWaiter.readyCh) + _, found := bq.waiters.Delete(nextKey) + if !found { + return fmt.Errorf("deleting waiter that doesn't exist") + } + continue + } + break + } + + return nil +} + +func (bq *BoundedQueue) TryAcquire(pendingBytes int64) bool { + bq.lock.Lock() + defer bq.lock.Unlock() + if bq.currentBytes+pendingBytes <= bq.maxLimitBytes { + bq.currentBytes += pendingBytes + return true + } + return false +} diff --git a/internal/otelarrow/admission/boundedqueue_test.go b/internal/otelarrow/admission/boundedqueue_test.go new file mode 100644 index 000000000000..a56ea86e7461 --- /dev/null +++ b/internal/otelarrow/admission/boundedqueue_test.go @@ -0,0 +1,189 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package admission + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/multierr" +) + +func min(x, y int64) int64 { + if x <= y { + return x + } + return y +} + +func max(x, y int64) int64 { + if x >= y { + return x + } + return y +} + +func abs(x int64) int64 { + if x < 0 { + return -x + } + return x +} +func TestAcquireSimpleNoWaiters(t *testing.T) { + maxLimitBytes := 1000 + maxLimitWaiters := 10 + numRequests := 40 + requestSize := 21 + + bq := NewBoundedQueue(int64(maxLimitBytes), int64(maxLimitWaiters)) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + for i := 0; i < numRequests; i++ { + go func() { + err := bq.Acquire(ctx, int64(requestSize)) + assert.NoError(t, err) + }() + } + + require.Never(t, func() bool { + return bq.waiters.Len() > 0 + }, 2*time.Second, 10*time.Millisecond) + + for i := 0; i < numRequests; i++ { + assert.NoError(t, bq.Release(int64(requestSize))) + assert.Equal(t, int64(0), bq.currentWaiters) + } + + assert.ErrorContains(t, bq.Release(int64(1)), "released more bytes than acquired") + assert.NoError(t, bq.Acquire(ctx, int64(maxLimitBytes))) +} + +func TestAcquireBoundedWithWaiters(t *testing.T) { + tests := []struct { + name string + maxLimitBytes int64 + maxLimitWaiters int64 + numRequests int64 + requestSize int64 + timeout time.Duration + }{ + { + name: "below max waiters above max bytes", + maxLimitBytes: 1000, + maxLimitWaiters: 100, + numRequests: 100, + requestSize: 21, + timeout: 5 * time.Second, + }, + { + name: "above max waiters above max bytes", + maxLimitBytes: 1000, + maxLimitWaiters: 100, + numRequests: 200, + requestSize: 21, + timeout: 5 * time.Second, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + bq := NewBoundedQueue(tt.maxLimitBytes, tt.maxLimitWaiters) + var blockedRequests int64 + numReqsUntilBlocked := tt.maxLimitBytes / tt.requestSize + requestsAboveLimit := abs(tt.numRequests - numReqsUntilBlocked) + tooManyWaiters := requestsAboveLimit > tt.maxLimitWaiters + numRejected := max(requestsAboveLimit-tt.maxLimitWaiters, int64(0)) + + // There should never be more blocked requests than maxLimitWaiters. + blockedRequests = min(tt.maxLimitWaiters, requestsAboveLimit) + + ctx, cancel := context.WithTimeout(context.Background(), tt.timeout) + defer cancel() + var errs error + for i := 0; i < int(tt.numRequests); i++ { + go func() { + err := bq.Acquire(ctx, tt.requestSize) + bq.lock.Lock() + defer bq.lock.Unlock() + errs = multierr.Append(errs, err) + }() + } + + require.Eventually(t, func() bool { + bq.lock.Lock() + defer bq.lock.Unlock() + return bq.waiters.Len() == int(blockedRequests) + }, 3*time.Second, 10*time.Millisecond) + + assert.NoError(t, bq.Release(tt.requestSize)) + assert.Equal(t, bq.waiters.Len(), int(blockedRequests)-1) + + for i := 0; i < int(tt.numRequests-numRejected)-1; i++ { + assert.NoError(t, bq.Release(tt.requestSize)) + } + + bq.lock.Lock() + if tooManyWaiters { + assert.ErrorContains(t, errs, ErrTooManyWaiters.Error()) + } else { + assert.NoError(t, errs) + } + bq.lock.Unlock() + + // confirm all bytes were released by acquiring maxLimitBytes. + assert.True(t, bq.TryAcquire(tt.maxLimitBytes)) + }) + } +} + +func TestAcquireContextCanceled(t *testing.T) { + maxLimitBytes := 1000 + maxLimitWaiters := 100 + numRequests := 100 + requestSize := 21 + numReqsUntilBlocked := maxLimitBytes / requestSize + requestsAboveLimit := abs(int64(numRequests) - int64(numReqsUntilBlocked)) + + blockedRequests := min(int64(maxLimitWaiters), requestsAboveLimit) + + bq := NewBoundedQueue(int64(maxLimitBytes), int64(maxLimitWaiters)) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + var errs error + var wg sync.WaitGroup + for i := 0; i < numRequests; i++ { + wg.Add(1) + go func() { + err := bq.Acquire(ctx, int64(requestSize)) + bq.lock.Lock() + defer bq.lock.Unlock() + errs = multierr.Append(errs, err) + wg.Done() + }() + } + + // Wait until all calls to Acquire() happen and we have the expected number of waiters. + require.Eventually(t, func() bool { + bq.lock.Lock() + defer bq.lock.Unlock() + return bq.waiters.Len() == int(blockedRequests) + }, 3*time.Second, 10*time.Millisecond) + + cancel() + wg.Wait() + assert.ErrorContains(t, errs, "context canceled") + + // Now all waiters should have returned and been removed. + assert.Equal(t, 0, bq.waiters.Len()) + + for i := 0; i < numReqsUntilBlocked; i++ { + assert.NoError(t, bq.Release(int64(requestSize))) + assert.Equal(t, int64(0), bq.currentWaiters) + } + assert.True(t, bq.TryAcquire(int64(maxLimitBytes))) +} diff --git a/internal/otelarrow/compression/README.md b/internal/otelarrow/compression/README.md new file mode 100644 index 000000000000..583bf2398e56 --- /dev/null +++ b/internal/otelarrow/compression/README.md @@ -0,0 +1,18 @@ +# Compression configuration package + +## Overview + +This package provides a mechanism for configuring the static +compressors registered with gRPC for the Zstd compression library. +This enables OpenTelemetry Collector components to control all aspects +of Zstd configuration for both encoding and decoding. + +## Usage + +Embed the `EncoderConfig` or `DecoderConfig` object into the component +configuration struct. Compression levels 1..10 are supported via +static compressor objects, registered with names "zstdarrow1", +"zstdarrow10" corresponding with Zstd level. Level-independent +settings are modified using `SetEncoderConfig` or `SetDecoderConfig`. + +For exporters, use `EncoderConfig.CallOption()` at the gRPC call site. diff --git a/internal/otelarrow/compression/zstd/mru.go b/internal/otelarrow/compression/zstd/mru.go new file mode 100644 index 000000000000..930692418b19 --- /dev/null +++ b/internal/otelarrow/compression/zstd/mru.go @@ -0,0 +1,106 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package zstd // import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/otelarrow/compression/zstd" + +import ( + "sync" + "time" +) + +// mru is a freelist whose two main benefits compared to sync.Pool are: +// +// - It doesn't perform any per-CPU caching; it has only a single +// cache. The cache is modeled as a stack, meaning that the most +// recently used item is always the next to be used. (Hence the name +// MRU.) +// +// - It isn't cleared when GC runs. Instead, items that haven't been used +// in a long time (1min) are released. +// +// An MRU freelist is most useful when the objects being freelisted are +// sufficiently valuable, or expensive to create, that they are worth keeping +// across GC passes. The drawbacks are that MRU isn't as performant under +// heavy concurrent access as sync.Pool, and that its sizing logic (1min TTL) +// is less sophisticated than sync.Pool's. +// +// A zero-initialized MRU is safe to use. Threadsafe. +type mru[T generational] struct { + mu sync.Mutex + reset Gen + freelist []T + putTimes []time.Time // putTimes[i] is when freelist[i] was Put() + zero T +} + +// Gen is the reset time. +type Gen time.Time + +type generational interface { + // generation uses monotonic time + generation() Gen +} + +// TTL is modified in testing. +var TTL = time.Minute + +// Get returns an object from the freelist. If the list is empty, the return +// value is the zero value of T. +func (mru *mru[T]) Get() (T, Gen) { + mru.mu.Lock() + defer mru.mu.Unlock() + + if n := len(mru.freelist); n > 0 { + ret := mru.freelist[n-1] + mru.freelist[n-1] = mru.zero // Allow GC to occur. + mru.freelist = mru.freelist[:n-1] + mru.putTimes = mru.putTimes[:n-1] + return ret, mru.reset + } + + return mru.zero, mru.reset +} + +func before(a, b Gen) bool { + return time.Time(a).Before(time.Time(b)) +} + +func (mru *mru[T]) Put(item T) { + mru.mu.Lock() + defer mru.mu.Unlock() + + if before(item.generation(), mru.reset) { + return + } + + now := time.Now() + + mru.freelist = append(mru.freelist, item) + mru.putTimes = append(mru.putTimes, now) + + // Evict any objects that haven't been touched recently. + for len(mru.putTimes) > 0 && now.Sub(mru.putTimes[0]) >= TTL { + // Shift values by one index in the slice, to preserve capacity. + l := len(mru.freelist) + copy(mru.freelist[0:l-1], mru.freelist[1:]) + copy(mru.putTimes[0:l-1], mru.putTimes[1:]) + mru.freelist[l-1] = mru.zero // Allow GC to occur. + mru.freelist = mru.freelist[:l-1] + mru.putTimes = mru.putTimes[:l-1] + } +} + +func (mru *mru[T]) Size() int { + mru.mu.Lock() + defer mru.mu.Unlock() + return len(mru.putTimes) +} + +func (mru *mru[T]) Reset() Gen { + mru.mu.Lock() + defer mru.mu.Unlock() + mru.reset = Gen(time.Now()) + mru.freelist = nil + mru.putTimes = nil + return mru.reset +} diff --git a/internal/otelarrow/compression/zstd/mru_test.go b/internal/otelarrow/compression/zstd/mru_test.go new file mode 100644 index 000000000000..c68875c4bcce --- /dev/null +++ b/internal/otelarrow/compression/zstd/mru_test.go @@ -0,0 +1,84 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package zstd + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +type gint struct { + value int + Gen +} + +func TestMRUGet(t *testing.T) { + defer resetTest() + + var m mru[*gint] + const cnt = 5 + + v, g := m.Get() + require.Nil(t, v) + + for i := 0; i < cnt; i++ { + p := &gint{ + value: i + 1, + Gen: g, + } + m.Put(p) + } + + for i := 0; i < cnt; i++ { + v, _ = m.Get() + require.Equal(t, 5-i, v.value) + } + + v, _ = m.Get() + require.Nil(t, v) +} + +func TestMRUPut(t *testing.T) { + defer resetTest() + + var m mru[*gint] + const cnt = 5 + + // Use zero TTL => no freelist + TTL = 0 + + g := m.Reset() + + for i := 0; i < cnt; i++ { + p := &gint{ + value: i + 1, + Gen: g, + } + m.Put(p) + } + require.Equal(t, 0, m.Size()) +} + +func TestMRUReset(t *testing.T) { + defer resetTest() + + var m mru[*gint] + + g := m.Reset() + + m.Put(&gint{ + Gen: g, + }) + require.Equal(t, 1, m.Size()) + + m.Reset() + require.Equal(t, 0, m.Size()) + + // This doesn't take because its generation is before the reset. + m.Put(&gint{ + Gen: g, + }) + require.Equal(t, 0, m.Size()) +} diff --git a/internal/otelarrow/compression/zstd/zstd.go b/internal/otelarrow/compression/zstd/zstd.go new file mode 100644 index 000000000000..a77bf4139faa --- /dev/null +++ b/internal/otelarrow/compression/zstd/zstd.go @@ -0,0 +1,324 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package zstd // import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/otelarrow/compression/zstd" + +import ( + "bytes" + "errors" + "fmt" + "io" + "runtime" + "sync" + + zstdlib "github.com/klauspost/compress/zstd" + "google.golang.org/grpc" + "google.golang.org/grpc/encoding" +) + +// NamePrefix is prefix, with N for compression level. +const NamePrefix = "zstdarrow" + +// Level is an integer value mapping to compression level. +// [0] implies disablement; not registered in grpc +// [1,2] fastest i.e., "zstdarrow1", "zstdarrow2" +// [3-5] default +// [6-9] better +// [10] best. +type Level uint + +const ( + // DefaultLevel is a reasonable balance of compression and cpu usage. + DefaultLevel Level = 5 + // MinLevel is fast and cheap. + MinLevel Level = 1 + // MaxLevel is slow and expensive. + MaxLevel Level = 10 +) + +type EncoderConfig struct { + // Level is meaningful in the range [0, 10]. No invalid + // values, they all map into 4 default configurations. (default: 5) + // See `zstdlib.WithEncoderLevel()`. + Level Level `mapstructure:"level"` + // WindowSizeMiB is a Zstd-library parameter that controls how + // much window of text is visible to the compressor at a time. + // It is the dominant factor that determines memory usage. + // If zero, the window size is determined by level. (default: 0) + // See `zstdlib.WithWindowSize()`. + WindowSizeMiB uint32 `mapstructure:"window_size_mib"` + // Concurrency is a Zstd-library parameter that configures the + // use of background goroutines to improve compression speed. + // 0 means to let the library decide (it will use up to GOMAXPROCS), + // and 1 means to avoid background workers. (default: 1) + // See `zstdlib.WithEncoderConcurrency()`. + Concurrency uint `mapstructure:"concurrency"` +} + +type DecoderConfig struct { + // MemoryLimitMiB is a memory limit control for the decoder, + // as a way to limit overall memory use by Zstd. + // See `zstdlib.WithDecoderMaxMemory()`. + MemoryLimitMiB uint32 `mapstructure:"memory_limit_mib"` + // MaxWindowSizeMiB limits window sizes that can be configured + // in the corresponding encoder's `EncoderConfig.WindowSizeMiB` + // setting, as a way to control memory usage. + // See `zstdlib.WithDecoderMaxWindow()`. + MaxWindowSizeMiB uint32 `mapstructure:"max_window_size_mib"` + // Concurrency is a Zstd-library parameter that configures the + // use of background goroutines to improve decompression speed. + // 0 means to let the library decide (it will use up to GOMAXPROCS), + // and 1 means to avoid background workers. (default: 1) + // See `zstdlib.WithDecoderConcurrency()`. + Concurrency uint `mapstructure:"concurrency"` +} + +type encoder struct { + lock sync.Mutex // protects cfg + cfg EncoderConfig + pool mru[*writer] +} + +type decoder struct { + lock sync.Mutex // protects cfg + cfg DecoderConfig + pool mru[*reader] +} + +type reader struct { + *zstdlib.Decoder + Gen + pool *mru[*reader] +} + +type writer struct { + *zstdlib.Encoder + Gen + pool *mru[*writer] +} + +type combined struct { + enc encoder + dec decoder +} + +type instance struct { + lock sync.Mutex + byLevel map[Level]*combined +} + +var _ encoding.Compressor = &combined{} + +var staticInstances = &instance{ + byLevel: map[Level]*combined{}, +} + +func (g *Gen) generation() Gen { + return *g +} + +func DefaultEncoderConfig() EncoderConfig { + return EncoderConfig{ + Level: DefaultLevel, // Determines other defaults + Concurrency: 1, // Avoids extra CPU/memory + } +} + +func DefaultDecoderConfig() DecoderConfig { + return DecoderConfig{ + Concurrency: 1, // Avoids extra CPU/memory + MemoryLimitMiB: 128, // More conservative than library default + MaxWindowSizeMiB: 32, // Corresponds w/ "best" level default + } +} + +func validate(level Level, f func() error) error { + if level > MaxLevel { + return fmt.Errorf("level out of range [0,10]: %d", level) + } + if level < MinLevel { + return fmt.Errorf("level out of range [0,10]: %d", level) + } + return f() +} + +func (cfg EncoderConfig) Validate() error { + return validate(cfg.Level, func() error { + var buf bytes.Buffer + test, err := zstdlib.NewWriter(&buf, cfg.options()...) + if test != nil { + test.Close() + } + return err + }) +} + +func (cfg DecoderConfig) Validate() error { + return validate(MinLevel, func() error { + var buf bytes.Buffer + test, err := zstdlib.NewReader(&buf, cfg.options()...) + if test != nil { + test.Close() + } + return err + }) +} + +func init() { + staticInstances.lock.Lock() + defer staticInstances.lock.Unlock() + resetLibrary() +} + +func resetLibrary() { + for level := MinLevel; level <= MaxLevel; level++ { + var combi combined + combi.enc.cfg = DefaultEncoderConfig() + combi.dec.cfg = DefaultDecoderConfig() + combi.enc.cfg.Level = level + encoding.RegisterCompressor(&combi) + staticInstances.byLevel[level] = &combi + } +} + +func SetEncoderConfig(cfg EncoderConfig) error { + if err := cfg.Validate(); err != nil { + return err + } + + updateOne := func(enc *encoder) { + enc.lock.Lock() + defer enc.lock.Unlock() + enc.cfg = cfg + enc.pool.Reset() + } + + staticInstances.lock.Lock() + defer staticInstances.lock.Unlock() + + updateOne(&staticInstances.byLevel[cfg.Level].enc) + return nil +} + +func SetDecoderConfig(cfg DecoderConfig) error { + if err := cfg.Validate(); err != nil { + return err + } + updateOne := func(dec *decoder) { + dec.lock.Lock() + defer dec.lock.Unlock() + dec.cfg = cfg + dec.pool.Reset() + } + + staticInstances.lock.Lock() + defer staticInstances.lock.Unlock() + + for level := MinLevel; level <= MaxLevel; level++ { + updateOne(&staticInstances.byLevel[level].dec) + } + return nil + +} + +func (cfg EncoderConfig) options() (opts []zstdlib.EOption) { + opts = append(opts, zstdlib.WithEncoderLevel(zstdlib.EncoderLevelFromZstd(int(cfg.Level)))) + + if cfg.Concurrency != 0 { + opts = append(opts, zstdlib.WithEncoderConcurrency(int(cfg.Concurrency))) + } + if cfg.WindowSizeMiB != 0 { + opts = append(opts, zstdlib.WithWindowSize(int(cfg.WindowSizeMiB<<20))) + } + + return opts +} + +func (e *encoder) getConfig() EncoderConfig { + e.lock.Lock() + defer e.lock.Unlock() + return e.cfg +} + +func (cfg EncoderConfig) Name() string { + return fmt.Sprint(NamePrefix, cfg.Level) +} + +func (cfg EncoderConfig) CallOption() grpc.CallOption { + if cfg.Level < MinLevel || cfg.Level > MaxLevel { + return grpc.UseCompressor(EncoderConfig{Level: DefaultLevel}.Name()) + } + return grpc.UseCompressor(cfg.Name()) +} + +func (cfg DecoderConfig) options() (opts []zstdlib.DOption) { + if cfg.Concurrency != 0 { + opts = append(opts, zstdlib.WithDecoderConcurrency(int(cfg.Concurrency))) + } + if cfg.MaxWindowSizeMiB != 0 { + opts = append(opts, zstdlib.WithDecoderMaxWindow(uint64(cfg.MaxWindowSizeMiB)<<20)) + } + if cfg.MemoryLimitMiB != 0 { + opts = append(opts, zstdlib.WithDecoderMaxMemory(uint64(cfg.MemoryLimitMiB)<<20)) + } + + return opts +} + +func (d *decoder) getConfig() DecoderConfig { + d.lock.Lock() + defer d.lock.Unlock() + return d.cfg +} + +func (c *combined) Compress(w io.Writer) (io.WriteCloser, error) { + z, gen := c.enc.pool.Get() + if z == nil { + encoder, err := zstdlib.NewWriter(w, c.enc.getConfig().options()...) + if err != nil { + return nil, err + } + z = &writer{Encoder: encoder, pool: &c.enc.pool, Gen: gen} + } else { + z.Encoder.Reset(w) + } + return z, nil +} + +func (w *writer) Close() error { + defer w.pool.Put(w) + return w.Encoder.Close() +} + +func (c *combined) Decompress(r io.Reader) (io.Reader, error) { + z, gen := c.dec.pool.Get() + if z == nil { + decoder, err := zstdlib.NewReader(r, c.dec.getConfig().options()...) + if err != nil { + return nil, err + } + z = &reader{Decoder: decoder, pool: &c.dec.pool, Gen: gen} + + // zstd decoders need to be closed when they are evicted from + // the freelist. Note that the finalizer is attached to the + // reader object, not to the decoder, because zstd maintains + // background references to the decoder that prevent it from + // being GC'ed. + runtime.SetFinalizer(z, (*reader).Close) + } else if err := z.Decoder.Reset(r); err != nil { + return nil, err + } + return z, nil +} + +func (r *reader) Read(p []byte) (n int, err error) { + n, err = r.Decoder.Read(p) + if errors.Is(err, io.EOF) { + r.pool.Put(r) + } + return n, err +} + +func (c *combined) Name() string { + return c.enc.cfg.Name() +} diff --git a/internal/otelarrow/compression/zstd/zstd_test.go b/internal/otelarrow/compression/zstd/zstd_test.go new file mode 100644 index 000000000000..6a529f9a6d24 --- /dev/null +++ b/internal/otelarrow/compression/zstd/zstd_test.go @@ -0,0 +1,181 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package zstd + +import ( + "bytes" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/encoding" +) + +func resetTest() { + TTL = time.Minute + resetLibrary() +} + +func TestCompressorNonNil(t *testing.T) { + defer resetTest() + + for i := 1; i <= 10; i++ { + require.NotNil(t, encoding.GetCompressor(fmt.Sprint(NamePrefix, i))) + } + require.Nil(t, encoding.GetCompressor(fmt.Sprint(NamePrefix, MinLevel-1))) + require.Nil(t, encoding.GetCompressor(fmt.Sprint(NamePrefix, MaxLevel+1))) +} + +func TestConfigLibraryError(t *testing.T) { + defer resetTest() + + require.Error(t, SetEncoderConfig(EncoderConfig{ + Level: 1, + WindowSizeMiB: 1024, + })) + require.NoError(t, SetDecoderConfig(DecoderConfig{ + MaxWindowSizeMiB: 1024, + })) +} + +func TestInvalidCompressorLevel(t *testing.T) { + defer resetTest() + + require.Error(t, SetEncoderConfig(EncoderConfig{ + Level: 12, + Concurrency: 10, + WindowSizeMiB: 16, + })) +} + +func TestAllCompressorOptions(t *testing.T) { + defer resetTest() + + require.NoError(t, SetEncoderConfig(EncoderConfig{ + Level: 9, + Concurrency: 10, + WindowSizeMiB: 16, + })) + require.NoError(t, SetDecoderConfig(DecoderConfig{ + Concurrency: 10, + MaxWindowSizeMiB: 16, + MemoryLimitMiB: 256, + })) +} + +func TestCompressorReset(t *testing.T) { + defer resetTest() + + // Get compressor configs 1 and 2. + comp1 := encoding.GetCompressor("zstdarrow1").(*combined) + comp2 := encoding.GetCompressor("zstdarrow2").(*combined) + + // Get an object for level 1 + var buf bytes.Buffer + wc, err := comp1.Compress(&buf) + require.NoError(t, err) + + // Put back the once, it will be saved. + save := wc.(*writer) + require.NoError(t, wc.Close()) + require.Equal(t, 1, comp1.enc.pool.Size()) + require.Equal(t, 0, comp2.enc.pool.Size()) + + // We get the same object pointer again. + wc, err = comp1.Compress(&buf) + require.NoError(t, err) + require.Equal(t, save, wc.(*writer)) + + // Modify 1's encoder configuration. + encCfg1 := comp1.enc.getConfig() + encCfg2 := comp2.enc.getConfig() + cpyCfg1 := encCfg1 + cpyCfg1.WindowSizeMiB = 32 + + require.Equal(t, Level(1), cpyCfg1.Level) + require.NotEqual(t, cpyCfg1, encCfg1, "see %v %v", cpyCfg1, encCfg1) + + require.NoError(t, SetEncoderConfig(cpyCfg1)) + + // The instances can't have changed. + require.Equal(t, comp1, encoding.GetCompressor("zstdarrow1").(*combined)) + require.Equal(t, comp2, encoding.GetCompressor("zstdarrow2").(*combined)) + + // Level 2 is unchanged + require.Equal(t, encCfg2, comp2.enc.getConfig()) + + // Level 1 is changed + require.NotEqual(t, encCfg1, comp1.enc.getConfig(), "see %v %v", encCfg1, comp1.enc.getConfig()) + + // Put back the saved item, it will not be placed back in the + // pool due to reset. + require.NoError(t, wc.Close()) + require.Equal(t, 0, comp1.enc.pool.Size()) + // Explicitly, we get a nil from the pool. + v, _ := comp1.enc.pool.Get() + require.Nil(t, v) +} + +func TestDecompressorReset(t *testing.T) { + defer resetTest() + + // Get compressor configs 3 and 4. + comp3 := encoding.GetCompressor("zstdarrow3").(*combined) + comp4 := encoding.GetCompressor("zstdarrow4").(*combined) + + // Get an object for level 3 + buf := new(bytes.Buffer) + rd, err := comp3.Decompress(buf) + require.NoError(t, err) + _, err = rd.Read([]byte{}) + require.Error(t, err) + + // We get the same object pointer again. + buf = new(bytes.Buffer) + rd, err = comp3.Decompress(buf) + require.NoError(t, err) + _, err = rd.Read(nil) + require.Error(t, err) + + // Modify 3's encoder configuration. + decCfg3 := comp3.dec.getConfig() + decCfg4 := comp4.dec.getConfig() + cpyCfg3 := decCfg3 + cpyCfg3.MaxWindowSizeMiB = 128 + + require.NotEqual(t, cpyCfg3, decCfg3, "see %v %v", cpyCfg3, decCfg3) + + require.NoError(t, SetDecoderConfig(cpyCfg3)) + + // The instances can't have changed. + require.Equal(t, comp3, encoding.GetCompressor("zstdarrow3").(*combined)) + require.Equal(t, comp4, encoding.GetCompressor("zstdarrow4").(*combined)) + + // Level 4 is _also changed_ + require.Equal(t, cpyCfg3, comp4.dec.getConfig()) + require.NotEqual(t, decCfg4, comp4.dec.getConfig()) + + // Level 3 is changed + require.NotEqual(t, decCfg3, comp3.dec.getConfig(), "see %v %v", decCfg3, comp3.dec.getConfig()) + + // Unlike the encoder test, which has an explicit Close() to its advantage, + // we aren't testing the behavior of the finalizer that puts back into the MRU. +} + +func TestGRPCCallOption(t *testing.T) { + cfgN := func(l Level) EncoderConfig { + return EncoderConfig{ + Level: l, + } + } + cfg2 := cfgN(2) + require.Equal(t, cfg2.Name(), cfg2.CallOption().(grpc.CompressorCallOption).CompressorType) + + cfgD := cfgN(DefaultLevel) + cfg13 := cfgN(13) + // Invalid maps to default call option + require.Equal(t, cfgD.Name(), cfg13.CallOption().(grpc.CompressorCallOption).CompressorType) +} diff --git a/internal/otelarrow/go.mod b/internal/otelarrow/go.mod new file mode 100644 index 000000000000..47a686770319 --- /dev/null +++ b/internal/otelarrow/go.mod @@ -0,0 +1,103 @@ +module github.com/open-telemetry/opentelemetry-collector-contrib/internal/otelarrow + +go 1.21.0 + +require ( + github.com/google/uuid v1.6.0 + github.com/klauspost/compress v1.17.9 + github.com/open-telemetry/opentelemetry-collector-contrib/exporter/otelarrowexporter v0.105.0 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/otelarrowreceiver v0.105.0 + github.com/open-telemetry/otel-arrow v0.24.0 + github.com/stretchr/testify v1.9.0 + github.com/wk8/go-ordered-map/v2 v2.1.8 + go.opentelemetry.io/collector/component v0.105.0 + go.opentelemetry.io/collector/config/configtelemetry v0.105.0 + go.opentelemetry.io/collector/consumer v0.105.0 + go.opentelemetry.io/collector/exporter v0.105.0 + go.opentelemetry.io/collector/pdata v1.12.0 + go.opentelemetry.io/collector/receiver v0.105.0 + go.opentelemetry.io/otel v1.28.0 + go.opentelemetry.io/otel/metric v1.28.0 + go.opentelemetry.io/otel/sdk v1.28.0 + go.opentelemetry.io/otel/sdk/metric v1.28.0 + go.opentelemetry.io/otel/trace v1.28.0 + go.uber.org/multierr v1.11.0 + go.uber.org/zap v1.27.0 + google.golang.org/grpc v1.65.0 +) + +require ( + github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect + github.com/apache/arrow/go/v16 v16.1.0 // indirect + github.com/axiomhq/hyperloglog v0.0.0-20230201085229-3ddf4bad03dc // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/brianvoe/gofakeit/v6 v6.17.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.4.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect + github.com/google/flatbuffers v24.3.25+incompatible // indirect + github.com/hashicorp/go-version v1.7.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.1.1 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/mostynb/go-grpc-compression v1.2.3 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/open-telemetry/otel-arrow/collector v0.24.0 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.19.1 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/x448/float16 v0.8.4 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect + go.opentelemetry.io/collector v0.105.0 // indirect + go.opentelemetry.io/collector/config/configauth v0.105.0 // indirect + go.opentelemetry.io/collector/config/configcompression v1.12.0 // indirect + go.opentelemetry.io/collector/config/configgrpc v0.105.0 // indirect + go.opentelemetry.io/collector/config/confignet v0.105.0 // indirect + go.opentelemetry.io/collector/config/configopaque v1.12.0 // indirect + go.opentelemetry.io/collector/config/configretry v1.12.0 // indirect + go.opentelemetry.io/collector/config/configtls v1.12.0 // indirect + go.opentelemetry.io/collector/config/internal v0.105.0 // indirect + go.opentelemetry.io/collector/confmap v0.105.0 // indirect + go.opentelemetry.io/collector/extension v0.105.0 // indirect + go.opentelemetry.io/collector/extension/auth v0.105.0 // indirect + go.opentelemetry.io/collector/featuregate v1.12.0 // indirect + go.opentelemetry.io/collector/internal/globalgates v0.105.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect + go.opentelemetry.io/otel/exporters/prometheus v0.50.0 // indirect + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.27.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/otelarrowreceiver => ../../receiver/otelarrowreceiver + +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/otelarrowexporter => ../../exporter/otelarrowexporter diff --git a/internal/otelarrow/go.sum b/internal/otelarrow/go.sum new file mode 100644 index 000000000000..af729aec4144 --- /dev/null +++ b/internal/otelarrow/go.sum @@ -0,0 +1,271 @@ +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= +github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/apache/arrow/go/v16 v16.1.0 h1:dwgfOya6s03CzH9JrjCBx6bkVb4yPD4ma3haj9p7FXI= +github.com/apache/arrow/go/v16 v16.1.0/go.mod h1:9wnc9mn6vEDTRIm4+27pEjQpRKuTvBaessPoEXQzxWA= +github.com/axiomhq/hyperloglog v0.0.0-20230201085229-3ddf4bad03dc h1:Keo7wQ7UODUaHcEi7ltENhbAK2VgZjfat6mLy03tQzo= +github.com/axiomhq/hyperloglog v0.0.0-20230201085229-3ddf4bad03dc/go.mod h1:k08r+Yj1PRAmuayFiRK6MYuR5Ve4IuZtTfxErMIh0+c= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/brianvoe/gofakeit/v6 v6.17.0 h1:obbQTJeHfktJtiZzq0Q1bEpsNUs+yHrYlPVWt7BtmJ4= +github.com/brianvoe/gofakeit/v6 v6.17.0/go.mod h1:Ow6qC71xtwm79anlwKRlWZW6zVq9D2XHE4QSSMP/rU8= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc h1:8WFBn63wegobsYAX0YjD+8suexZDga5CctH4CCTx2+8= +github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88= +github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsMBaUOKXq6HSv655U1c= +github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= +github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= +github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= +github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= +github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM= +github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mostynb/go-grpc-compression v1.2.3 h1:42/BKWMy0KEJGSdWvzqIyOZ95YcR9mLPqKctH7Uo//I= +github.com/mostynb/go-grpc-compression v1.2.3/go.mod h1:AghIxF3P57umzqM9yz795+y1Vjs47Km/Y2FE6ouQ7Lg= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/open-telemetry/otel-arrow v0.24.0 h1:hNUEbwHW/1gEOUiN+HoI+ITiXe2vSBaPWlE9FRwJwDE= +github.com/open-telemetry/otel-arrow v0.24.0/go.mod h1:uzoHixEh6CUBZkP+vkRvyiHYUnYsAOUwCcfByQkSMM0= +github.com/open-telemetry/otel-arrow/collector v0.24.0 h1:NYTcgtwG0lQnoGcEomTTtueZxzk03xt+XEXN4L5kqHA= +github.com/open-telemetry/otel-arrow/collector v0.24.0/go.mod h1:+jJ3Vfhh685hXSw2Z1P1wl/rTqEKlSaJ4FocZI+xs+0= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.opentelemetry.io/collector v0.105.0 h1:Qw/ONVMPT3aD8HjdDRcXCGoZrtSWH3jx4BkwAN1yrEM= +go.opentelemetry.io/collector v0.105.0/go.mod h1:UVapTqB4fJeZpGU/YgOo6665cxCSytqYmMkVmRlu2cg= +go.opentelemetry.io/collector/component v0.105.0 h1:/OdkWHd1xTNX7JRq9iW3AFoJAnYUOGZZyOprNQkGoTI= +go.opentelemetry.io/collector/component v0.105.0/go.mod h1:s8KoxOrhNIBzetkb0LHmzX1OI67DyZbaaUPOWIXS1mg= +go.opentelemetry.io/collector/config/configauth v0.105.0 h1:9Pa65Ay4kdmMsp5mg+/791GvCYy1hHOroIlKBiJzlps= +go.opentelemetry.io/collector/config/configauth v0.105.0/go.mod h1:iL62YzyFCNr1Se0EDYaQ792CFCBiFivSbTWekd4g1VE= +go.opentelemetry.io/collector/config/configcompression v1.12.0 h1:RxqSDVZPJyL7I3v+gdVDvnJ/9tV0ZWgraRDX/gaddfA= +go.opentelemetry.io/collector/config/configcompression v1.12.0/go.mod h1:6+m0GKCv7JKzaumn7u80A2dLNCuYf5wdR87HWreoBO0= +go.opentelemetry.io/collector/config/configgrpc v0.105.0 h1:3IbN6+5c42Bp6CGPKoXZ7QkkQpRxnV01KRse6oD+bd0= +go.opentelemetry.io/collector/config/configgrpc v0.105.0/go.mod h1:wrFPXVXk4so7yYisuPeebZtU6ECzG5aXOYNdfyoHSnI= +go.opentelemetry.io/collector/config/confignet v0.105.0 h1:O8kenkWnLPemp2XXVOqFv6OQd9wsnOvBUvl3OlJGPKI= +go.opentelemetry.io/collector/config/confignet v0.105.0/go.mod h1:pfOrCTfSZEB6H2rKtx41/3RN4dKs+X2EKQbw3MGRh0E= +go.opentelemetry.io/collector/config/configopaque v1.12.0 h1:aIsp9NdcLZSiG4YDoFPGXhmma03Tk+6e89+n8GtU/Mc= +go.opentelemetry.io/collector/config/configopaque v1.12.0/go.mod h1:0xURn2sOy5j4fbaocpEYfM97HPGsiffkkVudSPyTJlM= +go.opentelemetry.io/collector/config/configretry v1.12.0 h1:tEBwueO4AIkwWosxz6NWqnghdZ7y5SfHcIzLrvh6kB8= +go.opentelemetry.io/collector/config/configretry v1.12.0/go.mod h1:P+RA0IA+QoxnDn4072uyeAk1RIoYiCbxYsjpKX5eFC4= +go.opentelemetry.io/collector/config/configtelemetry v0.105.0 h1:wEfUxAjjstp47aLr2s1cMZiH0dt+k42m6VC6HigqgJA= +go.opentelemetry.io/collector/config/configtelemetry v0.105.0/go.mod h1:WxWKNVAQJg/Io1nA3xLgn/DWLE/W1QOB2+/Js3ACi40= +go.opentelemetry.io/collector/config/configtls v1.12.0 h1:Px0+GE4LE/9sXMgkwBb5g8QHWvnrnuRg9BLSa+QtxgM= +go.opentelemetry.io/collector/config/configtls v1.12.0/go.mod h1:aeCGPlvrWhc+EySpIKdelPAj4l9wXKzZPouQO3NIoTs= +go.opentelemetry.io/collector/config/internal v0.105.0 h1:PWnbeslkIGMjZzh5IJRjO6bA02d1Xrkjw2N60ixWzqQ= +go.opentelemetry.io/collector/config/internal v0.105.0/go.mod h1:+Y5vRJ+lio2uuYlVPfy9AZVrip9Y0B9PiUA5Vz7lzZw= +go.opentelemetry.io/collector/confmap v0.105.0 h1:3NP2BbUju42rjeQvRbmpCJGJGvbiV3WnGyXsVmocimo= +go.opentelemetry.io/collector/confmap v0.105.0/go.mod h1:Oj1xUBRvAuL8OWWMj9sSYf1uQpB+AErpj+FKGUQLBI0= +go.opentelemetry.io/collector/consumer v0.105.0 h1:pO5Tspoz7yvEs81+904HfDjByP8Z7uuNk+7pOr3lRHM= +go.opentelemetry.io/collector/consumer v0.105.0/go.mod h1:tnaPDHUfKBJ01OnsJNRecniG9iciE+xHYLqamYwFQOQ= +go.opentelemetry.io/collector/exporter v0.105.0 h1:O2xmjfaRbkbpo3XkwEcnuBHCoXc5kS9CjYO8geu+3vo= +go.opentelemetry.io/collector/exporter v0.105.0/go.mod h1:5ulGEHRZyGbX4DWHJa2Br6Fr/W1Lay8ayf++1WrVvgk= +go.opentelemetry.io/collector/extension v0.105.0 h1:R8i4HMvuSm20Nt3onyrLk19KKhjCNAsgS8FGh60rcZU= +go.opentelemetry.io/collector/extension v0.105.0/go.mod h1:oyX960URG27esNKitf3o2rqcBj0ajcx+dxkCxwRz34U= +go.opentelemetry.io/collector/extension/auth v0.105.0 h1:5gzRSHU0obVtZDzLLJQ/p4sIkacUsyEEpBiBRDs82Hk= +go.opentelemetry.io/collector/extension/auth v0.105.0/go.mod h1:zf45v7u1nKbdDHeMuhBVdSFwhbq2w9IWCbFKcDSkW5I= +go.opentelemetry.io/collector/featuregate v1.12.0 h1:l5WbV2vMQd2bL8ubfGrbKNtZaeJRckE12CTHvRe47Tw= +go.opentelemetry.io/collector/featuregate v1.12.0/go.mod h1:PsOINaGgTiFc+Tzu2K/X2jP+Ngmlp7YKGV1XrnBkH7U= +go.opentelemetry.io/collector/internal/globalgates v0.105.0 h1:U/CwnTUXtrblD1sZ6ri7KWfYoTNjQd7GjJKrX/phRik= +go.opentelemetry.io/collector/internal/globalgates v0.105.0/go.mod h1:Z5US6O2xkZAtxVSSBnHAPFZwPhFoxlyKLUvS67Vx4gc= +go.opentelemetry.io/collector/pdata v1.12.0 h1:Xx5VK1p4VO0md8MWm2icwC1MnJ7f8EimKItMWw46BmA= +go.opentelemetry.io/collector/pdata v1.12.0/go.mod h1:MYeB0MmMAxeM0hstCFrCqWLzdyeYySim2dG6pDT6nYI= +go.opentelemetry.io/collector/pdata/pprofile v0.105.0 h1:C+Hd7CNcepL/364OBV9f4lHzJil2jQSOxcEM1PFXGDg= +go.opentelemetry.io/collector/pdata/pprofile v0.105.0/go.mod h1:chr7lMJIzyXkccnPRkIPhyXtqLZLSReZYhwsggOGEfg= +go.opentelemetry.io/collector/pdata/testdata v0.105.0 h1:5sPZzanR4nJR3sNQk3MTdArdEZCK0NRAfC29t0Dtf60= +go.opentelemetry.io/collector/pdata/testdata v0.105.0/go.mod h1:NIfgaclQp/M1BZhgyc/7hDWD+/DumC/OMBQVI2KW+N0= +go.opentelemetry.io/collector/receiver v0.105.0 h1:eZF97kMUnKJ20Uc4PaDlgLIGmaA8kyLqhH+vMXjh92U= +go.opentelemetry.io/collector/receiver v0.105.0/go.mod h1:nGKDXLUGVHxMBJ5QLfsJ/bIhGvoMGqsN0pZtD5SC8sE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 h1:9G6E0TXzGFVfTnawRzrPl83iHOAV7L8NJiR8RSGYV1g= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0/go.mod h1:azvtTADFQJA8mX80jIH/akaE7h+dbm/sVuaHqN13w74= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/exporters/prometheus v0.50.0 h1:2Ewsda6hejmbhGFyUvWZjUThC98Cf8Zy6g0zkIimOng= +go.opentelemetry.io/otel/exporters/prometheus v0.50.0/go.mod h1:pMm5PkUo5YwbLiuEf7t2xg4wbP0/eSJrMxIMxKosynY= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/sdk/metric v1.28.0 h1:OkuaKgKrgAbYrrY0t92c+cC+2F6hsFNnCQArXCKlg08= +go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= +gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/internal/otelarrow/metadata.yaml b/internal/otelarrow/metadata.yaml new file mode 100644 index 000000000000..0bd31377fc0d --- /dev/null +++ b/internal/otelarrow/metadata.yaml @@ -0,0 +1,3 @@ +status: + codeowners: + active: [jmacd, moh-osman3] diff --git a/internal/otelarrow/netstats/README.md b/internal/otelarrow/netstats/README.md new file mode 100644 index 000000000000..55783ec7d2bf --- /dev/null +++ b/internal/otelarrow/netstats/README.md @@ -0,0 +1,37 @@ +# Network Statistics package + +## Overview + +This package provides general support for counting the number of bytes +read and written by an exporter or receiver component. This package +specifically supports monitoring the compression rate achieved by +OpenTelemetry Protocol with Apache Arrow, but it can be easily adopted +for any gRPC-based component, for both unary and streaming RPCs. + +## Usage + +To create a network reporter, pass the exporter or receiver settings +to `netstats.NewExporterNetworkReporter` or +`netstats.NewExporterNetworkReporter`, then register with gRPC: + +``` + dialOpts = append(dialOpts, grpc.WithStatsHandler(netReporter.Handler())) +``` + +Because OTel-Arrow supports the use of compressed payloads, configured +through Arrow IPC, it is necessary for the exporter and receiver +components to manually account for uncompressed payload sizes. + +The `SizesStruct` supports recording either one or both of the +compressed and uncompressed sizes. To report only uncompressed size +in the exporter case, for example: + +``` + var sized netstats.SizesStruct + sized.Method = s.method + sized.Length = int64(uncompressedSize) + netReporter.CountSend(ctx, sized) +``` + +Likewise, the receiver uses `CountRecv` with `sized.Length` set to +report its uncompressed size after OTel-Arrow decompression. diff --git a/internal/otelarrow/netstats/grpc.go b/internal/otelarrow/netstats/grpc.go new file mode 100644 index 000000000000..cc994c6b4970 --- /dev/null +++ b/internal/otelarrow/netstats/grpc.go @@ -0,0 +1,20 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package netstats // import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/otelarrow/netstats" + +import "google.golang.org/grpc" + +// GRPCStreamMethodName applies the logic gRPC uses but does not expose to construct +// method names. This allows direct calling of the netstats interface +// from outside a gRPC stats handler. +func GRPCStreamMethodName(desc grpc.ServiceDesc, stream grpc.StreamDesc) string { + return "/" + desc.ServiceName + "/" + stream.StreamName +} + +// GRPCUnaryMethodName applies the logic gRPC uses but does not expose to construct +// method names. This allows direct calling of the netstats interface +// from outside a gRPC stats handler. +func GRPCUnaryMethodName(desc grpc.ServiceDesc, method grpc.MethodDesc) string { + return "/" + desc.ServiceName + "/" + method.MethodName +} diff --git a/internal/otelarrow/netstats/handler.go b/internal/otelarrow/netstats/handler.go new file mode 100644 index 000000000000..73507fd35b60 --- /dev/null +++ b/internal/otelarrow/netstats/handler.go @@ -0,0 +1,83 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package netstats // import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/otelarrow/netstats" + +import ( + "context" + "strings" + + "google.golang.org/grpc/stats" +) + +type netstatsContext struct{} // value: string + +type statsHandler struct { + rep *NetworkReporter +} + +var _ stats.Handler = statsHandler{} + +func (rep *NetworkReporter) Handler() stats.Handler { + return statsHandler{rep: rep} +} + +// TagRPC implements grpc/stats.Handler +func (statsHandler) TagRPC(ctx context.Context, s *stats.RPCTagInfo) context.Context { + return context.WithValue(ctx, netstatsContext{}, s.FullMethodName) +} + +// trustUncompressed is a super hacky way of knowing when the +// uncompressed size is realistic. nothing else would work -- the +// same handler is used by both arrow and non-arrow, and the +// `*stats.Begin` which indicates streaming vs. not streaming does not +// appear in TagRPC() where we could store it in context. this +// approach is considered a better alternative than others, however +// ugly. when non-arrow RPCs are sent, the instrumentation works +// correctly, this avoids special instrumentation outside of the Arrow +// components. +func trustUncompressed(method string) bool { + return !strings.Contains(method, "arrow.v1") +} + +func (h statsHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) { + switch rs.(type) { + case *stats.InHeader, *stats.InTrailer, *stats.Begin, *stats.OutHeader, *stats.OutTrailer: + // Note we have some info about header WireLength, + // but intentionally not counting. + return + } + method := "unknown" + if name := ctx.Value(netstatsContext{}); name != nil { + method = name.(string) + } + switch s := rs.(type) { + case *stats.InPayload: + var ss SizesStruct + ss.Method = method + if trustUncompressed(method) { + ss.Length = int64(s.Length) + } + ss.WireLength = int64(s.WireLength) + h.rep.CountReceive(ctx, ss) + + case *stats.OutPayload: + var ss SizesStruct + ss.Method = method + if trustUncompressed(method) { + ss.Length = int64(s.Length) + } + ss.WireLength = int64(s.WireLength) + h.rep.CountSend(ctx, ss) + } +} + +// TagConn implements grpc/stats.Handler +func (statsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { + return ctx +} + +// HandleConn implements grpc/stats.Handler +func (statsHandler) HandleConn(_ context.Context, _ stats.ConnStats) { + // Note: ConnBegin and ConnEnd +} diff --git a/internal/otelarrow/netstats/netstats.go b/internal/otelarrow/netstats/netstats.go new file mode 100644 index 000000000000..2ccba74ad5b3 --- /dev/null +++ b/internal/otelarrow/netstats/netstats.go @@ -0,0 +1,263 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package netstats // import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/otelarrow/netstats" + +import ( + "context" + + "go.opentelemetry.io/collector/config/configtelemetry" + "go.opentelemetry.io/collector/exporter" + "go.opentelemetry.io/collector/receiver" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + noopmetric "go.opentelemetry.io/otel/metric/noop" + "go.opentelemetry.io/otel/trace" + "go.uber.org/multierr" +) + +const ( + // ExporterKey is an attribute name that identifies an + // exporter component that produces internal metrics, logs, + // and traces. + ExporterKey = "exporter" + + // ReceiverKey is an attribute name that identifies an + // receiver component that produces internal metrics, logs, + // and traces. + ReceiverKey = "receiver" + + // SentBytes is used to track bytes sent by exporters and receivers. + SentBytes = "sent" + + // SentWireBytes is used to track bytes sent on the wire + // (includes compression) by exporters and receivers. + SentWireBytes = "sent_wire" + + // RecvBytes is used to track bytes received by exporters and receivers. + RecvBytes = "recv" + + // RecvWireBytes is used to track bytes received on the wire + // (includes compression) by exporters and receivers. + RecvWireBytes = "recv_wire" + + // CompSize is used for compressed size histogram metrics. + CompSize = "compressed_size" + + scopeName = "github.com/open-telemetry/opentelemetry-collector-contrib/internal/otelarrow/netstats" +) + +// NetworkReporter is a helper to add network-level observability to +// an exporter or receiver. +type NetworkReporter struct { + isExporter bool + staticAttr attribute.KeyValue + sentBytes metric.Int64Counter + sentWireBytes metric.Int64Counter + recvBytes metric.Int64Counter + recvWireBytes metric.Int64Counter + compSizeHisto metric.Int64Histogram +} + +var _ Interface = &NetworkReporter{} + +// SizesStruct is used to pass uncompressed on-wire message lengths to +// the CountSend() and CountReceive() methods. +type SizesStruct struct { + // Method refers to the gRPC method name + Method string + // Length is the uncompressed size + Length int64 + // WireLength is compressed size + WireLength int64 +} + +// Interface describes a *NetworkReporter or a Noop. +type Interface interface { + // CountSend reports outbound bytes. + CountSend(ctx context.Context, ss SizesStruct) + + // CountSend reports inbound bytes. + CountReceive(ctx context.Context, ss SizesStruct) +} + +// Noop is a no-op implementation of Interface. +type Noop struct{} + +var _ Interface = Noop{} + +func (Noop) CountSend(context.Context, SizesStruct) {} +func (Noop) CountReceive(context.Context, SizesStruct) {} + +const ( + bytesUnit = "bytes" + sentDescription = "Number of bytes sent by the component." + sentWireDescription = "Number of bytes sent on the wire by the component." + recvDescription = "Number of bytes received by the component." + recvWireDescription = "Number of bytes received on the wire by the component." + compSizeDescription = "Size of compressed payload" +) + +// makeSentMetrics builds the sent and sent-wire metric instruments +// for an exporter or receiver using the corresponding `prefix`. +// major` indicates the major direction of the pipeline, +// which is true when sending for exporters, receiving for receivers. +func makeSentMetrics(prefix string, meter metric.Meter, major bool) (sent, sentWire metric.Int64Counter, _ error) { + var sentBytes metric.Int64Counter = noopmetric.Int64Counter{} + var err1 error + if major { + sentBytes, err1 = meter.Int64Counter(prefix+"_"+SentBytes, metric.WithDescription(sentDescription), metric.WithUnit(bytesUnit)) + } + sentWireBytes, err2 := meter.Int64Counter(prefix+"_"+SentWireBytes, metric.WithDescription(sentWireDescription), metric.WithUnit(bytesUnit)) + return sentBytes, sentWireBytes, multierr.Append(err1, err2) +} + +// makeRecvMetrics builds the received and received-wire metric +// instruments for an exporter or receiver using the corresponding +// `prefix`. `major` indicates the major direction of the pipeline, +// which is true when sending for exporters, receiving for receivers. +func makeRecvMetrics(prefix string, meter metric.Meter, major bool) (recv, recvWire metric.Int64Counter, _ error) { + var recvBytes metric.Int64Counter = noopmetric.Int64Counter{} + var err1 error + if major { + recvBytes, err1 = meter.Int64Counter(prefix+"_"+RecvBytes, metric.WithDescription(recvDescription), metric.WithUnit(bytesUnit)) + } + recvWireBytes, err2 := meter.Int64Counter(prefix+"_"+RecvWireBytes, metric.WithDescription(recvWireDescription), metric.WithUnit(bytesUnit)) + return recvBytes, recvWireBytes, multierr.Append(err1, err2) +} + +// NewExporterNetworkReporter creates a new NetworkReporter configured for an exporter. +func NewExporterNetworkReporter(settings exporter.Settings) (*NetworkReporter, error) { + level := settings.TelemetrySettings.MetricsLevel + + if level <= configtelemetry.LevelBasic { + // Note: NetworkReporter implements nil a check. + return nil, nil + } + + meter := settings.TelemetrySettings.MeterProvider.Meter(scopeName) + rep := &NetworkReporter{ + isExporter: true, + staticAttr: attribute.String(ExporterKey, settings.ID.String()), + compSizeHisto: noopmetric.Int64Histogram{}, + } + + var errors, err error + if level > configtelemetry.LevelNormal { + rep.compSizeHisto, err = meter.Int64Histogram(ExporterKey+"_"+CompSize, metric.WithDescription(compSizeDescription), metric.WithUnit(bytesUnit)) + errors = multierr.Append(errors, err) + } + + rep.sentBytes, rep.sentWireBytes, err = makeSentMetrics(ExporterKey, meter, true) + errors = multierr.Append(errors, err) + + // Normally, an exporter counts sent bytes, and skips received + // bytes. LevelDetailed will reveal exporter-received bytes. + if level > configtelemetry.LevelNormal { + rep.recvBytes, rep.recvWireBytes, err = makeRecvMetrics(ExporterKey, meter, false) + errors = multierr.Append(errors, err) + } + + return rep, errors +} + +// NewReceiverNetworkReporter creates a new NetworkReporter configured for an exporter. +func NewReceiverNetworkReporter(settings receiver.Settings) (*NetworkReporter, error) { + level := settings.TelemetrySettings.MetricsLevel + + if level <= configtelemetry.LevelBasic { + // Note: NetworkReporter implements nil a check. + return nil, nil + } + + meter := settings.MeterProvider.Meter(scopeName) + rep := &NetworkReporter{ + isExporter: false, + staticAttr: attribute.String(ReceiverKey, settings.ID.String()), + compSizeHisto: noopmetric.Int64Histogram{}, + } + + var errors, err error + if level > configtelemetry.LevelNormal { + rep.compSizeHisto, err = meter.Int64Histogram(ReceiverKey+"_"+CompSize, metric.WithDescription(compSizeDescription), metric.WithUnit(bytesUnit)) + errors = multierr.Append(errors, err) + } + + rep.recvBytes, rep.recvWireBytes, err = makeRecvMetrics(ReceiverKey, meter, true) + errors = multierr.Append(errors, err) + + // Normally, a receiver counts received bytes, and skips sent + // bytes. LevelDetailed will reveal receiver-sent bytes. + if level > configtelemetry.LevelNormal { + rep.sentBytes, rep.sentWireBytes, err = makeSentMetrics(ReceiverKey, meter, false) + errors = multierr.Append(errors, err) + } + + return rep, errors +} + +// CountSend is used to report a message sent by the component. For +// exporters, SizesStruct indicates the size of a request. For +// receivers, SizesStruct indicates the size of a response. +func (rep *NetworkReporter) CountSend(ctx context.Context, ss SizesStruct) { + // Indicates basic level telemetry, not counting bytes. + if rep == nil { + return + } + + span := trace.SpanFromContext(ctx) + attrs := metric.WithAttributes(rep.staticAttr, attribute.String("method", ss.Method)) + + if ss.Length > 0 { + if rep.sentBytes != nil { + rep.sentBytes.Add(ctx, ss.Length, attrs) + } + if span.IsRecording() { + span.SetAttributes(attribute.Int64("sent_uncompressed", ss.Length)) + } + } + if ss.WireLength > 0 { + if rep.isExporter && rep.compSizeHisto != nil { + rep.compSizeHisto.Record(ctx, ss.WireLength, attrs) + } + if rep.sentWireBytes != nil { + rep.sentWireBytes.Add(ctx, ss.WireLength, attrs) + } + if span.IsRecording() { + span.SetAttributes(attribute.Int64("sent_compressed", ss.WireLength)) + } + } +} + +// CountReceive is used to report a message received by the component. For +// exporters, SizesStruct indicates the size of a response. For +// receivers, SizesStruct indicates the size of a request. +func (rep *NetworkReporter) CountReceive(ctx context.Context, ss SizesStruct) { + // Indicates basic level telemetry, not counting bytes. + if rep == nil { + return + } + + span := trace.SpanFromContext(ctx) + attrs := metric.WithAttributes(rep.staticAttr, attribute.String("method", ss.Method)) + + if ss.Length > 0 { + if rep.recvBytes != nil { + rep.recvBytes.Add(ctx, ss.Length, attrs) + } + if span.IsRecording() { + span.SetAttributes(attribute.Int64("received_uncompressed", ss.Length)) + } + } + if ss.WireLength > 0 { + if !rep.isExporter && rep.compSizeHisto != nil { + rep.compSizeHisto.Record(ctx, ss.WireLength, attrs) + } + if rep.recvWireBytes != nil { + rep.recvWireBytes.Add(ctx, ss.WireLength, attrs) + } + if span.IsRecording() { + span.SetAttributes(attribute.Int64("received_compressed", ss.WireLength)) + } + } +} diff --git a/internal/otelarrow/netstats/netstats_test.go b/internal/otelarrow/netstats/netstats_test.go new file mode 100644 index 000000000000..ec52e65c173f --- /dev/null +++ b/internal/otelarrow/netstats/netstats_test.go @@ -0,0 +1,326 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package netstats + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/configtelemetry" + "go.opentelemetry.io/collector/exporter" + "go.opentelemetry.io/collector/receiver" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "google.golang.org/grpc/stats" +) + +func metricValues(t *testing.T, rm metricdata.ResourceMetrics, expectMethod string) map[string]any { + res := map[string]any{} + for _, sm := range rm.ScopeMetrics { + for _, mm := range sm.Metrics { + var value int64 + var attrs attribute.Set + switch t := mm.Data.(type) { + case metricdata.Histogram[int64]: + for _, dp := range t.DataPoints { + value = dp.Sum // histogram tested as the sum + attrs = dp.Attributes + } + case metricdata.Sum[int64]: + for _, dp := range t.DataPoints { + value = dp.Value + attrs = dp.Attributes + } + } + var method string + for _, attr := range attrs.ToSlice() { + if attr.Key == "method" { + method = attr.Value.AsString() + } + } + + require.Equal(t, expectMethod, method) + res[mm.Name] = value + } + } + return res +} + +func TestNetStatsExporterNone(t *testing.T) { + testNetStatsExporter(t, configtelemetry.LevelNone, map[string]any{}) +} + +func TestNetStatsExporterNormal(t *testing.T) { + testNetStatsExporter(t, configtelemetry.LevelNormal, map[string]any{ + "exporter_sent": int64(1000), + "exporter_sent_wire": int64(100), + }) +} + +func TestNetStatsExporterDetailed(t *testing.T) { + testNetStatsExporter(t, configtelemetry.LevelDetailed, map[string]any{ + "exporter_sent": int64(1000), + "exporter_sent_wire": int64(100), + "exporter_recv_wire": int64(10), + "exporter_compressed_size": int64(100), // same as sent_wire b/c sum metricValue uses histogram sum + }) +} + +func testNetStatsExporter(t *testing.T, level configtelemetry.Level, expect map[string]any) { + for _, apiDirect := range []bool{true, false} { + t.Run(func() string { + if apiDirect { + return "direct" + } + return "grpc" + }(), func(t *testing.T) { + rdr := metric.NewManualReader() + mp := metric.NewMeterProvider( + metric.WithResource(resource.Empty()), + metric.WithReader(rdr), + ) + enr, err := NewExporterNetworkReporter(exporter.Settings{ + ID: component.NewID(component.MustNewType("test")), + TelemetrySettings: component.TelemetrySettings{ + MeterProvider: mp, + MetricsLevel: level, + }, + }) + require.NoError(t, err) + handler := enr.Handler() + + ctx := context.Background() + for i := 0; i < 10; i++ { + if apiDirect { + // use the direct API + enr.CountSend(ctx, SizesStruct{ + Method: "Hello", + Length: 100, + WireLength: 10, + }) + enr.CountReceive(ctx, SizesStruct{ + Method: "Hello", + Length: 10, + WireLength: 1, + }) + } else { + // simulate the RPC path + handler.HandleRPC(handler.TagRPC(ctx, &stats.RPCTagInfo{ + FullMethodName: "Hello", + }), &stats.OutPayload{ + Length: 100, + WireLength: 10, + }) + handler.HandleRPC(handler.TagRPC(ctx, &stats.RPCTagInfo{ + FullMethodName: "Hello", + }), &stats.InPayload{ + Length: 10, + WireLength: 1, + }) + } + } + var rm metricdata.ResourceMetrics + err = rdr.Collect(ctx, &rm) + require.NoError(t, err) + + require.Equal(t, expect, metricValues(t, rm, "Hello")) + }) + } +} + +func TestNetStatsSetSpanAttrs(t *testing.T) { + tests := []struct { + name string + attrs []attribute.KeyValue + isExporter bool + length int + wireLength int + }{ + { + name: "set exporter attributes", + isExporter: true, + length: 1234567, + wireLength: 123, + attrs: []attribute.KeyValue{ + attribute.Int("sent_uncompressed", 1234567), + attribute.Int("sent_compressed", 123), + attribute.Int("received_uncompressed", 1234567*2), + attribute.Int("received_compressed", 123*2), + }, + }, + { + name: "set receiver attributes", + isExporter: false, + length: 8901234, + wireLength: 890, + attrs: []attribute.KeyValue{ + attribute.Int("sent_uncompressed", 8901234), + attribute.Int("sent_compressed", 890), + attribute.Int("received_uncompressed", 8901234*2), + attribute.Int("received_compressed", 890*2), + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + enr := &NetworkReporter{ + isExporter: tc.isExporter, + } + + tp := sdktrace.NewTracerProvider() + ctx, sp := tp.Tracer("test/span").Start(context.Background(), "test-op") + + var sized SizesStruct + sized.Method = "test" + sized.Length = int64(tc.length) + sized.WireLength = int64(tc.wireLength) + enr.CountSend(ctx, sized) + sized.Length *= 2 + sized.WireLength *= 2 + enr.CountReceive(ctx, sized) + + actualAttrs := sp.(sdktrace.ReadOnlySpan).Attributes() + + require.Equal(t, attribute.NewSet(tc.attrs...), attribute.NewSet(actualAttrs...)) + }) + } +} + +func TestNetStatsReceiverNone(t *testing.T) { + testNetStatsReceiver(t, configtelemetry.LevelNone, map[string]any{}) +} + +func TestNetStatsReceiverNormal(t *testing.T) { + testNetStatsReceiver(t, configtelemetry.LevelNormal, map[string]any{ + "receiver_recv": int64(1000), + "receiver_recv_wire": int64(100), + }) +} + +func TestNetStatsReceiverDetailed(t *testing.T) { + testNetStatsReceiver(t, configtelemetry.LevelDetailed, map[string]any{ + "receiver_recv": int64(1000), + "receiver_recv_wire": int64(100), + "receiver_sent_wire": int64(10), + "receiver_compressed_size": int64(100), // same as recv_wire b/c sum metricValue uses histogram sum + }) +} + +func testNetStatsReceiver(t *testing.T, level configtelemetry.Level, expect map[string]any) { + for _, apiDirect := range []bool{true, false} { + t.Run(func() string { + if apiDirect { + return "direct" + } + return "grpc" + }(), func(t *testing.T) { + rdr := metric.NewManualReader() + mp := metric.NewMeterProvider( + metric.WithResource(resource.Empty()), + metric.WithReader(rdr), + ) + rer, err := NewReceiverNetworkReporter(receiver.Settings{ + ID: component.NewID(component.MustNewType("test")), + TelemetrySettings: component.TelemetrySettings{ + MeterProvider: mp, + MetricsLevel: level, + }, + }) + require.NoError(t, err) + handler := rer.Handler() + + ctx := context.Background() + for i := 0; i < 10; i++ { + if apiDirect { + // use the direct API + rer.CountReceive(ctx, SizesStruct{ + Method: "Hello", + Length: 100, + WireLength: 10, + }) + rer.CountSend(ctx, SizesStruct{ + Method: "Hello", + Length: 10, + WireLength: 1, + }) + } else { + // simulate the RPC path + handler.HandleRPC(handler.TagRPC(ctx, &stats.RPCTagInfo{ + FullMethodName: "Hello", + }), &stats.InPayload{ + Length: 100, + WireLength: 10, + }) + handler.HandleRPC(handler.TagRPC(ctx, &stats.RPCTagInfo{ + FullMethodName: "Hello", + }), &stats.OutPayload{ + Length: 10, + WireLength: 1, + }) + } + } + var rm metricdata.ResourceMetrics + err = rdr.Collect(ctx, &rm) + require.NoError(t, err) + + require.Equal(t, expect, metricValues(t, rm, "Hello")) + }) + } +} + +func TestUncompressedSizeBypass(t *testing.T) { + rdr := metric.NewManualReader() + mp := metric.NewMeterProvider( + metric.WithResource(resource.Empty()), + metric.WithReader(rdr), + ) + enr, err := NewExporterNetworkReporter(exporter.Settings{ + ID: component.NewID(component.MustNewType("test")), + TelemetrySettings: component.TelemetrySettings{ + MeterProvider: mp, + MetricsLevel: configtelemetry.LevelDetailed, + }, + }) + require.NoError(t, err) + handler := enr.Handler() + + ctx := context.Background() + for i := 0; i < 10; i++ { + // simulate the RPC path + handler.HandleRPC(handler.TagRPC(ctx, &stats.RPCTagInfo{ + FullMethodName: "my.arrow.v1.method", + }), &stats.OutPayload{ + Length: 9999, + WireLength: 10, + }) + handler.HandleRPC(handler.TagRPC(ctx, &stats.RPCTagInfo{ + FullMethodName: "my.arrow.v1.method", + }), &stats.InPayload{ + Length: 9999, + WireLength: 1, + }) + // There would bo no uncompressed size metric w/o this call + // and if the bypass didn't work, we would count the 9999s above. + enr.CountSend(ctx, SizesStruct{ + Method: "my.arrow.v1.method", + Length: 100, + }) + } + var rm metricdata.ResourceMetrics + err = rdr.Collect(ctx, &rm) + require.NoError(t, err) + + expect := map[string]any{ + "exporter_sent": int64(1000), + "exporter_sent_wire": int64(100), + "exporter_recv_wire": int64(10), + "exporter_compressed_size": int64(100), + } + require.Equal(t, expect, metricValues(t, rm, "my.arrow.v1.method")) +} diff --git a/internal/otelarrow/test/e2e_test.go b/internal/otelarrow/test/e2e_test.go new file mode 100644 index 000000000000..4b6ce785d257 --- /dev/null +++ b/internal/otelarrow/test/e2e_test.go @@ -0,0 +1,378 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "context" + "encoding/json" + "fmt" + "math/rand" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/open-telemetry/otel-arrow/pkg/datagen" + "github.com/open-telemetry/otel-arrow/pkg/otel/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/consumer/consumererror" + "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/exporter" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/ptrace" + "go.opentelemetry.io/collector/pdata/ptrace/ptraceotlp" + "go.opentelemetry.io/collector/receiver" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/otelarrowexporter" + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/otelarrow/testutil" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/otelarrowreceiver" +) + +type testParams struct { + threadCount int + requestCount int +} + +var normalParams = testParams{ + threadCount: 10, + requestCount: 100, +} + +var memoryLimitParams = testParams{ + threadCount: 10, + requestCount: 10, +} + +type testConsumer struct { + sink consumertest.TracesSink + recvLogs *observer.ObservedLogs + expLogs *observer.ObservedLogs +} + +var _ consumer.Traces = &testConsumer{} + +type ExpConfig = otelarrowexporter.Config +type RecvConfig = otelarrowreceiver.Config +type CfgFunc func(*ExpConfig, *RecvConfig) +type GenFunc func(int) ptrace.Traces +type MkGen func() GenFunc +type EndFunc func(t *testing.T, tp testParams, testCon *testConsumer, expect [][]ptrace.Traces) +type ConsumerErrFunc func(t *testing.T, err error) + +func (*testConsumer) Capabilities() consumer.Capabilities { + return consumer.Capabilities{} +} + +func (tc *testConsumer) ConsumeTraces(ctx context.Context, td ptrace.Traces) error { + time.Sleep(time.Duration(float64(time.Millisecond) * (1 + rand.Float64()))) + return tc.sink.ConsumeTraces(ctx, td) +} + +func testLoggerSettings(_ *testing.T) (component.TelemetrySettings, *observer.ObservedLogs) { + tset := componenttest.NewNopTelemetrySettings() + + core, obslogs := observer.New(zapcore.InfoLevel) + + // Note: if you want to see these logs in development, use: + // tset.Logger = zap.New(zapcore.NewTee(core, zaptest.NewLogger(t).Core())) + // Also see failureMemoryLimitEnding() for explicit tests based on the + // logs observer. + tset.Logger = zap.New(core) + + return tset, obslogs +} + +func basicTestConfig(t *testing.T, cfgF CfgFunc) (*testConsumer, exporter.Traces, receiver.Traces) { + ctx := context.Background() + + efact := otelarrowexporter.NewFactory() + rfact := otelarrowreceiver.NewFactory() + + ecfg := efact.CreateDefaultConfig() + rcfg := rfact.CreateDefaultConfig() + + receiverCfg := rcfg.(*RecvConfig) + exporterCfg := ecfg.(*ExpConfig) + + addr := testutil.GetAvailableLocalAddress(t) + + receiverCfg.Protocols.GRPC.NetAddr.Endpoint = addr + exporterCfg.ClientConfig.Endpoint = addr + exporterCfg.ClientConfig.WaitForReady = true + exporterCfg.ClientConfig.TLSSetting.Insecure = true + exporterCfg.TimeoutSettings.Timeout = time.Minute + exporterCfg.QueueSettings.Enabled = false + exporterCfg.RetryConfig.Enabled = false + exporterCfg.Arrow.NumStreams = 1 + + if cfgF != nil { + cfgF(exporterCfg, receiverCfg) + } + + expTset, expLogs := testLoggerSettings(t) + recvTset, recvLogs := testLoggerSettings(t) + + testCon := &testConsumer{ + recvLogs: recvLogs, + expLogs: expLogs, + } + + receiver, err := rfact.CreateTracesReceiver(ctx, receiver.Settings{ + ID: component.MustNewID("otelarrowreceiver"), + TelemetrySettings: recvTset, + }, receiverCfg, testCon) + require.NoError(t, err) + + exporter, err := efact.CreateTracesExporter(ctx, exporter.Settings{ + ID: component.MustNewID("otelarrowexporter"), + TelemetrySettings: expTset, + }, exporterCfg) + require.NoError(t, err) + + return testCon, exporter, receiver + +} + +func testIntegrationTraces(ctx context.Context, t *testing.T, tp testParams, cfgf CfgFunc, mkgen MkGen, errf ConsumerErrFunc, endf EndFunc) { + host := componenttest.NewNopHost() + + testCon, exporter, receiver := basicTestConfig(t, cfgf) + + var startWG sync.WaitGroup + var exporterShutdownWG sync.WaitGroup + var startExporterShutdownWG sync.WaitGroup + var receiverShutdownWG sync.WaitGroup // wait for receiver shutdown + + receiverShutdownWG.Add(1) + exporterShutdownWG.Add(1) + startExporterShutdownWG.Add(1) + startWG.Add(1) + + // Run the receiver, shutdown after exporter does. + go func() { + defer receiverShutdownWG.Done() + require.NoError(t, receiver.Start(ctx, host)) + exporterShutdownWG.Wait() + require.NoError(t, receiver.Shutdown(ctx)) + }() + + // Run the exporter and wait for clients to finish + go func() { + defer exporterShutdownWG.Done() + require.NoError(t, exporter.Start(ctx, host)) + startWG.Done() + startExporterShutdownWG.Wait() + require.NoError(t, exporter.Shutdown(ctx)) + }() + + // wait for the exporter to start + startWG.Wait() + var clientDoneWG sync.WaitGroup // wait for client to finish + + expect := make([][]ptrace.Traces, tp.threadCount) + + for num := 0; num < tp.threadCount; num++ { + clientDoneWG.Add(1) + go func(num int) { + defer clientDoneWG.Done() + generator := mkgen() + for i := 0; i < tp.requestCount; i++ { + td := generator(i) + + errf(t, exporter.ConsumeTraces(ctx, td)) + expect[num] = append(expect[num], td) + } + }(num) + } + + // wait til senders finish + clientDoneWG.Wait() + + // shut down exporter; it triggers receiver to shut down + startExporterShutdownWG.Done() + + // wait for receiver to shut down + receiverShutdownWG.Wait() + + endf(t, tp, testCon, expect) +} + +func makeTestTraces(i int) ptrace.Traces { + td := ptrace.NewTraces() + td.ResourceSpans().AppendEmpty().Resource().Attributes().PutStr("resource-attr", fmt.Sprint("resource-attr-val-", i)) + + ss := td.ResourceSpans().At(0).ScopeSpans().AppendEmpty().Spans() + span := ss.AppendEmpty() + + span.SetName("operationA") + span.SetStartTimestamp(pcommon.NewTimestampFromTime(time.Now())) + span.SetEndTimestamp(pcommon.NewTimestampFromTime(time.Now())) + + span.SetTraceID(testutil.UInt64ToTraceID(rand.Uint64(), rand.Uint64())) + span.SetSpanID(testutil.UInt64ToSpanID(rand.Uint64())) + evs := span.Events() + ev0 := evs.AppendEmpty() + ev0.SetTimestamp(pcommon.NewTimestampFromTime(time.Now())) + ev0.SetName("event-with-attr") + ev0.Attributes().PutStr("span-event-attr", "span-event-attr-val") + ev0.SetDroppedAttributesCount(2) + ev1 := evs.AppendEmpty() + ev1.SetTimestamp(pcommon.NewTimestampFromTime(time.Now())) + ev1.SetName("event") + ev1.SetDroppedAttributesCount(2) + span.SetDroppedEventsCount(1) + status := span.Status() + status.SetCode(ptrace.StatusCodeError) + status.SetMessage("status-cancelled") + + return td +} + +func bulkyGenFunc() MkGen { + return func() GenFunc { + entropy := datagen.NewTestEntropy(int64(rand.Uint64())) //nolint:gosec // only used for testing + + tracesGen := datagen.NewTracesGenerator( + entropy, + entropy.NewStandardResourceAttributes(), + entropy.NewStandardInstrumentationScopes(), + ) + return func(_ int) ptrace.Traces { + return tracesGen.Generate(1000, time.Minute) + } + } + +} + +func standardEnding(t *testing.T, tp testParams, testCon *testConsumer, expect [][]ptrace.Traces) { + // Check for matching request count and data + require.Equal(t, tp.requestCount*tp.threadCount, testCon.sink.SpanCount()) + + var expectJSON []json.Marshaler + for _, tdn := range expect { + for _, td := range tdn { + expectJSON = append(expectJSON, ptraceotlp.NewExportRequestFromTraces(td)) + } + } + var receivedJSON []json.Marshaler + + for _, td := range testCon.sink.AllTraces() { + receivedJSON = append(receivedJSON, ptraceotlp.NewExportRequestFromTraces(td)) + } + asserter := assert.NewStdUnitTest(t) + assert.Equiv(asserter, expectJSON, receivedJSON) +} + +// logSigs computes a signature of a structured log message emitted by +// the component via the Zap observer. The encoding is the message, +// "|||", followed by the field names in order, separated by "///". +// +// Specifically, we expect "arrow stream error" messages. When this is +// the case, the corresponding message is returned as a slice. +func logSigs(obs *observer.ObservedLogs) (map[string]int, []string) { + counts := map[string]int{} + var msgs []string + for _, rl := range obs.All() { + var attrs []string + for _, f := range rl.Context { + attrs = append(attrs, f.Key) + + if rl.Message == "arrow stream error" && f.Key == "message" { + msgs = append(msgs, f.String) + } + } + var sig strings.Builder + sig.WriteString(rl.Message) + sig.WriteString("|||") + sig.WriteString(strings.Join(attrs, "///")) + counts[sig.String()]++ + } + return counts, msgs +} + +var limitRegexp = regexp.MustCompile(`memory limit exceeded`) + +func countMemoryLimitErrors(msgs []string) (cnt int) { + for _, msg := range msgs { + if limitRegexp.MatchString(msg) { + cnt++ + } + } + return +} + +func failureMemoryLimitEnding(t *testing.T, _ testParams, testCon *testConsumer, _ [][]ptrace.Traces) { + require.Equal(t, 0, testCon.sink.SpanCount()) + + eSigs, eMsgs := logSigs(testCon.expLogs) + rSigs, rMsgs := logSigs(testCon.recvLogs) + + // Test for arrow stream errors. + + require.Less(t, 0, eSigs["arrow stream error|||code///message///where"], "should have exporter arrow stream errors: %v", eSigs) + require.Less(t, 0, rSigs["arrow stream error|||code///message///where"], "should have receiver arrow stream errors: %v", rSigs) + + // Ensure the errors include memory limit errors. + + require.Less(t, 0, countMemoryLimitErrors(rMsgs), "should have memory limit errors: %v", rMsgs) + require.Less(t, 0, countMemoryLimitErrors(eMsgs), "should have memory limit errors: %v", eMsgs) +} + +func consumerSuccess(t *testing.T, err error) { + require.NoError(t, err) +} + +func consumerFailure(t *testing.T, err error) { + require.Error(t, err) + + // there should be no permanent errors anywhere in this test. + require.True(t, !consumererror.IsPermanent(err), + "should not be permanent: %v", err) + + stat, ok := status.FromError(err) + require.True(t, ok, "should be a status error: %v", err) + + switch stat.Code() { + case codes.ResourceExhausted, codes.Canceled: + // Cool + default: + // Not cool + t.Fatalf("unexpected status code %v", stat) + } +} + +func TestIntegrationTracesSimple(t *testing.T) { + for _, n := range []int{1, 2, 4, 8} { + t.Run(fmt.Sprint(n), func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + testIntegrationTraces(ctx, t, normalParams, func(ecfg *ExpConfig, _ *RecvConfig) { + ecfg.Arrow.NumStreams = n + }, func() GenFunc { return makeTestTraces }, consumerSuccess, standardEnding) + }) + } +} + +func TestIntegrationMemoryLimited(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(5 * time.Second) + cancel() + }() + testIntegrationTraces(ctx, t, memoryLimitParams, func(ecfg *ExpConfig, rcfg *RecvConfig) { + rcfg.Arrow.MemoryLimitMiB = 1 + ecfg.Arrow.NumStreams = 10 + ecfg.TimeoutSettings.Timeout = 5 * time.Second + }, bulkyGenFunc(), consumerFailure, failureMemoryLimitEnding) +} diff --git a/internal/otelarrow/testdata/common.go b/internal/otelarrow/testdata/common.go new file mode 100644 index 000000000000..b62cf24724d4 --- /dev/null +++ b/internal/otelarrow/testdata/common.go @@ -0,0 +1,30 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package testdata + +import ( + "go.opentelemetry.io/collector/pdata/pcommon" +) + +func initMetricExemplarAttributes(dest pcommon.Map) { + dest.PutStr("exemplar-attachment", "exemplar-attachment-value") +} + +func initMetricAttributes1(dest pcommon.Map) { + dest.PutStr("label-1", "label-value-1") +} + +func initMetricAttributes2(dest pcommon.Map) { + dest.PutStr("label-2", "label-value-2") +} + +func initMetricAttributes12(dest pcommon.Map) { + initMetricAttributes1(dest) + initMetricAttributes2(dest) +} + +func initMetricAttributes13(dest pcommon.Map) { + initMetricAttributes1(dest) + dest.PutStr("label-3", "label-value-3") +} diff --git a/internal/otelarrow/testdata/log.go b/internal/otelarrow/testdata/log.go new file mode 100644 index 000000000000..f8c3237995ee --- /dev/null +++ b/internal/otelarrow/testdata/log.go @@ -0,0 +1,59 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package testdata + +import ( + "time" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" +) + +var ( + logTimestamp = pcommon.NewTimestampFromTime(time.Date(2020, 2, 11, 20, 26, 13, 789, time.UTC)) +) + +func GenerateLogs(count int) plog.Logs { + ld := plog.NewLogs() + initResource(ld.ResourceLogs().AppendEmpty().Resource()) + logs := ld.ResourceLogs().At(0).ScopeLogs().AppendEmpty().LogRecords() + logs.EnsureCapacity(count) + for i := 0; i < count; i++ { + switch i % 2 { + case 0: + fillLogTwo(logs.AppendEmpty()) + case 1: + fillLogOne(logs.AppendEmpty()) + } + } + return ld +} + +func fillLogOne(log plog.LogRecord) { + log.SetTimestamp(logTimestamp) + log.SetDroppedAttributesCount(1) + log.SetSeverityNumber(plog.SeverityNumberInfo) + log.SetSeverityText("Info") + log.SetSpanID([8]byte{0x01, 0x02, 0x04, 0x08}) + log.SetTraceID([16]byte{0x08, 0x04, 0x02, 0x01}) + + attrs := log.Attributes() + attrs.PutStr("app", "server") + attrs.PutInt("instance_num", 1) + + log.Body().SetStr("This is a log message") +} + +func fillLogTwo(log plog.LogRecord) { + log.SetTimestamp(logTimestamp + 1) + log.SetDroppedAttributesCount(1) + log.SetSeverityNumber(plog.SeverityNumberInfo) + log.SetSeverityText("Info") + + attrs := log.Attributes() + attrs.PutStr("customer", "acme") + attrs.PutStr("env", "dev") + + log.Body().SetStr("something happened") +} diff --git a/internal/otelarrow/testdata/metric.go b/internal/otelarrow/testdata/metric.go new file mode 100644 index 000000000000..26a5ab79ac56 --- /dev/null +++ b/internal/otelarrow/testdata/metric.go @@ -0,0 +1,293 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package testdata + +import ( + "time" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" +) + +var ( + metricStartTimestamp = pcommon.NewTimestampFromTime(time.Date(2020, 2, 11, 20, 26, 12, 321, time.UTC)) + metricExemplarTimestamp = pcommon.NewTimestampFromTime(time.Date(2020, 2, 11, 20, 26, 13, 123, time.UTC)) + metricTimestamp = pcommon.NewTimestampFromTime(time.Date(2020, 2, 11, 20, 26, 13, 789, time.UTC)) +) + +const ( + TestGaugeDoubleMetricName = "gauge-double" + TestGaugeIntMetricName = "gauge-int" + TestSumDoubleMetricName = "sum-double" + TestSumIntMetricName = "sum-int" + TestHistogramMetricName = "histogram" + TestExponentialHistogramMetricName = "exponential-histogram" + TestSummaryMetricName = "summary" +) + +func generateMetricsOneEmptyInstrumentationScope() pmetric.Metrics { + md := pmetric.NewMetrics() + initResource(md.ResourceMetrics().AppendEmpty().Resource()) + md.ResourceMetrics().At(0).ScopeMetrics().AppendEmpty() + return md +} + +func GenerateMetricsAllTypesEmpty() pmetric.Metrics { + md := generateMetricsOneEmptyInstrumentationScope() + ms := md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics() + + doubleGauge := ms.AppendEmpty() + initMetric(doubleGauge, TestGaugeDoubleMetricName, pmetric.MetricTypeGauge) + doubleGauge.Gauge().DataPoints().AppendEmpty() + intGauge := ms.AppendEmpty() + initMetric(intGauge, TestGaugeIntMetricName, pmetric.MetricTypeGauge) + intGauge.Gauge().DataPoints().AppendEmpty() + doubleSum := ms.AppendEmpty() + initMetric(doubleSum, TestSumDoubleMetricName, pmetric.MetricTypeSum) + doubleSum.Sum().DataPoints().AppendEmpty() + intSum := ms.AppendEmpty() + initMetric(intSum, TestSumIntMetricName, pmetric.MetricTypeSum) + intSum.Sum().DataPoints().AppendEmpty() + histogram := ms.AppendEmpty() + initMetric(histogram, TestHistogramMetricName, pmetric.MetricTypeHistogram) + histogram.Histogram().DataPoints().AppendEmpty() + summary := ms.AppendEmpty() + initMetric(summary, TestSummaryMetricName, pmetric.MetricTypeSummary) + summary.Summary().DataPoints().AppendEmpty() + return md +} + +func GenerateMetricsMetricTypeInvalid() pmetric.Metrics { + md := generateMetricsOneEmptyInstrumentationScope() + initMetric(md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().AppendEmpty(), TestSumIntMetricName, pmetric.MetricTypeEmpty) + return md +} + +func GenerateMetricsAllTypes() pmetric.Metrics { + md := generateMetricsOneEmptyInstrumentationScope() + ms := md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics() + initGaugeIntMetric(ms.AppendEmpty()) + initGaugeDoubleMetric(ms.AppendEmpty()) + initSumIntMetric(ms.AppendEmpty()) + initSumDoubleMetric(ms.AppendEmpty()) + initHistogramMetric(ms.AppendEmpty()) + initExponentialHistogramMetric(ms.AppendEmpty()) + initSummaryMetric(ms.AppendEmpty()) + return md +} + +func GenerateMetrics(count int) pmetric.Metrics { + md := generateMetricsOneEmptyInstrumentationScope() + ms := md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics() + ms.EnsureCapacity(count) + for i := 0; i < count; i++ { + switch i % 7 { + case 0: + initGaugeIntMetric(ms.AppendEmpty()) + case 1: + initGaugeDoubleMetric(ms.AppendEmpty()) + case 2: + initSumIntMetric(ms.AppendEmpty()) + case 3: + initSumDoubleMetric(ms.AppendEmpty()) + case 4: + initHistogramMetric(ms.AppendEmpty()) + case 5: + initExponentialHistogramMetric(ms.AppendEmpty()) + case 6: + initSummaryMetric(ms.AppendEmpty()) + } + } + return md +} + +func initGaugeIntMetric(im pmetric.Metric) { + initMetric(im, TestGaugeIntMetricName, pmetric.MetricTypeGauge) + + idps := im.Gauge().DataPoints() + idp0 := idps.AppendEmpty() + initMetricAttributes1(idp0.Attributes()) + idp0.SetStartTimestamp(metricStartTimestamp) + idp0.SetTimestamp(metricTimestamp) + idp0.SetIntValue(123) + idp1 := idps.AppendEmpty() + initMetricAttributes2(idp1.Attributes()) + idp1.SetStartTimestamp(metricStartTimestamp) + idp1.SetTimestamp(metricTimestamp) + idp1.SetIntValue(456) +} + +func initGaugeDoubleMetric(im pmetric.Metric) { + initMetric(im, TestGaugeDoubleMetricName, pmetric.MetricTypeGauge) + + idps := im.Gauge().DataPoints() + idp0 := idps.AppendEmpty() + initMetricAttributes12(idp0.Attributes()) + idp0.SetStartTimestamp(metricStartTimestamp) + idp0.SetTimestamp(metricTimestamp) + idp0.SetDoubleValue(1.23) + idp1 := idps.AppendEmpty() + initMetricAttributes13(idp1.Attributes()) + idp1.SetStartTimestamp(metricStartTimestamp) + idp1.SetTimestamp(metricTimestamp) + idp1.SetDoubleValue(4.56) +} + +func initSumIntMetric(im pmetric.Metric) { + initMetric(im, TestSumIntMetricName, pmetric.MetricTypeSum) + + idps := im.Sum().DataPoints() + idp0 := idps.AppendEmpty() + initMetricAttributes1(idp0.Attributes()) + idp0.SetStartTimestamp(metricStartTimestamp) + idp0.SetTimestamp(metricTimestamp) + idp0.SetIntValue(123) + idp1 := idps.AppendEmpty() + initMetricAttributes2(idp1.Attributes()) + idp1.SetStartTimestamp(metricStartTimestamp) + idp1.SetTimestamp(metricTimestamp) + idp1.SetIntValue(456) +} + +func initSumDoubleMetric(dm pmetric.Metric) { + initMetric(dm, TestSumDoubleMetricName, pmetric.MetricTypeSum) + + ddps := dm.Sum().DataPoints() + ddp0 := ddps.AppendEmpty() + initMetricAttributes12(ddp0.Attributes()) + ddp0.SetStartTimestamp(metricStartTimestamp) + ddp0.SetTimestamp(metricTimestamp) + ddp0.SetDoubleValue(1.23) + + ddp1 := ddps.AppendEmpty() + initMetricAttributes13(ddp1.Attributes()) + ddp1.SetStartTimestamp(metricStartTimestamp) + ddp1.SetTimestamp(metricTimestamp) + ddp1.SetDoubleValue(4.56) +} + +func initHistogramMetric(hm pmetric.Metric) { + initMetric(hm, TestHistogramMetricName, pmetric.MetricTypeHistogram) + + hdps := hm.Histogram().DataPoints() + hdp0 := hdps.AppendEmpty() + initMetricAttributes13(hdp0.Attributes()) + hdp0.SetStartTimestamp(metricStartTimestamp) + hdp0.SetTimestamp(metricTimestamp) + hdp0.SetCount(1) + hdp0.SetSum(15) + + hdp1 := hdps.AppendEmpty() + initMetricAttributes2(hdp1.Attributes()) + hdp1.SetStartTimestamp(metricStartTimestamp) + hdp1.SetTimestamp(metricTimestamp) + hdp1.SetCount(1) + hdp1.SetSum(15) + hdp1.SetMin(15) + hdp1.SetMax(15) + hdp1.BucketCounts().FromRaw([]uint64{0, 1}) + exemplar := hdp1.Exemplars().AppendEmpty() + exemplar.SetTimestamp(metricExemplarTimestamp) + exemplar.SetDoubleValue(15) + initMetricExemplarAttributes(exemplar.FilteredAttributes()) + hdp1.ExplicitBounds().FromRaw([]float64{1}) +} + +func initExponentialHistogramMetric(hm pmetric.Metric) { + initMetric(hm, TestExponentialHistogramMetricName, pmetric.MetricTypeExponentialHistogram) + + hdps := hm.ExponentialHistogram().DataPoints() + hdp0 := hdps.AppendEmpty() + initMetricAttributes13(hdp0.Attributes()) + hdp0.SetStartTimestamp(metricStartTimestamp) + hdp0.SetTimestamp(metricTimestamp) + hdp0.SetCount(5) + hdp0.SetSum(0.15) + hdp0.SetZeroCount(1) + hdp0.SetScale(1) + + // positive index 1 and 2 are values sqrt(2), 2 at scale 1 + hdp0.Positive().SetOffset(1) + hdp0.Positive().BucketCounts().FromRaw([]uint64{1, 1}) + // negative index -1 and 0 are values -1/sqrt(2), -1 at scale 1 + hdp0.Negative().SetOffset(-1) + hdp0.Negative().BucketCounts().FromRaw([]uint64{1, 1}) + + // The above will print: + // Bucket (-1.414214, -1.000000], Count: 1 + // Bucket (-1.000000, -0.707107], Count: 1 + // Bucket [0, 0], Count: 1 + // Bucket [0.707107, 1.000000), Count: 1 + // Bucket [1.000000, 1.414214), Count: 1 + + hdp1 := hdps.AppendEmpty() + initMetricAttributes2(hdp1.Attributes()) + hdp1.SetStartTimestamp(metricStartTimestamp) + hdp1.SetTimestamp(metricTimestamp) + hdp1.SetCount(3) + hdp1.SetSum(1.25) + hdp1.SetMin(0) + hdp1.SetMax(1) + hdp1.SetZeroCount(1) + hdp1.SetScale(-1) + + // index -1 and 0 are values 0.25, 1 at scale -1 + hdp1.Positive().SetOffset(-1) + hdp1.Positive().BucketCounts().FromRaw([]uint64{1, 1}) + + // The above will print: + // Bucket [0, 0], Count: 1 + // Bucket [0.250000, 1.000000), Count: 1 + // Bucket [1.000000, 4.000000), Count: 1 + + exemplar := hdp1.Exemplars().AppendEmpty() + exemplar.SetTimestamp(metricExemplarTimestamp) + exemplar.SetDoubleValue(15) + initMetricExemplarAttributes(exemplar.FilteredAttributes()) +} + +func initSummaryMetric(sm pmetric.Metric) { + initMetric(sm, TestSummaryMetricName, pmetric.MetricTypeSummary) + + sdps := sm.Summary().DataPoints() + sdp0 := sdps.AppendEmpty() + initMetricAttributes13(sdp0.Attributes()) + sdp0.SetStartTimestamp(metricStartTimestamp) + sdp0.SetTimestamp(metricTimestamp) + sdp0.SetCount(1) + sdp0.SetSum(15) + + sdp1 := sdps.AppendEmpty() + initMetricAttributes2(sdp1.Attributes()) + sdp1.SetStartTimestamp(metricStartTimestamp) + sdp1.SetTimestamp(metricTimestamp) + sdp1.SetCount(1) + sdp1.SetSum(15) + + quantile := sdp1.QuantileValues().AppendEmpty() + quantile.SetQuantile(0.01) + quantile.SetValue(15) +} + +func initMetric(m pmetric.Metric, name string, ty pmetric.MetricType) { + m.SetName(name) + m.SetDescription("") + m.SetUnit("1") + switch ty { + case pmetric.MetricTypeGauge: + m.SetEmptyGauge() + case pmetric.MetricTypeSum: + sum := m.SetEmptySum() + sum.SetIsMonotonic(true) + sum.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + case pmetric.MetricTypeHistogram: + histo := m.SetEmptyHistogram() + histo.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + case pmetric.MetricTypeExponentialHistogram: + histo := m.SetEmptyExponentialHistogram() + histo.SetAggregationTemporality(pmetric.AggregationTemporalityDelta) + case pmetric.MetricTypeSummary: + m.SetEmptySummary() + } +} diff --git a/internal/otelarrow/testdata/resource.go b/internal/otelarrow/testdata/resource.go new file mode 100644 index 000000000000..cac7a4046021 --- /dev/null +++ b/internal/otelarrow/testdata/resource.go @@ -0,0 +1,10 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package testdata + +import "go.opentelemetry.io/collector/pdata/pcommon" + +func initResource(r pcommon.Resource) { + r.Attributes().PutStr("resource-attr", "resource-attr-val-1") +} diff --git a/internal/otelarrow/testdata/trace.go b/internal/otelarrow/testdata/trace.go new file mode 100644 index 000000000000..c4fd6e5e51f4 --- /dev/null +++ b/internal/otelarrow/testdata/trace.go @@ -0,0 +1,71 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package testdata + +import ( + "time" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/ptrace" +) + +var ( + spanStartTimestamp = pcommon.NewTimestampFromTime(time.Date(2020, 2, 11, 20, 26, 12, 321, time.UTC)) + spanEventTimestamp = pcommon.NewTimestampFromTime(time.Date(2020, 2, 11, 20, 26, 13, 123, time.UTC)) + spanEndTimestamp = pcommon.NewTimestampFromTime(time.Date(2020, 2, 11, 20, 26, 13, 789, time.UTC)) +) + +func GenerateTraces(spanCount int) ptrace.Traces { + td := ptrace.NewTraces() + initResource(td.ResourceSpans().AppendEmpty().Resource()) + ss := td.ResourceSpans().At(0).ScopeSpans().AppendEmpty().Spans() + ss.EnsureCapacity(spanCount) + for i := 0; i < spanCount; i++ { + switch i % 2 { + case 0: + fillSpanOne(ss.AppendEmpty()) + case 1: + fillSpanTwo(ss.AppendEmpty()) + } + } + return td +} + +func fillSpanOne(span ptrace.Span) { + span.SetName("operationA") + span.SetStartTimestamp(spanStartTimestamp) + span.SetEndTimestamp(spanEndTimestamp) + span.SetDroppedAttributesCount(1) + span.SetTraceID([16]byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10}) + span.SetSpanID([8]byte{0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}) + evs := span.Events() + + ev0 := evs.AppendEmpty() + ev0.SetTimestamp(spanEventTimestamp) + ev0.SetName("event") + ev0.SetDroppedAttributesCount(2) + + ev1 := evs.AppendEmpty() + ev1.SetTimestamp(spanEventTimestamp) + ev1.SetName("event-with-attr") + ev1.Attributes().PutStr("span-event-attr", "span-event-attr-val") + ev1.SetDroppedAttributesCount(2) + + span.SetDroppedEventsCount(1) + status := span.Status() + status.SetCode(ptrace.StatusCodeError) + status.SetMessage("status-cancelled") +} + +func fillSpanTwo(span ptrace.Span) { + span.SetName("operationB") + span.SetStartTimestamp(spanStartTimestamp) + span.SetEndTimestamp(spanEndTimestamp) + link0 := span.Links().AppendEmpty() + link0.Attributes().PutStr("span-link-attr", "span-link-attr-val") + link0.SetDroppedAttributesCount(4) + link1 := span.Links().AppendEmpty() + link1.SetDroppedAttributesCount(4) + span.SetDroppedLinksCount(3) +} diff --git a/internal/otelarrow/testutil/testutil.go b/internal/otelarrow/testutil/testutil.go new file mode 100644 index 000000000000..1f75806356e6 --- /dev/null +++ b/internal/otelarrow/testutil/testutil.go @@ -0,0 +1,117 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package testutil // import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/otelarrow/testutil" + +import ( + "encoding/binary" + "net" + "os/exec" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/pcommon" +) + +type portpair struct { + first string + last string +} + +// GetAvailableLocalAddress finds an available local port and returns an endpoint +// describing it. The port is available for opening when this function returns +// provided that there is no race by some other code to grab the same port +// immediately. +func GetAvailableLocalAddress(t testing.TB) string { + // Retry has been added for windows as net.Listen can return a port that is not actually available. Details can be + // found in https://github.com/docker/for-win/issues/3171 but to summarize Hyper-V will reserve ranges of ports + // which do not show up under the "netstat -ano" but can only be found by + // "netsh interface ipv4 show excludedportrange protocol=tcp". We'll use []exclusions to hold those ranges and + // retry if the port returned by GetAvailableLocalAddress falls in one of those them. + var exclusions []portpair + portFound := false + if runtime.GOOS == "windows" { + exclusions = getExclusionsList(t) + } + + var endpoint string + for !portFound { + endpoint = findAvailableAddress(t) + _, port, err := net.SplitHostPort(endpoint) + require.NoError(t, err) + portFound = true + if runtime.GOOS == "windows" { + for _, pair := range exclusions { + if port >= pair.first && port <= pair.last { + portFound = false + break + } + } + } + } + + return endpoint +} + +func findAvailableAddress(t testing.TB) string { + ln, err := net.Listen("tcp", "localhost:0") + require.NoError(t, err, "Failed to get a free local port") + // There is a possible race if something else takes this same port before + // the test uses it, however, that is unlikely in practice. + defer func() { + assert.NoError(t, ln.Close()) + }() + return ln.Addr().String() +} + +// Get excluded ports on Windows from the command: netsh interface ipv4 show excludedportrange protocol=tcp +func getExclusionsList(t testing.TB) []portpair { + cmdTCP := exec.Command("netsh", "interface", "ipv4", "show", "excludedportrange", "protocol=tcp") + outputTCP, errTCP := cmdTCP.CombinedOutput() + require.NoError(t, errTCP) + exclusions := createExclusionsList(string(outputTCP), t) + + cmdUDP := exec.Command("netsh", "interface", "ipv4", "show", "excludedportrange", "protocol=udp") + outputUDP, errUDP := cmdUDP.CombinedOutput() + require.NoError(t, errUDP) + exclusions = append(exclusions, createExclusionsList(string(outputUDP), t)...) + + return exclusions +} + +func createExclusionsList(exclusionsText string, t testing.TB) []portpair { + var exclusions []portpair + + parts := strings.Split(exclusionsText, "--------") + require.Equal(t, len(parts), 3) + portsText := strings.Split(parts[2], "*") + require.Greater(t, len(portsText), 1) // original text may have a suffix like " - Administered port exclusions." + lines := strings.Split(portsText[0], "\n") + for _, line := range lines { + if strings.TrimSpace(line) != "" { + entries := strings.Fields(strings.TrimSpace(line)) + require.Equal(t, len(entries), 2) + pair := portpair{entries[0], entries[1]} + exclusions = append(exclusions, pair) + } + } + return exclusions +} + +// UInt64ToTraceID is from collector-contrib/internal/idutils +func UInt64ToTraceID(high, low uint64) pcommon.TraceID { + traceID := [16]byte{} + binary.BigEndian.PutUint64(traceID[:8], high) + binary.BigEndian.PutUint64(traceID[8:], low) + return traceID +} + +// UInt64ToSpanID is from collector-contrib/internal/idutils +func UInt64ToSpanID(id uint64) pcommon.SpanID { + spanID := [8]byte{} + binary.BigEndian.PutUint64(spanID[:8], id) + return spanID +} diff --git a/internal/otelarrow/testutil/testutil_test.go b/internal/otelarrow/testutil/testutil_test.go new file mode 100644 index 000000000000..05b38a1de6ac --- /dev/null +++ b/internal/otelarrow/testutil/testutil_test.go @@ -0,0 +1,57 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetAvailableLocalAddress(t *testing.T) { + endpoint := GetAvailableLocalAddress(t) + + // Endpoint should be free. + ln0, err := net.Listen("tcp", endpoint) + require.NoError(t, err) + require.NotNil(t, ln0) + t.Cleanup(func() { + assert.NoError(t, ln0.Close()) + }) + + // Ensure that the endpoint wasn't something like ":0" by checking that a + // second listener will fail. + ln1, err := net.Listen("tcp", endpoint) + require.Error(t, err) + require.Nil(t, ln1) +} + +func TestCreateExclusionsList(t *testing.T) { + // Test two examples of typical output from "netsh interface ipv4 show excludedportrange protocol=tcp" + emptyExclusionsText := ` + +Protocol tcp Port Exclusion Ranges + +Start Port End Port +---------- -------- + +* - Administered port exclusions.` + + exclusionsText := ` + +Start Port End Port +---------- -------- + 49697 49796 + 49797 49896 + +* - Administered port exclusions. +` + exclusions := createExclusionsList(exclusionsText, t) + require.Equal(t, len(exclusions), 2) + + emptyExclusions := createExclusionsList(emptyExclusionsText, t) + require.Equal(t, len(emptyExclusions), 0) +} diff --git a/versions.yaml b/versions.yaml index 2e56cdfd38a9..30ee09b2f42e 100644 --- a/versions.yaml +++ b/versions.yaml @@ -129,6 +129,7 @@ module-sets: - github.com/open-telemetry/opentelemetry-collector-contrib/internal/kubelet - github.com/open-telemetry/opentelemetry-collector-contrib/internal/metadataproviders - github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil + - github.com/open-telemetry/opentelemetry-collector-contrib/internal/otelarrow - github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent - github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk - github.com/open-telemetry/opentelemetry-collector-contrib/internal/sqlquery