Add integration test templates (#32)

* Add integration test templates

* Avoid bytes reader allocation

.github/workflows/test.yml

@@ -0,0 +1,32 @@
+name: IntegrationTest
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.20'
+
+      - uses: actions/checkout@v3
+      - name: Start containers
+        run: make compose
+
+      - uses: actions/checkout@v3
+      - name: Integration Test
+        run: go test -v test/integration/integration_test.go
+        env:
+          INPUT_PUBLISH: false
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Makefile

@@ -17,3 +17,6 @@ linter:
 
 run:
 	go run main.go
+
+compose:
+	docker compose up --wait --build --force-recreate --remove-orphans

docker-compose.yml

@@ -0,0 +1,19 @@
+version: "3.8"
+services:
+  couchbase:
+    build:
+      context: "test/couchbase"
+    ports:
+      - "8091:8091"
+      - "11210:11210"
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://user:123456@localhost:8091/pools/default/buckets/dcp-test" ]
+      interval: 2s
+      timeout: 3s
+      retries: 60
+  elasticsearch:
+    build:
+      context: "test/elasticsearch"
+    ports:
+      - "9200:9200"
+      - "9300:9300"

elasticsearch/bulk/bulk.go

@@ -21,6 +21,7 @@ import (
 )
 
 type Bulk struct {
+	reader                 *bytes.Reader
 	errorLogger            logger.Logger
 	logger                 logger.Logger
 	dcpCheckpointCommit    func()
@@ -70,6 +71,7 @@ func NewBulk(
 		metric:                 &Metric{},
 		collectionIndexMapping: config.Elasticsearch.CollectionIndexMapping,
 		typeName:               helper.Byte(config.Elasticsearch.TypeName),
+		reader:                 bytes.NewReader(nil),
 	}
 	return bulk, nil
 }
@@ -196,8 +198,8 @@ func (b *Bulk) flushMessages() {
 
 func (b *Bulk) bulkRequest() error {
 	startedTime := time.Now()
-	reader := bytes.NewReader(b.batch)
-	r, err := b.esClient.Bulk(reader)
+	b.reader.Reset(b.batch)
+	r, err := b.esClient.Bulk(b.reader)
 	b.metric.BulkRequestProcessLatencyMs = time.Since(startedTime).Milliseconds()
 	if err != nil {
 		return err

elasticsearch/client/client.go

@@ -10,11 +10,10 @@ import (
 
 func NewElasticClient(config *config.Config) (*elasticsearch.Client, error) {
 	es, err := elasticsearch.NewClient(elasticsearch.Config{
-		MaxRetries:           math.MaxInt,
-		Addresses:            config.Elasticsearch.Urls,
-		DiscoverNodesOnStart: true,
-		Transport:            newTransport(config.Elasticsearch),
-		CompressRequestBody:  config.Elasticsearch.CompressionEnabled,
+		MaxRetries:          math.MaxInt,
+		Addresses:           config.Elasticsearch.Urls,
+		Transport:           newTransport(config.Elasticsearch),
+		CompressRequestBody: config.Elasticsearch.CompressionEnabled,
 	})
 	if err != nil {
 		return nil, err

test/couchbase/Dockerfile

@@ -0,0 +1,10 @@
+#FROM couchbase/server:community-7.1.0-aarch64
+FROM couchbase:community-7.1.0
+
+ADD configure.sh /configure.sh
+RUN chmod +x /configure.sh
+RUN echo " image starting"
+
+EXPOSE 8091 8092 8093 8094 8095 8096 11207 11210 11211
+
+CMD ["/configure.sh"]

test/couchbase/configure.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Enables job control
+set -m
+
+# Enables error propagation
+set -e
+
+# Run the server and send it to the background
+/entrypoint.sh couchbase-server &
+
+# Check if couchbase server is up
+check_db() {
+  curl --silent http://127.0.0.1:8091/pools > /dev/null
+  echo $?
+}
+
+# Variable used in echo
+i=1
+# Echo with
+log() {
+  echo "[$i] [$(date +"%T")] $@"
+  i=`expr $i + 1`
+}
+
+# Wait until it's ready
+until [[ $(check_db) = 0 ]]; do
+  >&2 log "Waiting for Couchbase Server to be available ..."
+  sleep 1
+done
+
+couchbase-cli cluster-init -c localhost --cluster-name Cluster --cluster-username user \
+  --cluster-password 123456 --services data --cluster-ramsize 1024
+
+couchbase-cli bucket-create -c couchbase --username user --password 123456 --bucket dcp-test --bucket-type couchbase --bucket-ramsize 1024
+
+cbimport json -c couchbase://127.0.0.1 -u user -p 123456 --bucket-quota 1024 -b dcp-test -d file://opt/couchbase/samples/travel-sample.zip -f sample
+
+echo "couchbase-dev started"
+
+fg 1

test/elasticsearch/Dockerfile

@@ -0,0 +1,165 @@
+################################################################################
+# This Dockerfile was generated from the template at distribution/src/docker/Dockerfile
+#
+# Beginning of multi stage Dockerfile
+################################################################################
+
+################################################################################
+# Build stage 0 `builder`:
+# Extract Elasticsearch artifact
+################################################################################
+FROM ubuntu:20.04 AS builder
+
+# Install required packages to extract the Elasticsearch distribution
+
+RUN for iter in 1 2 3 4 5 6 7 8 9 10; do \
+      apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl  && \
+      exit_code=0 && break || \
+        exit_code=$? && echo "apt-get error: retry $iter in 10s" && sleep 10; \
+    done; \
+    exit $exit_code
+
+# `tini` is a tiny but valid init for containers. This is used to cleanly
+# control how ES and any child processes are shut down.
+#
+# The tini GitHub page gives instructions for verifying the binary using
+# gpg, but the keyservers are slow to return the key and this can fail the
+# build. Instead, we check the binary against the published checksum.
+RUN set -eux ; \
+    tini_bin="" ; \
+    case "$(arch)" in \
+        aarch64) tini_bin='tini-arm64' ;; \
+        x86_64)  tini_bin='tini-amd64' ;; \
+        *) echo >&2 ; echo >&2 "Unsupported architecture $(arch)" ; echo >&2 ; exit 1 ;; \
+    esac ; \
+    curl --retry 10 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/${tini_bin} ; \
+    curl --retry 10 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/${tini_bin}.sha256sum ; \
+    sha256sum -c ${tini_bin}.sha256sum ; \
+    rm ${tini_bin}.sha256sum ; \
+    mv ${tini_bin} /bin/tini ; \
+    chmod 0555 /bin/tini
+
+RUN mkdir /usr/share/elasticsearch
+WORKDIR /usr/share/elasticsearch
+
+RUN curl --retry 10 -S -L --output /tmp/elasticsearch.tar.gz https://artifacts-no-kpi.elastic.co/downloads/elasticsearch/elasticsearch-7.17.12-linux-$(arch).tar.gz
+
+RUN tar -zxf /tmp/elasticsearch.tar.gz --strip-components=1
+
+# The distribution includes a `config` directory, no need to create it
+COPY config/elasticsearch.yml config/
+COPY config/log4j2.properties config/log4j2.docker.properties
+
+#  1. Configure the distribution for Docker
+#  2. Create required directory
+#  3. Move the distribution's default logging config aside
+#  4. Move the generated docker logging config so that it is the default
+#  5. Reset permissions on all directories
+#  6. Reset permissions on all files
+#  7. Make CLI tools executable
+#  8. Make some directories writable. `bin` must be writable because
+#     plugins can install their own CLI utilities.
+#  9. Make some files writable
+RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elasticsearch-env && \
+    mkdir data && \
+    mv config/log4j2.properties config/log4j2.file.properties && \
+    mv config/log4j2.docker.properties config/log4j2.properties && \
+    find . -type d -exec chmod 0555 {} + && \
+    find . -type f -exec chmod 0444 {} + && \
+    chmod 0555 bin/* jdk/bin/* jdk/lib/jspawnhelper modules/x-pack-ml/platform/linux-*/bin/* && \
+    chmod 0775 bin config config/jvm.options.d data logs plugins && \
+    find config -type f -exec chmod 0664 {} +
+
+################################################################################
+# Build stage 1 (the actual Elasticsearch image):
+#
+# Copy elasticsearch from stage 0
+# Add entrypoint
+################################################################################
+
+FROM ubuntu:20.04
+
+# Change default shell to bash, then install required packages with retries.
+RUN yes no | dpkg-reconfigure dash && \
+    for iter in 1 2 3 4 5 6 7 8 9 10; do \
+      export DEBIAN_FRONTEND=noninteractive && \
+      apt-get update && \
+      apt-get upgrade -y && \
+      apt-get install -y --no-install-recommends \
+        ca-certificates curl netcat p11-kit unzip zip && \
+      apt-get clean && \
+      rm -rf /var/lib/apt/lists/* && \
+      exit_code=0 && break || \
+        exit_code=$? && echo "apt-get error: retry $iter in 10s" && sleep 10; \
+    done; \
+    exit $exit_code
+
+RUN groupadd -g 1000 elasticsearch && \
+    adduser --uid 1000 --gid 1000 --home /usr/share/elasticsearch elasticsearch && \
+    adduser elasticsearch root && \
+    chown -R 0:0 /usr/share/elasticsearch
+
+ENV ELASTIC_CONTAINER true
+
+WORKDIR /usr/share/elasticsearch
+
+COPY --from=builder --chown=0:0 /usr/share/elasticsearch /usr/share/elasticsearch
+COPY --from=builder --chown=0:0 /bin/tini /bin/tini
+
+ENV PATH /usr/share/elasticsearch/bin:$PATH
+
+COPY bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+
+# 1. Sync the user and group permissions of /etc/passwd
+# 2. Set correct permissions of the entrypoint
+# 3. Ensure that there are no files with setuid or setgid, in order to mitigate "stackclash" attacks.
+#    We've already run this in previous layers so it ought to be a no-op.
+# 4. Replace OpenJDK's built-in CA certificate keystore with the one from the OS
+#    vendor. The latter is superior in several ways.
+#    REF: https://github.com/elastic/elasticsearch-docker/issues/171
+# 5. Tighten up permissions on the ES home dir (the permissions of the contents are handled earlier)
+# 6. You can't install plugins that include configuration when running as `elasticsearch` and the `config`
+#    dir is owned by `root`, because the installed tries to manipulate the permissions on the plugin's
+#    config directory.
+RUN chmod g=u /etc/passwd && \
+    chmod 0555 /usr/local/bin/docker-entrypoint.sh && \
+    find / -xdev -perm -4000 -exec chmod ug-s {} + && \
+    chmod 0775 /usr/share/elasticsearch && \
+    chown elasticsearch bin config config/jvm.options.d data logs plugins
+
+# Update "cacerts" bundle to use Ubuntu's CA certificates (and make sure it
+# stays up-to-date with changes to Ubuntu's store)
+COPY bin/docker-openjdk /etc/ca-certificates/update.d/docker-openjdk
+RUN /etc/ca-certificates/update.d/docker-openjdk
+
+EXPOSE 9200 9300
+
+LABEL org.label-schema.build-date="2023-07-20T05:33:33.690180787Z" \
+  org.label-schema.license="Elastic-License-2.0" \
+  org.label-schema.name="Elasticsearch" \
+  org.label-schema.schema-version="1.0" \
+  org.label-schema.url="https://www.elastic.co/products/elasticsearch" \
+  org.label-schema.usage="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \
+  org.label-schema.vcs-ref="e3b0c3d3c5c130e1dc6d567d6baef1c73eeb2059" \
+  org.label-schema.vcs-url="https://github.com/elastic/elasticsearch" \
+  org.label-schema.vendor="Elastic" \
+  org.label-schema.version="7.17.12" \
+  org.opencontainers.image.created="2023-07-20T05:33:33.690180787Z" \
+  org.opencontainers.image.documentation="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \
+  org.opencontainers.image.licenses="Elastic-License-2.0" \
+  org.opencontainers.image.revision="e3b0c3d3c5c130e1dc6d567d6baef1c73eeb2059" \
+  org.opencontainers.image.source="https://github.com/elastic/elasticsearch" \
+  org.opencontainers.image.title="Elasticsearch" \
+  org.opencontainers.image.url="https://www.elastic.co/products/elasticsearch" \
+  org.opencontainers.image.vendor="Elastic" \
+  org.opencontainers.image.version="7.17.12"
+
+# Our actual entrypoint is `tini`, a minimal but functional init program. It
+# calls the entrypoint we provide, while correctly forwarding signals.
+ENTRYPOINT ["/bin/tini", "--", "/usr/local/bin/docker-entrypoint.sh"]
+# Dummy overridable parameter parsed by entrypoint
+CMD ["eswrapper"]
+
+################################################################################
+# End of multi-stage Dockerfile
+################################################################################