[GitLab] Add sidekiq and pages datastreams

packages/gitlab/_dev/build/docs/README.md

@@ -10,8 +10,12 @@ This integration is for ingesting logs from [GitLab](https://about.gitlab.com/).
 
 - **auth**: Collect logs for protected paths abusive requests or requests over the Rate Limit.
 
+- **pages**: Collect logs for Pages
+
 - **production**: Collect logs for Rails controller requests received from GitLab.
 
+- **sidekiq**: Collect logs from [sidekiq](https://sidekiq.org/) for jobs background jobs that take a long time
+
 See the GitLab [Log system docs](https://docs.gitlab.com/ee/administration/logs/) for more information.
 
 ## Compatibility
@@ -79,10 +83,26 @@ Collect logs for abusive protect paths requests or requests over the Rate Limit.
 
 {{event "auth"}}
 
+### pages
+
+Collect logs for Pages. Check out the [GitLab Pages log docs](https://docs.gitlab.com/ee/administration/logs/#pages-logs) for more information.
+
+{{fields "pages"}}
+
+{{event "pages"}}
+
 ### production
 
 Collect logs for Rails controller requests received from GitLab. Check out the [GitLab production log docs](https://docs.gitlab.com/ee/administration/logs/#production_jsonlog) for more information.
 
 {{fields "production"}}
 
 {{event "production"}}
+
+### sidekiq
+
+Collect logs from sidekiq for jobs background jobs that take a long time. Check out the [GitLab sidekiq log docs](https://docs.gitlab.com/ee/administration/logs/#sidekiq-logs) for more information.
+
+{{fields "sidekiq"}}
+
+{{event "sidekiq"}}

packages/gitlab/_dev/deploy/docker/sample_logs/test-gitlab-pages.log

@@ -0,0 +1,3 @@
+{"level": "info","msg": "GitLab Pages Daemon","revision": "52b2899","time": "2020-04-22T17:53:12Z","version": "1.17.0"}
+{"level": "info","msg": "URL: https://gitlab.com/gitlab-org/gitlab-pages","time": "2020-04-22T17:53:12Z"}
+{"gid": 998,"in-place": false,"level": "info","msg": "running the daemon as unprivileged user","time": "2020-04-22T17:53:12Z","uid": 998}

packages/gitlab/_dev/deploy/docker/sample_logs/test-gitlab-sidekiq.log

@@ -0,0 +1 @@
+{"severity": "INFO","time": "2018-04-03T22:57:22.071Z","queue": "cronjob:update_all_mirrors","args": [],"class": "UpdateAllMirrorsWorker","retry": false,"queue_namespace": "cronjob","jid": "06aeaa3b0aadacf9981f368e","created_at": "2018-04-03T22:57:21.930Z","enqueued_at": "2018-04-03T22:57:21.931Z","pid": 10077,"worker_id": "sidekiq_0","message": "UpdateAllMirrorsWorker JID-06aeaa3b0aadacf9981f368e: done: 0.139 sec","job_status": "done","duration": 0.139,"completed_at": "2018-04-03T22:57:22.071Z","db_duration": 0.05,"db_duration_s": 0.0005,"gitaly_duration": 0,"gitaly_calls": 0}

packages/gitlab/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: 1.1.0
+  changes:
+    - description: Add sidekiq and pages datastreams
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11234
 - version: 1.0.0
   changes:
     - description: Release package as GA.

packages/gitlab/data_stream/pages/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,3 @@
+fields:
+  tags:
+    - preserve_original_event

packages/gitlab/data_stream/pages/_dev/test/pipeline/test-pages.log

@@ -0,0 +1,3 @@
+{"level": "info","msg": "GitLab Pages Daemon","revision": "52b2899","time": "2020-04-22T17:53:12Z","version": "1.17.0"}
+{"level": "info","msg": "URL: https://gitlab.com/gitlab-org/gitlab-pages","time": "2020-04-22T17:53:12Z"}
+{"gid": 998,"in-place": false,"level": "info","msg": "running the daemon as unprivileged user","time": "2020-04-22T17:53:12Z","uid": 998}

packages/gitlab/data_stream/pages/_dev/test/pipeline/test-pages.log-expected.json

@@ -0,0 +1,89 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2020-04-22T17:53:12.000Z",
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "category": [
+                    "web"
+                ],
+                "kind": "event",
+                "level": 6,
+                "original": "{\"level\": \"info\",\"msg\": \"GitLab Pages Daemon\",\"revision\": \"52b2899\",\"time\": \"2020-04-22T17:53:12Z\",\"version\": \"1.17.0\"}",
+                "type": [
+                    "info"
+                ]
+            },
+            "gitlab": {
+                "pages": {
+                    "revision": "52b2899",
+                    "version": "1.17.0"
+                }
+            },
+            "message": "GitLab Pages Daemon",
+            "tags": [
+                "preserve_original_event"
+            ]
+        },
+        {
+            "@timestamp": "2020-04-22T17:53:12.000Z",
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "category": [
+                    "web"
+                ],
+                "kind": "event",
+                "level": 6,
+                "original": "{\"level\": \"info\",\"msg\": \"URL: https://gitlab.com/gitlab-org/gitlab-pages\",\"time\": \"2020-04-22T17:53:12Z\"}",
+                "type": [
+                    "info"
+                ]
+            },
+            "message": "URL: https://gitlab.com/gitlab-org/gitlab-pages",
+            "tags": [
+                "preserve_original_event"
+            ]
+        },
+        {
+            "@timestamp": "2020-04-22T17:53:12.000Z",
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "category": [
+                    "web"
+                ],
+                "kind": "event",
+                "level": 6,
+                "original": "{\"gid\": 998,\"in-place\": false,\"level\": \"info\",\"msg\": \"running the daemon as unprivileged user\",\"time\": \"2020-04-22T17:53:12Z\",\"uid\": 998}",
+                "type": [
+                    "info"
+                ]
+            },
+            "gitlab": {
+                "pages": {
+                    "in_place": false
+                }
+            },
+            "group": {
+                "id": "998"
+            },
+            "message": "running the daemon as unprivileged user",
+            "related": {
+                "user": [
+                    "998"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user": {
+                "id": "998"
+            }
+        }
+    ]
+}

packages/gitlab/data_stream/pages/_dev/test/system/test-default-config.yml

@@ -0,0 +1,15 @@
+service: gitlab-filestream
+input: filestream
+data_stream:
+  vars:
+    preserve_original_event: true
+    paths:
+      - '{{SERVICE_LOGS_DIR}}/test-gitlab-pages.log'
+numeric_keyword_fields:
+  - log.file.device_id
+  - log.file.inode
+  - log.file.idxhi
+  - log.file.idxlo
+  - log.file.vol
+assert:
+  hit_count: 3

packages/gitlab/data_stream/pages/agent/stream/filestream.yml.hbs

@@ -0,0 +1,27 @@
+paths:
+{{#each paths as |path|}}
+  - {{path}}
+{{/each}}
+{{#if exclude_files}}
+prospector.scanner.exclude_files:
+{{#each exclude_files as |pattern|}}
+  - {{pattern}}
+{{/each}}
+{{/if}}
+{{#if custom}}
+{{custom}}
+{{/if}}
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/gitlab/data_stream/pages/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,161 @@
+---
+description: Pipeline for processing pages logs
+processors:
+  - set:
+      field: ecs.version
+      tag: set_ecs_version
+      value: 8.11.0
+  - rename:
+      field: message
+      target_field: event.original
+      tag: rename_message
+      ignore_missing: true
+      if: ctx.event?.original == null
+  - remove:
+      field: message
+      ignore_missing: true
+      tag: remove_message
+      if: ctx.event?.original != null
+  - drop: 
+      if: ctx.event.original.startsWith('#')
+      description: Drop if logline contains header(s), which startswith `#`.
+  - append:
+      field: event.category
+      value: web
+  - append:
+      field: event.type
+      value: info
+  - set:
+      field: event.kind
+      value: event
+  - json:
+      field: event.original
+      tag: 'json_decoding'
+      target_field: gitlab.pages
+  - date:
+      field: gitlab.pages.time
+      formats:
+        - "ISO8601"
+      target_field: "@timestamp"
+      timezone: "UTC"
+      tag: date_event_created_time_epoch
+      on_failure:
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+      if: ctx.gitlab?.pages?.time != null
+  - remove:
+      field: gitlab.pages.time
+  - lowercase:
+      field: gitlab.pages.level
+      ignore_missing: true
+  - set:
+      field: event.level
+      value: 0
+      if: ctx.gitlab?.pages?.level == 'emergency'
+      tag: set_level_0
+  - set:
+      field: event.level
+      value: 1
+      if: ctx.gitlab?.pages?.level == 'alert'
+      tag: set_level_1
+  - set:
+      field: event.level
+      value: 2
+      if: ctx.gitlab?.pages?.level == 'critical'
+      tag: set_level_2
+  - set:
+      field: event.level
+      value: 3
+      if: ctx.gitlab?.pages?.level == 'error'
+      tag: set_level_3
+  - set:
+      field: event.level
+      value: 4
+      if: ctx.gitlab?.pages?.level == 'warn'
+      tag: set_level_4
+  - set:
+      field: event.level
+      value: 5
+      if: ctx.gitlab?.pages?.level == 'notice'
+      tag: set_level_5
+  - set:
+      field: event.level
+      value: 6
+      if: ctx.gitlab?.pages?.level == 'info'
+      tag: set_level_6
+  - set:
+      field: event.level
+      value: 7
+      if: ctx.gitlab?.pages?.level == 'debug'
+      tag: set_level_7
+  - remove:
+      field: gitlab.pages.level
+      ignore_missing: true
+  - rename:
+      field: gitlab.pages.correlation_id
+      target_field: event.id
+      ignore_missing: true
+  - rename: 
+      field: gitlab.pages.gid
+      target_field: group.id
+      ignore_missing: true
+  - convert:
+      field: group.id
+      type: string
+      ignore_missing: true
+  - rename: 
+      field: gitlab.pages.in-place
+      target_field: gitlab.pages.in_place
+      ignore_missing: true
+  - rename: 
+      field: gitlab.pages.uid
+      target_field: user.id
+      ignore_missing: true
+  - convert: 
+      field: user.id
+      type: string
+      ignore_missing: true
+  - rename: 
+      field: gitlab.pages.msg
+      target_field: message
+  - append:
+      field: related.user
+      value: '{{user.id}}'
+      if: ctx.user?.id != null
+      allow_duplicates: false
+  - remove:
+      field: event.original
+      tag: remove_event_original
+      ignore_missing: true
+      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
+  - script:
+      tag: script_to_drop_null_values
+      lang: painless
+      description: Drops null/empty values recursively.
+      source: |-
+        boolean drop(Object o) {
+          if (o == null || o == '') {
+            return true;
+          } else if (o instanceof Map) {
+            ((Map) o).values().removeIf(v -> drop(v));
+            return (((Map) o).size() == 0);
+          } else if (o instanceof List) {
+            ((List) o).removeIf(v -> drop(v));
+            return (((List) o).length == 0);
+          }
+          return false;
+        }
+        drop(ctx);
+  - append:
+      field: event.kind
+      value: pipeline_error
+      allow_duplicates: false
+      if: ctx.error?.message != null
+on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
+      field: error.message
+      value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'

packages/gitlab/data_stream/pages/fields/agent.yml

@@ -0,0 +1,33 @@
+- name: cloud
+  title: Cloud
+  group: 2
+  description: Fields related to the cloud or infrastructure the events are coming from.
+  footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.'
+  type: group
+  fields:
+    - name: image.id
+      type: keyword
+      description: Image ID for the cloud instance.
+- name: host
+  title: Host
+  group: 2
+  description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.'
+  type: group
+  fields:
+    - name: containerized
+      type: boolean
+      description: >
+        If the host is a container.
+
+    - name: os.build
+      type: keyword
+      example: "18D109"
+      description: >
+        OS build information.
+
+    - name: os.codename
+      type: keyword
+      example: "stretch"
+      description: >
+        OS codename, if any.
+