From 3fd8e2d4a25df031b51dea0702b322801bc2acdb Mon Sep 17 00:00:00 2001 From: tehbooom Date: Tue, 24 Sep 2024 08:53:09 -0400 Subject: [PATCH 1/5] feat: Add sidekiq and pages datastreams to GitLab --- packages/gitlab/_dev/build/docs/README.md | 20 ++ .../docker/sample_logs/test-gitlab-pages.log | 3 + .../sample_logs/test-gitlab-sidekiq.log | 1 + .../_dev/test/pipeline/test-common-config.yml | 3 + .../pages/_dev/test/pipeline/test-pages.log | 3 + .../pipeline/test-pages.log-expected.json | 89 +++++++ .../_dev/test/system/test-default-config.yml | 15 ++ .../pages/agent/stream/filestream.yml.hbs | 27 +++ .../elasticsearch/ingest_pipeline/default.yml | 161 +++++++++++++ .../gitlab/data_stream/pages/fields/agent.yml | 33 +++ .../data_stream/pages/fields/base-fields.yml | 20 ++ .../gitlab/data_stream/pages/fields/beats.yml | 18 ++ .../data_stream/pages/fields/fields.yml | 12 + .../gitlab/data_stream/pages/manifest.yml | 88 +++++++ .../data_stream/pages/sample_event.json | 60 +++++ .../_dev/test/pipeline/test-common-config.yml | 3 + .../test/pipeline/test-gitlab-sidekiq.log | 1 + .../test-gitlab-sidekiq.log-expected.json | 51 ++++ .../test/system/test-filestream-config.yml | 15 ++ .../sidekiq/agent/stream/filestream.yml.hbs | 27 +++ .../elasticsearch/ingest_pipeline/default.yml | 186 +++++++++++++++ .../data_stream/sidekiq/fields/agent.yml | 33 +++ .../sidekiq/fields/base-fields.yml | 20 ++ .../data_stream/sidekiq/fields/beats.yml | 18 ++ .../data_stream/sidekiq/fields/fields.yml | 38 +++ .../gitlab/data_stream/sidekiq/manifest.yml | 89 +++++++ .../data_stream/sidekiq/sample_event.json | 80 +++++++ packages/gitlab/docs/README.md | 220 ++++++++++++++++++ 28 files changed, 1334 insertions(+) create mode 100644 packages/gitlab/_dev/deploy/docker/sample_logs/test-gitlab-pages.log create mode 100644 packages/gitlab/_dev/deploy/docker/sample_logs/test-gitlab-sidekiq.log create mode 100644 packages/gitlab/data_stream/pages/_dev/test/pipeline/test-common-config.yml create mode 100644 packages/gitlab/data_stream/pages/_dev/test/pipeline/test-pages.log create mode 100644 packages/gitlab/data_stream/pages/_dev/test/pipeline/test-pages.log-expected.json create mode 100644 packages/gitlab/data_stream/pages/_dev/test/system/test-default-config.yml create mode 100644 packages/gitlab/data_stream/pages/agent/stream/filestream.yml.hbs create mode 100644 packages/gitlab/data_stream/pages/elasticsearch/ingest_pipeline/default.yml create mode 100644 packages/gitlab/data_stream/pages/fields/agent.yml create mode 100644 packages/gitlab/data_stream/pages/fields/base-fields.yml create mode 100644 packages/gitlab/data_stream/pages/fields/beats.yml create mode 100644 packages/gitlab/data_stream/pages/fields/fields.yml create mode 100644 packages/gitlab/data_stream/pages/manifest.yml create mode 100644 packages/gitlab/data_stream/pages/sample_event.json create mode 100644 packages/gitlab/data_stream/sidekiq/_dev/test/pipeline/test-common-config.yml create mode 100644 packages/gitlab/data_stream/sidekiq/_dev/test/pipeline/test-gitlab-sidekiq.log create mode 100644 packages/gitlab/data_stream/sidekiq/_dev/test/pipeline/test-gitlab-sidekiq.log-expected.json create mode 100644 packages/gitlab/data_stream/sidekiq/_dev/test/system/test-filestream-config.yml create mode 100644 packages/gitlab/data_stream/sidekiq/agent/stream/filestream.yml.hbs create mode 100644 packages/gitlab/data_stream/sidekiq/elasticsearch/ingest_pipeline/default.yml create mode 100644 packages/gitlab/data_stream/sidekiq/fields/agent.yml create mode 100644 packages/gitlab/data_stream/sidekiq/fields/base-fields.yml create mode 100644 packages/gitlab/data_stream/sidekiq/fields/beats.yml create mode 100644 packages/gitlab/data_stream/sidekiq/fields/fields.yml create mode 100644 packages/gitlab/data_stream/sidekiq/manifest.yml create mode 100644 packages/gitlab/data_stream/sidekiq/sample_event.json diff --git a/packages/gitlab/_dev/build/docs/README.md b/packages/gitlab/_dev/build/docs/README.md index 695084d48233..dea518efd691 100644 --- a/packages/gitlab/_dev/build/docs/README.md +++ b/packages/gitlab/_dev/build/docs/README.md @@ -10,8 +10,12 @@ This integration is for ingesting logs from [GitLab](https://about.gitlab.com/). - `auth`: Collect logs for protected paths abusive requests or requests over the Rate Limit. +- `pages`: Collect logs for Pages + - `production`: Collect logs for Rails controller requests received from GitLab. +- `sidekiq`: Collect logs from [sidekiq](https://sidekiq.org/) for jobs background jobs that take a long time + See the GitLab [Log system docs](https://docs.gitlab.com/ee/administration/logs/) for more information. ## Compatibility @@ -56,6 +60,14 @@ Collect logs for abusive protect paths requests or requests over the Rate Limit. {{event "auth"}} +### pages + +Collect logs for Pages. Check out the [GitLab Pages log docs](https://docs.gitlab.com/ee/administration/logs/#pages-logs) for more information. + +{{fields "pages"}} + +{{event "pages"}} + ### production Collect logs for Rails controller requests received from GitLab. Check out the [GitLab production log docs](https://docs.gitlab.com/ee/administration/logs/#production_jsonlog) for more information. @@ -63,3 +75,11 @@ Collect logs for Rails controller requests received from GitLab. Check out the [ {{fields "production"}} {{event "production"}} + +### sidekiq + +Collect logs from sidekiq for jobs background jobs that take a long time. Check out the [GitLab sidekiq log docs](https://docs.gitlab.com/ee/administration/logs/#sidekiq-logs) for more information. + +{{fields "sidekiq"}} + +{{event "sidekiq"}} diff --git a/packages/gitlab/_dev/deploy/docker/sample_logs/test-gitlab-pages.log b/packages/gitlab/_dev/deploy/docker/sample_logs/test-gitlab-pages.log new file mode 100644 index 000000000000..c3c270aa5800 --- /dev/null +++ b/packages/gitlab/_dev/deploy/docker/sample_logs/test-gitlab-pages.log @@ -0,0 +1,3 @@ +{"level": "info","msg": "GitLab Pages Daemon","revision": "52b2899","time": "2020-04-22T17:53:12Z","version": "1.17.0"} +{"level": "info","msg": "URL: https://gitlab.com/gitlab-org/gitlab-pages","time": "2020-04-22T17:53:12Z"} +{"gid": 998,"in-place": false,"level": "info","msg": "running the daemon as unprivileged user","time": "2020-04-22T17:53:12Z","uid": 998} diff --git a/packages/gitlab/_dev/deploy/docker/sample_logs/test-gitlab-sidekiq.log b/packages/gitlab/_dev/deploy/docker/sample_logs/test-gitlab-sidekiq.log new file mode 100644 index 000000000000..338fa73f22c9 --- /dev/null +++ b/packages/gitlab/_dev/deploy/docker/sample_logs/test-gitlab-sidekiq.log @@ -0,0 +1 @@ +{"severity": "INFO","time": "2018-04-03T22:57:22.071Z","queue": "cronjob:update_all_mirrors","args": [],"class": "UpdateAllMirrorsWorker","retry": false,"queue_namespace": "cronjob","jid": "06aeaa3b0aadacf9981f368e","created_at": "2018-04-03T22:57:21.930Z","enqueued_at": "2018-04-03T22:57:21.931Z","pid": 10077,"worker_id": "sidekiq_0","message": "UpdateAllMirrorsWorker JID-06aeaa3b0aadacf9981f368e: done: 0.139 sec","job_status": "done","duration": 0.139,"completed_at": "2018-04-03T22:57:22.071Z","db_duration": 0.05,"db_duration_s": 0.0005,"gitaly_duration": 0,"gitaly_calls": 0} diff --git a/packages/gitlab/data_stream/pages/_dev/test/pipeline/test-common-config.yml b/packages/gitlab/data_stream/pages/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 000000000000..4da226416540 --- /dev/null +++ b/packages/gitlab/data_stream/pages/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,3 @@ +fields: + tags: + - preserve_original_event diff --git a/packages/gitlab/data_stream/pages/_dev/test/pipeline/test-pages.log b/packages/gitlab/data_stream/pages/_dev/test/pipeline/test-pages.log new file mode 100644 index 000000000000..c3c270aa5800 --- /dev/null +++ b/packages/gitlab/data_stream/pages/_dev/test/pipeline/test-pages.log @@ -0,0 +1,3 @@ +{"level": "info","msg": "GitLab Pages Daemon","revision": "52b2899","time": "2020-04-22T17:53:12Z","version": "1.17.0"} +{"level": "info","msg": "URL: https://gitlab.com/gitlab-org/gitlab-pages","time": "2020-04-22T17:53:12Z"} +{"gid": 998,"in-place": false,"level": "info","msg": "running the daemon as unprivileged user","time": "2020-04-22T17:53:12Z","uid": 998} diff --git a/packages/gitlab/data_stream/pages/_dev/test/pipeline/test-pages.log-expected.json b/packages/gitlab/data_stream/pages/_dev/test/pipeline/test-pages.log-expected.json new file mode 100644 index 000000000000..deda421e88be --- /dev/null +++ b/packages/gitlab/data_stream/pages/_dev/test/pipeline/test-pages.log-expected.json @@ -0,0 +1,89 @@ +{ + "expected": [ + { + "@timestamp": "2020-04-22T17:53:12.000Z", + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "web" + ], + "kind": "event", + "level": 6, + "original": "{\"level\": \"info\",\"msg\": \"GitLab Pages Daemon\",\"revision\": \"52b2899\",\"time\": \"2020-04-22T17:53:12Z\",\"version\": \"1.17.0\"}", + "type": [ + "info" + ] + }, + "gitlab": { + "pages": { + "revision": "52b2899", + "version": "1.17.0" + } + }, + "message": "GitLab Pages Daemon", + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2020-04-22T17:53:12.000Z", + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "web" + ], + "kind": "event", + "level": 6, + "original": "{\"level\": \"info\",\"msg\": \"URL: https://gitlab.com/gitlab-org/gitlab-pages\",\"time\": \"2020-04-22T17:53:12Z\"}", + "type": [ + "info" + ] + }, + "message": "URL: https://gitlab.com/gitlab-org/gitlab-pages", + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2020-04-22T17:53:12.000Z", + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "web" + ], + "kind": "event", + "level": 6, + "original": "{\"gid\": 998,\"in-place\": false,\"level\": \"info\",\"msg\": \"running the daemon as unprivileged user\",\"time\": \"2020-04-22T17:53:12Z\",\"uid\": 998}", + "type": [ + "info" + ] + }, + "gitlab": { + "pages": { + "in_place": false + } + }, + "group": { + "id": "998" + }, + "message": "running the daemon as unprivileged user", + "related": { + "user": [ + "998" + ] + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "id": "998" + } + } + ] +} \ No newline at end of file diff --git a/packages/gitlab/data_stream/pages/_dev/test/system/test-default-config.yml b/packages/gitlab/data_stream/pages/_dev/test/system/test-default-config.yml new file mode 100644 index 000000000000..16e338c6f3c0 --- /dev/null +++ b/packages/gitlab/data_stream/pages/_dev/test/system/test-default-config.yml @@ -0,0 +1,15 @@ +service: gitlab-filestream +input: filestream +data_stream: + vars: + preserve_original_event: true + paths: + - '{{SERVICE_LOGS_DIR}}/test-gitlab-pages.log' +numeric_keyword_fields: + - log.file.device_id + - log.file.inode + - log.file.idxhi + - log.file.idxlo + - log.file.vol +assert: + hit_count: 3 diff --git a/packages/gitlab/data_stream/pages/agent/stream/filestream.yml.hbs b/packages/gitlab/data_stream/pages/agent/stream/filestream.yml.hbs new file mode 100644 index 000000000000..ec554604ee9d --- /dev/null +++ b/packages/gitlab/data_stream/pages/agent/stream/filestream.yml.hbs @@ -0,0 +1,27 @@ +paths: +{{#each paths as |path|}} + - {{path}} +{{/each}} +{{#if exclude_files}} +prospector.scanner.exclude_files: +{{#each exclude_files as |pattern|}} + - {{pattern}} +{{/each}} +{{/if}} +{{#if custom}} +{{custom}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/gitlab/data_stream/pages/elasticsearch/ingest_pipeline/default.yml b/packages/gitlab/data_stream/pages/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000000..e9be8652f524 --- /dev/null +++ b/packages/gitlab/data_stream/pages/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,161 @@ +--- +description: Pipeline for processing pages logs +processors: + - set: + field: ecs.version + tag: set_ecs_version + value: 8.11.0 + - rename: + field: message + target_field: event.original + tag: rename_message + ignore_missing: true + if: ctx.event?.original == null + - remove: + field: message + ignore_missing: true + tag: remove_message + if: ctx.event?.original != null + - drop: + if: ctx.event.original.startsWith('#') + description: Drop if logline contains header(s), which startswith `#`. + - append: + field: event.category + value: web + - append: + field: event.type + value: info + - set: + field: event.kind + value: event + - json: + field: event.original + tag: 'json_decoding' + target_field: gitlab.pages + - date: + field: gitlab.pages.time + formats: + - "ISO8601" + target_field: "@timestamp" + timezone: "UTC" + tag: date_event_created_time_epoch + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + if: ctx.gitlab?.pages?.time != null + - remove: + field: gitlab.pages.time + - lowercase: + field: gitlab.pages.level + ignore_missing: true + - set: + field: event.level + value: 0 + if: ctx.gitlab?.pages?.level == 'emergency' + tag: set_level_0 + - set: + field: event.level + value: 1 + if: ctx.gitlab?.pages?.level == 'alert' + tag: set_level_1 + - set: + field: event.level + value: 2 + if: ctx.gitlab?.pages?.level == 'critical' + tag: set_level_2 + - set: + field: event.level + value: 3 + if: ctx.gitlab?.pages?.level == 'error' + tag: set_level_3 + - set: + field: event.level + value: 4 + if: ctx.gitlab?.pages?.level == 'warn' + tag: set_level_4 + - set: + field: event.level + value: 5 + if: ctx.gitlab?.pages?.level == 'notice' + tag: set_level_5 + - set: + field: event.level + value: 6 + if: ctx.gitlab?.pages?.level == 'info' + tag: set_level_6 + - set: + field: event.level + value: 7 + if: ctx.gitlab?.pages?.level == 'debug' + tag: set_level_7 + - remove: + field: gitlab.pages.level + ignore_missing: true + - rename: + field: gitlab.pages.correlation_id + target_field: event.id + ignore_missing: true + - rename: + field: gitlab.pages.gid + target_field: group.id + ignore_missing: true + - convert: + field: group.id + type: string + ignore_missing: true + - rename: + field: gitlab.pages.in-place + target_field: gitlab.pages.in_place + ignore_missing: true + - rename: + field: gitlab.pages.uid + target_field: user.id + ignore_missing: true + - convert: + field: user.id + type: string + ignore_missing: true + - rename: + field: gitlab.pages.msg + target_field: message + - append: + field: related.user + value: '{{user.id}}' + if: ctx.user?.id != null + allow_duplicates: false + - remove: + field: event.original + tag: remove_event_original + ignore_missing: true + if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) + - script: + tag: script_to_drop_null_values + lang: painless + description: Drops null/empty values recursively. + source: |- + boolean drop(Object o) { + if (o == null || o == '') { + return true; + } else if (o instanceof Map) { + ((Map) o).values().removeIf(v -> drop(v)); + return (((Map) o).size() == 0); + } else if (o instanceof List) { + ((List) o).removeIf(v -> drop(v)); + return (((List) o).length == 0); + } + return false; + } + drop(ctx); + - append: + field: event.kind + value: pipeline_error + allow_duplicates: false + if: ctx.error?.message != null +on_failure: + - set: + field: event.kind + value: pipeline_error + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' diff --git a/packages/gitlab/data_stream/pages/fields/agent.yml b/packages/gitlab/data_stream/pages/fields/agent.yml new file mode 100644 index 000000000000..df92bfa51a9d --- /dev/null +++ b/packages/gitlab/data_stream/pages/fields/agent.yml @@ -0,0 +1,33 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/packages/gitlab/data_stream/pages/fields/base-fields.yml b/packages/gitlab/data_stream/pages/fields/base-fields.yml new file mode 100644 index 000000000000..9caf79d0a39b --- /dev/null +++ b/packages/gitlab/data_stream/pages/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset name. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module + value: gitlab +- name: event.dataset + type: constant_keyword + description: Event dataset + value: gitlab.pages +- name: "@timestamp" + type: date + description: Event timestamp. diff --git a/packages/gitlab/data_stream/pages/fields/beats.yml b/packages/gitlab/data_stream/pages/fields/beats.yml new file mode 100644 index 000000000000..f30dc2f91f44 --- /dev/null +++ b/packages/gitlab/data_stream/pages/fields/beats.yml @@ -0,0 +1,18 @@ +- name: input.type + type: keyword + description: Type of Filebeat input. +- name: log.flags + type: keyword + description: Flags for the log file. +- name: log.offset + type: long + description: Offset of the entry in the log file. +- name: log.file + type: group + fields: + - name: device_id + type: keyword + description: ID of the device containing the filesystem where the file resides. + - name: inode + type: keyword + description: Inode number of the log file. diff --git a/packages/gitlab/data_stream/pages/fields/fields.yml b/packages/gitlab/data_stream/pages/fields/fields.yml new file mode 100644 index 000000000000..6b2b75c3db67 --- /dev/null +++ b/packages/gitlab/data_stream/pages/fields/fields.yml @@ -0,0 +1,12 @@ +- name: gitlab + type: group + fields: + - name: pages + type: group + fields: + - name: in_place + type: boolean + - name: revision + type: keyword + - name: version + type: keyword diff --git a/packages/gitlab/data_stream/pages/manifest.yml b/packages/gitlab/data_stream/pages/manifest.yml new file mode 100644 index 000000000000..25a1e5d2f2db --- /dev/null +++ b/packages/gitlab/data_stream/pages/manifest.yml @@ -0,0 +1,88 @@ +title: GitLab Pages logs +type: logs +streams: + - input: filestream + template_path: filestream.yml.hbs + title: GitLab Pages logs + description: Collect logs for Pages logs + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - '/var/log/gitlab/gitlab-pages/current' + description: The full path to the related log file. + - name: exclude_files + type: text + title: Exclude Files + description: A list of regular expressions to match the files that you want Filebeat to ignore. + required: false + show_user: true + default: + - '\.gz$' + - name: custom + type: yaml + title: Additional Filestream Configuration Options + required: false + show_user: false + description: >- + Configuration options that can be used to further change input configuration. Check the [Filebeat documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-filestream.html) for more information. + default: |- + #encoding: plain + prospector.scanner.recursive_glob: true + #prospector.scanner.symlinks: true + #prospector.scanner.include_files: [''] + #prospector.scanner.resend_on_touch: false + #prospector.scanner.check_interval: 10s + #prospector.scanner.fingerprint.enabled: false + #prospector.scanner.fingerprint.offset: 0 + #prospector.scanner.fingerprint.length: 1024 + #ignore_older: 0 + #ignore_inactive: '' + #close.on_state_change.inactive: 5m + #close.on_state_change.renamed: false + #close.on_state_change.removed: false + #close.reader.on_eof: false + #close.reader.after_interval: 0 + #clean_inactive: 0 + #clean_removed: true + #backoff.init: 2s + #backoff.max: 10s + #file_identity.native: ~ + #file_identity.path: '' + #file_identity.inode_marker.path: '' + #file_identity.fingerprint: ~ + #rotation.external.strategy.copytruncate.suffix_regex: '\.\d$' + #rotation.external.strategy.copytruncate.dateformat: '-20060102' + #include_lines: ['sometext'] + #exclude_lines: ['^DBG'] + #buffer_size: 16384 + #message_max_bytes: 1048576 + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - gitlab-pages + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/gitlab/data_stream/pages/sample_event.json b/packages/gitlab/data_stream/pages/sample_event.json new file mode 100644 index 000000000000..dda1d558c165 --- /dev/null +++ b/packages/gitlab/data_stream/pages/sample_event.json @@ -0,0 +1,60 @@ +{ + "@timestamp": "2020-04-22T17:53:12.000Z", + "agent": { + "ephemeral_id": "9c660d07-7fab-4b63-baad-3d850d847ec1", + "id": "20a743fb-5e21-4498-ba58-6ebb10053f90", + "name": "elastic-agent-63322", + "type": "filebeat", + "version": "8.15.0" + }, + "data_stream": { + "dataset": "gitlab.pages", + "namespace": "81772", + "type": "logs" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "20a743fb-5e21-4498-ba58-6ebb10053f90", + "snapshot": false, + "version": "8.15.0" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "web" + ], + "dataset": "gitlab.pages", + "ingested": "2024-09-23T20:44:09Z", + "kind": "event", + "level": 6, + "original": "{\"level\": \"info\",\"msg\": \"GitLab Pages Daemon\",\"revision\": \"52b2899\",\"time\": \"2020-04-22T17:53:12Z\",\"version\": \"1.17.0\"}", + "type": [ + "info" + ] + }, + "gitlab": { + "pages": { + "revision": "52b2899", + "version": "1.17.0" + } + }, + "input": { + "type": "filestream" + }, + "log": { + "file": { + "device_id": "30", + "inode": "163", + "path": "/tmp/service_logs/test-gitlab-pages.log" + }, + "offset": 0 + }, + "message": "GitLab Pages Daemon", + "tags": [ + "preserve_original_event", + "forwarded", + "gitlab-pages" + ] +} \ No newline at end of file diff --git a/packages/gitlab/data_stream/sidekiq/_dev/test/pipeline/test-common-config.yml b/packages/gitlab/data_stream/sidekiq/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 000000000000..4da226416540 --- /dev/null +++ b/packages/gitlab/data_stream/sidekiq/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,3 @@ +fields: + tags: + - preserve_original_event diff --git a/packages/gitlab/data_stream/sidekiq/_dev/test/pipeline/test-gitlab-sidekiq.log b/packages/gitlab/data_stream/sidekiq/_dev/test/pipeline/test-gitlab-sidekiq.log new file mode 100644 index 000000000000..338fa73f22c9 --- /dev/null +++ b/packages/gitlab/data_stream/sidekiq/_dev/test/pipeline/test-gitlab-sidekiq.log @@ -0,0 +1 @@ +{"severity": "INFO","time": "2018-04-03T22:57:22.071Z","queue": "cronjob:update_all_mirrors","args": [],"class": "UpdateAllMirrorsWorker","retry": false,"queue_namespace": "cronjob","jid": "06aeaa3b0aadacf9981f368e","created_at": "2018-04-03T22:57:21.930Z","enqueued_at": "2018-04-03T22:57:21.931Z","pid": 10077,"worker_id": "sidekiq_0","message": "UpdateAllMirrorsWorker JID-06aeaa3b0aadacf9981f368e: done: 0.139 sec","job_status": "done","duration": 0.139,"completed_at": "2018-04-03T22:57:22.071Z","db_duration": 0.05,"db_duration_s": 0.0005,"gitaly_duration": 0,"gitaly_calls": 0} diff --git a/packages/gitlab/data_stream/sidekiq/_dev/test/pipeline/test-gitlab-sidekiq.log-expected.json b/packages/gitlab/data_stream/sidekiq/_dev/test/pipeline/test-gitlab-sidekiq.log-expected.json new file mode 100644 index 000000000000..9e5fe96bc5e1 --- /dev/null +++ b/packages/gitlab/data_stream/sidekiq/_dev/test/pipeline/test-gitlab-sidekiq.log-expected.json @@ -0,0 +1,51 @@ +{ + "expected": [ + { + "@timestamp": "2018-04-03T22:57:22.071Z", + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "web" + ], + "duration": 0.139, + "end": "2018-04-03T22:57:22.071Z", + "kind": "event", + "original": "{\"severity\": \"INFO\",\"time\": \"2018-04-03T22:57:22.071Z\",\"queue\": \"cronjob:update_all_mirrors\",\"args\": [],\"class\": \"UpdateAllMirrorsWorker\",\"retry\": false,\"queue_namespace\": \"cronjob\",\"jid\": \"06aeaa3b0aadacf9981f368e\",\"created_at\": \"2018-04-03T22:57:21.930Z\",\"enqueued_at\": \"2018-04-03T22:57:21.931Z\",\"pid\": 10077,\"worker_id\": \"sidekiq_0\",\"message\": \"UpdateAllMirrorsWorker JID-06aeaa3b0aadacf9981f368e: done: 0.139 sec\",\"job_status\": \"done\",\"duration\": 0.139,\"completed_at\": \"2018-04-03T22:57:22.071Z\",\"db_duration\": 0.05,\"db_duration_s\": 0.0005,\"gitaly_duration\": 0,\"gitaly_calls\": 0}", + "severity": 6, + "start": "2018-04-03T22:57:21.930Z", + "type": [ + "info" + ] + }, + "gitlab": { + "gitaly": { + "calls": 0, + "duration": 0 + }, + "sidekiq": { + "class": "UpdateAllMirrorsWorker", + "db": { + "duration_m": 0.05, + "duration_s": 5.0E-4 + }, + "enqueued_at": "2018-04-03T22:57:21.931Z", + "jid": "06aeaa3b0aadacf9981f368e", + "job_status": "done", + "queue": "cronjob:update_all_mirrors", + "queue_namespace": "cronjob", + "retry": false, + "worker_id": "sidekiq_0" + } + }, + "message": "UpdateAllMirrorsWorker JID-06aeaa3b0aadacf9981f368e: done: 0.139 sec", + "process": { + "pid": 10077 + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/packages/gitlab/data_stream/sidekiq/_dev/test/system/test-filestream-config.yml b/packages/gitlab/data_stream/sidekiq/_dev/test/system/test-filestream-config.yml new file mode 100644 index 000000000000..00203df053ab --- /dev/null +++ b/packages/gitlab/data_stream/sidekiq/_dev/test/system/test-filestream-config.yml @@ -0,0 +1,15 @@ +service: gitlab-filestream +input: filestream +data_stream: + vars: + preserve_original_event: true + paths: + - '{{SERVICE_LOGS_DIR}}/test-gitlab-sidekiq.log' +numeric_keyword_fields: + - log.file.device_id + - log.file.inode + - log.file.idxhi + - log.file.idxlo + - log.file.vol +assert: + hit_count: 1 diff --git a/packages/gitlab/data_stream/sidekiq/agent/stream/filestream.yml.hbs b/packages/gitlab/data_stream/sidekiq/agent/stream/filestream.yml.hbs new file mode 100644 index 000000000000..ec554604ee9d --- /dev/null +++ b/packages/gitlab/data_stream/sidekiq/agent/stream/filestream.yml.hbs @@ -0,0 +1,27 @@ +paths: +{{#each paths as |path|}} + - {{path}} +{{/each}} +{{#if exclude_files}} +prospector.scanner.exclude_files: +{{#each exclude_files as |pattern|}} + - {{pattern}} +{{/each}} +{{/if}} +{{#if custom}} +{{custom}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/gitlab/data_stream/sidekiq/elasticsearch/ingest_pipeline/default.yml b/packages/gitlab/data_stream/sidekiq/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000000..94285075de50 --- /dev/null +++ b/packages/gitlab/data_stream/sidekiq/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,186 @@ +--- +description: Pipeline for processing auth logs +processors: + - set: + field: ecs.version + value: 8.11.0 + - rename: + field: message + tag: rename_message_to_event_original + target_field: event.original + if: ctx.event?.original == null + ignore_missing: true + - remove: + field: message + ignore_missing: true + tag: remove_message + if: ctx.event?.original != null + - drop: + if: ctx.event.original.startsWith('#') + description: Drop if logline contains header(s), which startswith `#`. + - json: + field: event.original + tag: 'json_decoding' + target_field: gitlab.sidekiq + - append: + field: event.category + value: web + - append: + field: event.type + value: info + - set: + field: event.kind + value: event + - date: + field: gitlab.sidekiq.time + formats: + - "ISO8601" + target_field: "@timestamp" + timezone: "UTC" + tag: date_event_created_time_epoch + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + if: ctx.gitlab?.sidekiq?.time != null + - remove: + field: gitlab.sidekiq.time + - lowercase: + field: gitlab.sidekiq.severity + - set: + field: event.severity + value: 0 + if: ctx.gitlab?.sidekiq?.severity == 'emergency' + tag: set_severity_0 + - set: + field: event.severity + value: 1 + if: ctx.gitlab?.sidekiq?.severity == 'alert' + tag: set_severity_1 + - set: + field: event.severity + value: 2 + if: ctx.gitlab?.sidekiq?.severity == 'critical' + tag: set_severity_2 + - set: + field: event.severity + value: 3 + if: ctx.gitlab?.sidekiq?.severity == 'error' + tag: set_severity_3 + - set: + field: event.severity + value: 4 + if: ctx.gitlab?.sidekiq?.severity == 'warn' + tag: set_severity_4 + - set: + field: event.severity + value: 5 + if: ctx.gitlab?.sidekiq?.severity == 'notice' + tag: set_severity_5 + - set: + field: event.severity + value: 6 + if: ctx.gitlab?.sidekiq?.severity == 'info' + tag: set_severity_6 + - set: + field: event.severity + value: 7 + if: ctx.gitlab?.sidekiq?.severity == 'debug' + tag: set_severity_7 + - remove: + field: gitlab.sidekiq.severity + ignore_missing: true + - date: + field: gitlab.sidekiq.created_at + formats: + - "ISO8601" + target_field: event.start + timezone: "UTC" + tag: start_date_event_created_time_epoch + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - remove: + field: gitlab.sidekiq.created_at + - date: + field: gitlab.sidekiq.enqueued_at + formats: + - "ISO8601" + target_field: gitlab.sidekiq.enqueued_at + timezone: "UTC" + tag: enqueued_date_event_created_time_epoch + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - rename: + field: gitlab.sidekiq.pid + target_field: process.pid + ignore_missing: true + - rename: + field: gitlab.sidekiq.message + target_field: message + - rename: + field: gitlab.sidekiq.duration + target_field: event.duration + - date: + field: gitlab.sidekiq.completed_at + formats: + - "ISO8601" + target_field: event.end + timezone: "UTC" + tag: end_date_event_created_time_epoch + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - remove: + field: gitlab.sidekiq.completed_at + - rename: + field: gitlab.sidekiq.db_duration + target_field: gitlab.sidekiq.db.duration_m + - rename: + field: gitlab.sidekiq.db_duration_s + target_field: gitlab.sidekiq.db.duration_s + - rename: + field: gitlab.sidekiq.gitaly_duration + target_field: gitlab.gitaly.duration + - rename: + field: gitlab.sidekiq.gitaly_calls + target_field: gitlab.gitaly.calls + - remove: + field: event.original + tag: remove_event_original + ignore_missing: true + if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) + - script: + tag: script_to_drop_null_values + lang: painless + description: Drops null/empty values recursively. + source: |- + boolean drop(Object o) { + if (o == null || o == '') { + return true; + } else if (o instanceof Map) { + ((Map) o).values().removeIf(v -> drop(v)); + return (((Map) o).size() == 0); + } else if (o instanceof List) { + ((List) o).removeIf(v -> drop(v)); + return (((List) o).length == 0); + } + return false; + } + drop(ctx); + - append: + field: event.kind + value: pipeline_error + allow_duplicates: false + if: ctx.error?.message != null +on_failure: + - set: + field: event.kind + value: pipeline_error + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' diff --git a/packages/gitlab/data_stream/sidekiq/fields/agent.yml b/packages/gitlab/data_stream/sidekiq/fields/agent.yml new file mode 100644 index 000000000000..df92bfa51a9d --- /dev/null +++ b/packages/gitlab/data_stream/sidekiq/fields/agent.yml @@ -0,0 +1,33 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/packages/gitlab/data_stream/sidekiq/fields/base-fields.yml b/packages/gitlab/data_stream/sidekiq/fields/base-fields.yml new file mode 100644 index 000000000000..e9cf4f3720e3 --- /dev/null +++ b/packages/gitlab/data_stream/sidekiq/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset name. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module + value: gitlab +- name: event.dataset + type: constant_keyword + description: Event dataset + value: gitlab.sidekiq +- name: "@timestamp" + type: date + description: Event timestamp. diff --git a/packages/gitlab/data_stream/sidekiq/fields/beats.yml b/packages/gitlab/data_stream/sidekiq/fields/beats.yml new file mode 100644 index 000000000000..f30dc2f91f44 --- /dev/null +++ b/packages/gitlab/data_stream/sidekiq/fields/beats.yml @@ -0,0 +1,18 @@ +- name: input.type + type: keyword + description: Type of Filebeat input. +- name: log.flags + type: keyword + description: Flags for the log file. +- name: log.offset + type: long + description: Offset of the entry in the log file. +- name: log.file + type: group + fields: + - name: device_id + type: keyword + description: ID of the device containing the filesystem where the file resides. + - name: inode + type: keyword + description: Inode number of the log file. diff --git a/packages/gitlab/data_stream/sidekiq/fields/fields.yml b/packages/gitlab/data_stream/sidekiq/fields/fields.yml new file mode 100644 index 000000000000..5b838d84727d --- /dev/null +++ b/packages/gitlab/data_stream/sidekiq/fields/fields.yml @@ -0,0 +1,38 @@ +- name: gitlab + type: group + fields: + - name: sidekiq + type: group + fields: + - name: queue + type: keyword + - name: args + type: keyword + - name: class + type: keyword + - name: retry + type: boolean + - name: queue_namespace + type: keyword + - name: jid + type: keyword + - name: enqueued_at + type: date + - name: worker_id + type: keyword + - name: job_status + type: keyword + - name: db + type: group + fields: + - name: duration_m + type: float + - name: duration_s + type: float + - name: gitaly + type: group + fields: + - name: calls + type: long + - name: duration + type: long diff --git a/packages/gitlab/data_stream/sidekiq/manifest.yml b/packages/gitlab/data_stream/sidekiq/manifest.yml new file mode 100644 index 000000000000..26fa6ae7a5f0 --- /dev/null +++ b/packages/gitlab/data_stream/sidekiq/manifest.yml @@ -0,0 +1,89 @@ +title: GitLab Sidekiq logs +type: logs +streams: + - input: filestream + template_path: filestream.yml.hbs + title: GitLab Sidekiq logs + description: Collect logs for Sidekiq + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - '/var/log/gitlab/sidekiq/current' + - '/home/git/gitlab/log/sidekiq.log' + description: The full path to the related log file. + - name: exclude_files + type: text + title: Exclude Files + description: A list of regular expressions to match the files that you want Filebeat to ignore. + required: false + show_user: true + default: + - '\.gz$' + - name: custom + type: yaml + title: Additional Filestream Configuration Options + required: false + show_user: false + description: >- + Configuration options that can be used to further change input configuration. Check the [Filebeat documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-filestream.html) for more information. + default: |- + #encoding: plain + prospector.scanner.recursive_glob: true + #prospector.scanner.symlinks: true + #prospector.scanner.include_files: [''] + #prospector.scanner.resend_on_touch: false + #prospector.scanner.check_interval: 10s + #prospector.scanner.fingerprint.enabled: false + #prospector.scanner.fingerprint.offset: 0 + #prospector.scanner.fingerprint.length: 1024 + #ignore_older: 0 + #ignore_inactive: '' + #close.on_state_change.inactive: 5m + #close.on_state_change.renamed: false + #close.on_state_change.removed: false + #close.reader.on_eof: false + #close.reader.after_interval: 0 + #clean_inactive: 0 + #clean_removed: true + #backoff.init: 2s + #backoff.max: 10s + #file_identity.native: ~ + #file_identity.path: '' + #file_identity.inode_marker.path: '' + #file_identity.fingerprint: ~ + #rotation.external.strategy.copytruncate.suffix_regex: '\.\d$' + #rotation.external.strategy.copytruncate.dateformat: '-20060102' + #include_lines: ['sometext'] + #exclude_lines: ['^DBG'] + #buffer_size: 16384 + #message_max_bytes: 1048576 + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - gitlab-sidekiq + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/gitlab/data_stream/sidekiq/sample_event.json b/packages/gitlab/data_stream/sidekiq/sample_event.json new file mode 100644 index 000000000000..5f0eb09ca7a2 --- /dev/null +++ b/packages/gitlab/data_stream/sidekiq/sample_event.json @@ -0,0 +1,80 @@ +{ + "@timestamp": "2018-04-03T22:57:22.071Z", + "agent": { + "ephemeral_id": "cc5ea64a-1be7-4bf7-ac2e-a934734ba7d0", + "id": "d0298772-a948-4edb-95bf-6b9152967f34", + "name": "elastic-agent-16126", + "type": "filebeat", + "version": "8.15.0" + }, + "data_stream": { + "dataset": "gitlab.sidekiq", + "namespace": "99205", + "type": "logs" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "d0298772-a948-4edb-95bf-6b9152967f34", + "snapshot": false, + "version": "8.15.0" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "web" + ], + "dataset": "gitlab.sidekiq", + "duration": 0.139, + "end": "2018-04-03T22:57:22.071Z", + "ingested": "2024-09-23T21:03:08Z", + "kind": "event", + "original": "{\"severity\": \"INFO\",\"time\": \"2018-04-03T22:57:22.071Z\",\"queue\": \"cronjob:update_all_mirrors\",\"args\": [],\"class\": \"UpdateAllMirrorsWorker\",\"retry\": false,\"queue_namespace\": \"cronjob\",\"jid\": \"06aeaa3b0aadacf9981f368e\",\"created_at\": \"2018-04-03T22:57:21.930Z\",\"enqueued_at\": \"2018-04-03T22:57:21.931Z\",\"pid\": 10077,\"worker_id\": \"sidekiq_0\",\"message\": \"UpdateAllMirrorsWorker JID-06aeaa3b0aadacf9981f368e: done: 0.139 sec\",\"job_status\": \"done\",\"duration\": 0.139,\"completed_at\": \"2018-04-03T22:57:22.071Z\",\"db_duration\": 0.05,\"db_duration_s\": 0.0005,\"gitaly_duration\": 0,\"gitaly_calls\": 0}", + "severity": 6, + "start": "2018-04-03T22:57:21.930Z", + "type": [ + "info" + ] + }, + "gitlab": { + "gitaly": { + "calls": 0, + "duration": 0 + }, + "sidekiq": { + "class": "UpdateAllMirrorsWorker", + "db": { + "duration_m": 0.05, + "duration_s": 0.0005 + }, + "enqueued_at": "2018-04-03T22:57:21.931Z", + "jid": "06aeaa3b0aadacf9981f368e", + "job_status": "done", + "queue": "cronjob:update_all_mirrors", + "queue_namespace": "cronjob", + "retry": false, + "worker_id": "sidekiq_0" + } + }, + "input": { + "type": "filestream" + }, + "log": { + "file": { + "device_id": "30", + "inode": "215", + "path": "/tmp/service_logs/test-gitlab-sidekiq.log" + }, + "offset": 0 + }, + "message": "UpdateAllMirrorsWorker JID-06aeaa3b0aadacf9981f368e: done: 0.139 sec", + "process": { + "pid": 10077 + }, + "tags": [ + "preserve_original_event", + "forwarded", + "gitlab-sidekiq" + ] +} \ No newline at end of file diff --git a/packages/gitlab/docs/README.md b/packages/gitlab/docs/README.md index 3d6104f903bc..c987823408ac 100644 --- a/packages/gitlab/docs/README.md +++ b/packages/gitlab/docs/README.md @@ -10,8 +10,12 @@ This integration is for ingesting logs from [GitLab](https://about.gitlab.com/). - `auth`: Collect logs for protected paths abusive requests or requests over the Rate Limit. +- `pages`: Collect logs for Pages + - `production`: Collect logs for Rails controller requests received from GitLab. +- `sidekiq`: Collect logs from [sidekiq](https://sidekiq.org/) for jobs background jobs that take a long time + See the GitLab [Log system docs](https://docs.gitlab.com/ee/administration/logs/) for more information. ## Compatibility @@ -974,6 +978,99 @@ An example event for `auth` looks as following: } ``` +### pages + +Collect logs for Pages. Check out the [GitLab Pages log docs](https://docs.gitlab.com/ee/administration/logs/#pages-logs) for more information. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| data_stream.dataset | Data stream dataset name. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | +| gitlab.pages.in_place | | boolean | +| gitlab.pages.revision | | keyword | +| gitlab.pages.version | | keyword | +| host.containerized | If the host is a container. | boolean | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| input.type | Type of Filebeat input. | keyword | +| log.file.device_id | ID of the device containing the filesystem where the file resides. | keyword | +| log.file.inode | Inode number of the log file. | keyword | +| log.flags | Flags for the log file. | keyword | +| log.offset | Offset of the entry in the log file. | long | + + +An example event for `pages` looks as following: + +```json +{ + "@timestamp": "2020-04-22T17:53:12.000Z", + "agent": { + "ephemeral_id": "9c660d07-7fab-4b63-baad-3d850d847ec1", + "id": "20a743fb-5e21-4498-ba58-6ebb10053f90", + "name": "elastic-agent-63322", + "type": "filebeat", + "version": "8.15.0" + }, + "data_stream": { + "dataset": "gitlab.pages", + "namespace": "81772", + "type": "logs" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "20a743fb-5e21-4498-ba58-6ebb10053f90", + "snapshot": false, + "version": "8.15.0" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "web" + ], + "dataset": "gitlab.pages", + "ingested": "2024-09-23T20:44:09Z", + "kind": "event", + "level": 6, + "original": "{\"level\": \"info\",\"msg\": \"GitLab Pages Daemon\",\"revision\": \"52b2899\",\"time\": \"2020-04-22T17:53:12Z\",\"version\": \"1.17.0\"}", + "type": [ + "info" + ] + }, + "gitlab": { + "pages": { + "revision": "52b2899", + "version": "1.17.0" + } + }, + "input": { + "type": "filestream" + }, + "log": { + "file": { + "device_id": "30", + "inode": "163", + "path": "/tmp/service_logs/test-gitlab-pages.log" + }, + "offset": 0 + }, + "message": "GitLab Pages Daemon", + "tags": [ + "preserve_original_event", + "forwarded", + "gitlab-pages" + ] +} +``` + ### production Collect logs for Rails controller requests received from GitLab. Check out the [GitLab production log docs](https://docs.gitlab.com/ee/administration/logs/#production_jsonlog) for more information. @@ -1270,3 +1367,126 @@ An example event for `production` looks as following: } } ``` + +### sidekiq + +Collect logs from sidekiq for jobs background jobs that take a long time. Check out the [GitLab sidekiq log docs](https://docs.gitlab.com/ee/administration/logs/#sidekiq-logs) for more information. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| data_stream.dataset | Data stream dataset name. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | +| gitlab.gitaly.calls | | long | +| gitlab.gitaly.duration | | long | +| gitlab.sidekiq.args | | keyword | +| gitlab.sidekiq.class | | keyword | +| gitlab.sidekiq.db.duration_m | | float | +| gitlab.sidekiq.db.duration_s | | float | +| gitlab.sidekiq.enqueued_at | | date | +| gitlab.sidekiq.jid | | keyword | +| gitlab.sidekiq.job_status | | keyword | +| gitlab.sidekiq.queue | | keyword | +| gitlab.sidekiq.queue_namespace | | keyword | +| gitlab.sidekiq.retry | | boolean | +| gitlab.sidekiq.worker_id | | keyword | +| host.containerized | If the host is a container. | boolean | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| input.type | Type of Filebeat input. | keyword | +| log.file.device_id | ID of the device containing the filesystem where the file resides. | keyword | +| log.file.inode | Inode number of the log file. | keyword | +| log.flags | Flags for the log file. | keyword | +| log.offset | Offset of the entry in the log file. | long | + + +An example event for `sidekiq` looks as following: + +```json +{ + "@timestamp": "2018-04-03T22:57:22.071Z", + "agent": { + "ephemeral_id": "cc5ea64a-1be7-4bf7-ac2e-a934734ba7d0", + "id": "d0298772-a948-4edb-95bf-6b9152967f34", + "name": "elastic-agent-16126", + "type": "filebeat", + "version": "8.15.0" + }, + "data_stream": { + "dataset": "gitlab.sidekiq", + "namespace": "99205", + "type": "logs" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "d0298772-a948-4edb-95bf-6b9152967f34", + "snapshot": false, + "version": "8.15.0" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "web" + ], + "dataset": "gitlab.sidekiq", + "duration": 0.139, + "end": "2018-04-03T22:57:22.071Z", + "ingested": "2024-09-23T21:03:08Z", + "kind": "event", + "original": "{\"severity\": \"INFO\",\"time\": \"2018-04-03T22:57:22.071Z\",\"queue\": \"cronjob:update_all_mirrors\",\"args\": [],\"class\": \"UpdateAllMirrorsWorker\",\"retry\": false,\"queue_namespace\": \"cronjob\",\"jid\": \"06aeaa3b0aadacf9981f368e\",\"created_at\": \"2018-04-03T22:57:21.930Z\",\"enqueued_at\": \"2018-04-03T22:57:21.931Z\",\"pid\": 10077,\"worker_id\": \"sidekiq_0\",\"message\": \"UpdateAllMirrorsWorker JID-06aeaa3b0aadacf9981f368e: done: 0.139 sec\",\"job_status\": \"done\",\"duration\": 0.139,\"completed_at\": \"2018-04-03T22:57:22.071Z\",\"db_duration\": 0.05,\"db_duration_s\": 0.0005,\"gitaly_duration\": 0,\"gitaly_calls\": 0}", + "severity": 6, + "start": "2018-04-03T22:57:21.930Z", + "type": [ + "info" + ] + }, + "gitlab": { + "gitaly": { + "calls": 0, + "duration": 0 + }, + "sidekiq": { + "class": "UpdateAllMirrorsWorker", + "db": { + "duration_m": 0.05, + "duration_s": 0.0005 + }, + "enqueued_at": "2018-04-03T22:57:21.931Z", + "jid": "06aeaa3b0aadacf9981f368e", + "job_status": "done", + "queue": "cronjob:update_all_mirrors", + "queue_namespace": "cronjob", + "retry": false, + "worker_id": "sidekiq_0" + } + }, + "input": { + "type": "filestream" + }, + "log": { + "file": { + "device_id": "30", + "inode": "215", + "path": "/tmp/service_logs/test-gitlab-sidekiq.log" + }, + "offset": 0 + }, + "message": "UpdateAllMirrorsWorker JID-06aeaa3b0aadacf9981f368e: done: 0.139 sec", + "process": { + "pid": 10077 + }, + "tags": [ + "preserve_original_event", + "forwarded", + "gitlab-sidekiq" + ] +} +``` From bb03859bc1bc651ec38b57f2e6c209e4dacd9d70 Mon Sep 17 00:00:00 2001 From: tehbooom Date: Tue, 24 Sep 2024 08:56:10 -0400 Subject: [PATCH 2/5] chore: Update PR number and add changelog --- packages/gitlab/changelog.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/gitlab/changelog.yml b/packages/gitlab/changelog.yml index e8e6e1695385..7c62677d60d2 100644 --- a/packages/gitlab/changelog.yml +++ b/packages/gitlab/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: 0.4.0 + changes: + - description: Add sidekiq and pages datastreams + type: enhancement + link: https://github.com/elastic/integrations/pull/11234 - version: 0.3.1 changes: - description: Make path configuration consistent between data streams. From f9253eb3280198673ec165eace842bc7c97e9f07 Mon Sep 17 00:00:00 2001 From: tehbooom Date: Tue, 24 Sep 2024 09:10:03 -0400 Subject: [PATCH 3/5] chore: Bump manifest version --- packages/gitlab/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gitlab/manifest.yml b/packages/gitlab/manifest.yml index c4b15d15a570..72dc64fb0b32 100644 --- a/packages/gitlab/manifest.yml +++ b/packages/gitlab/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.1.3 name: gitlab title: GitLab -version: 0.3.1 +version: 0.4.0 description: Collect logs from GitLab with Elastic Agent. type: integration categories: From eedaef0e5c2dea35a9bc593b83884d4c57b2fc2f Mon Sep 17 00:00:00 2001 From: tehbooom Date: Tue, 24 Sep 2024 09:34:13 -0400 Subject: [PATCH 4/5] test: Bump Kibana version --- packages/gitlab/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gitlab/manifest.yml b/packages/gitlab/manifest.yml index 72dc64fb0b32..e42b6d9a0e98 100644 --- a/packages/gitlab/manifest.yml +++ b/packages/gitlab/manifest.yml @@ -9,7 +9,7 @@ categories: - productivity_security conditions: kibana: - version: ^8.13.0 + version: ^8.14.0 icons: - src: /img/gitlab-logo.svg title: gitlab Logo From c52e99e5781abfd4f0537a58b6ba7eb6ef3714ea Mon Sep 17 00:00:00 2001 From: tehbooom Date: Fri, 27 Sep 2024 08:05:32 -0400 Subject: [PATCH 5/5] fix: Revert Kibana version to 8.13.0 --- packages/gitlab/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gitlab/manifest.yml b/packages/gitlab/manifest.yml index e42b6d9a0e98..72dc64fb0b32 100644 --- a/packages/gitlab/manifest.yml +++ b/packages/gitlab/manifest.yml @@ -9,7 +9,7 @@ categories: - productivity_security conditions: kibana: - version: ^8.14.0 + version: ^8.13.0 icons: - src: /img/gitlab-logo.svg title: gitlab Logo