Update "aws" integration (version: 0.2.7)

elastic · Aug 24, 2020 · 23a00f1 · 23a00f1
1 parent 2cb0381
commit 23a00f1
Show file tree

Hide file tree

Showing 65 changed files with 2,942 additions and 2,672 deletions.
diff --git a/packages/aws/0.2.7/dataset/billing/agent/stream/stream.yml.hbs b/packages/aws/0.2.7/dataset/billing/agent/stream/stream.yml.hbs
@@ -21,3 +21,6 @@ role_arn: {{role_arn}}
 {{#if regions}}
 regions: {{regions}}
 {{/if}}
+{{#if aws_partition}}
+aws_partition: {{aws_partition}}
+{{/if}}
diff --git a/packages/aws/0.2.7/dataset/billing/fields/base-fields.yml b/packages/aws/0.2.7/dataset/billing/fields/base-fields.yml
@@ -10,4 +10,3 @@
 - name: '@timestamp'
   type: date
   description: Event timestamp.
-
diff --git a/packages/aws/0.2.7/dataset/billing/fields/fields.yml b/packages/aws/0.2.7/dataset/billing/fields/fields.yml
@@ -4,12 +4,12 @@
   - name: dimensions
     type: group
     fields:
-      - name: ServiceName
-        type: keyword
-        description: AWS service name.
-      - name: Currency
-        type: keyword
-        description: Currency name.
+    - name: ServiceName
+      type: keyword
+      description: AWS service name.
+    - name: Currency
+      type: keyword
+      description: Currency name.
   - name: billing
     type: group
     fields:

diff --git a/packages/aws/0.2.7/dataset/billing/sample_event.json b/packages/aws/0.2.7/dataset/billing/sample_event.json
@@ -1,67 +1,67 @@
 {
-  "_index": "metrics-aws.billing-default-000001",
-  "_id": "IMxJXHIBpGMSUzkZo-s0",
-  "_version": 1,
-  "_score": null,
-  "_source": {
-    "@timestamp": "2020-05-28T17:17:06.212Z",
-    "cloud": {
-      "provider": "aws",
-      "region": "us-east-1",
-      "account": {
-        "id": "428152502467",
-        "name": "elastic-beats"
-      }
-    },
-    "event": {
-      "dataset": "aws.billing",
-      "module": "aws",
-      "duration": 1938760247
-    },
-    "metricset": {
-      "name": "billing",
-      "period": 43200000
-    },
-    "ecs": {
-      "version": "1.5.0"
-    },
-    "aws": {
-      "billing": {
-        "metrics": {
-          "EstimatedCharges": {
-            "max": 1625.41
-          }
+    "_index": "metrics-aws.billing-default-000001",
+    "_id": "IMxJXHIBpGMSUzkZo-s0",
+    "_version": 1,
+    "_score": null,
+    "_source": {
+        "@timestamp": "2020-05-28T17:17:06.212Z",
+        "cloud": {
+            "provider": "aws",
+            "region": "us-east-1",
+            "account": {
+                "id": "428152502467",
+                "name": "elastic-beats"
+            }
+        },
+        "event": {
+            "dataset": "aws.billing",
+            "module": "aws",
+            "duration": 1938760247
+        },
+        "metricset": {
+            "name": "billing",
+            "period": 43200000
+        },
+        "ecs": {
+            "version": "1.5.0"
+        },
+        "aws": {
+            "billing": {
+                "metrics": {
+                    "EstimatedCharges": {
+                        "max": 1625.41
+                    }
+                }
+            },
+            "cloudwatch": {
+                "namespace": "AWS/Billing"
+            },
+            "dimensions": {
+                "Currency": "USD"
+            }
+        },
+        "service": {
+            "type": "aws"
+        },
+        "stream": {
+            "type": "metrics",
+            "dataset": "aws.billing",
+            "namespace": "default"
+        },
+        "agent": {
+            "id": "12f376ef-5186-4e8b-a175-70f1140a8f30",
+            "name": "MacBook-Elastic.local",
+            "type": "metricbeat",
+            "version": "8.0.0",
+            "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b"
         }
-      },
-      "cloudwatch": {
-        "namespace": "AWS/Billing"
-      },
-      "dimensions": {
-        "Currency": "USD"
-      }
-    },
-    "service": {
-      "type": "aws"
     },
-    "stream": {
-      "type": "metrics",
-      "dataset": "aws.billing",
-      "namespace": "default"
+    "fields": {
+        "@timestamp": [
+            "2020-05-28T17:17:06.212Z"
+        ]
     },
-    "agent": {
-      "id": "12f376ef-5186-4e8b-a175-70f1140a8f30",
-      "name": "MacBook-Elastic.local",
-      "type": "metricbeat",
-      "version": "8.0.0",
-      "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b"
-    }
-  },
-  "fields": {
-    "@timestamp": [
-      "2020-05-28T17:17:06.212Z"
+    "sort": [
+        1590686226212
     ]
-  },
-  "sort": [
-    1590686226212
-  ]
 }
diff --git a/packages/aws/0.2.7/dataset/cloudtrail/agent/stream/s3.yml.hbs b/packages/aws/0.2.7/dataset/cloudtrail/agent/stream/s3.yml.hbs
@@ -27,6 +27,9 @@ session_token: {{session_token}}
 {{#if role_arn}}
 role_arn: {{role_arn}}
 {{/if}}
+{{#if aws_partition}}
+aws_partition: {{aws_partition}}
+{{/if}}
 processors:
   - add_fields:
       target: ''

diff --git a/packages/aws/0.2.7/dataset/cloudtrail/elasticsearch/ingest_pipeline/default.yml b/packages/aws/0.2.7/dataset/cloudtrail/elasticsearch/ingest_pipeline/default.yml
@@ -1,6 +1,9 @@
 ---
 description: Pipeline for AWS CloudTrail Logs
 processors:
+  - set:
+      field: event.ingested
+      value: '{{_ingest.timestamp}}'
   - rename:
       field: "message"
       target_field: "event.original"
@@ -84,6 +87,7 @@ processors:
       field: "event.action"
       value: "{{json.eventName}}"
       ignore_failure: true
+      ignore_empty_value: true
   - rename:
       field: "json.awsRegion"
       target_field: "cloud.region"
@@ -134,25 +138,37 @@ processors:
       field: "json.errorMessage"
       target_field: "aws.cloudtrail.error_message"
       ignore_failure: true
+  - rename:
+      field: json.requestParameters
+      target_field: "aws.cloudtrail.flattened.request_parameters"
+      if: ctx.json.requestParameters != null
   - script:
       lang: painless
       source: |
-        if (ctx.json.requestParameters != null) {
-          ctx.aws.cloudtrail.request_parameters = ctx.json.requestParameters.toString();
+        if (ctx.aws.cloudtrail.flattened.request_parameters != null) {
+          ctx.aws.cloudtrail.request_parameters = ctx.aws.cloudtrail.flattened.request_parameters.toString();
         }
       ignore_failure: true
+  - rename:
+      field: json.responseElements
+      target_field: "aws.cloudtrail.flattened.response_elements"
+      if: ctx.json.responseElements != null
   - script:
       lang: painless
       source: |
-        if (ctx.json.responseElements != null) {
-          ctx.aws.cloudtrail.response_elements = ctx.json.responseElements.toString();
+        if (ctx.aws.cloudtrail.flattened.response_elements != null) {
+          ctx.aws.cloudtrail.response_elements = ctx.aws.cloudtrail.flattened.response_elements.toString();
         }
       ignore_failure: true
+  - rename:
+      field: json.additionalEventData
+      target_field: "aws.cloudtrail.flattened.additional_eventdata"
+      if: ctx?.json?.additionalEventData != null
   - script:
       lang: painless
       source: |
-        if (ctx.json.additionalEventData != null) {
-          ctx.aws.cloudtrail.additional_eventdata = ctx.json.additionalEventData.toString();
+        if (ctx.aws.cloudtrail.flattened.additional_eventdata != null) {
+          ctx.aws.cloudtrail.additional_eventdata = ctx.aws.cloudtrail.flattened.additional_eventdata.toString();
         }
       ignore_failure: true
   - rename:
@@ -195,11 +211,15 @@ processors:
       field: "json.recipientAccountId"
       target_field: "aws.cloudtrail.recipient_account_id"
       ignore_failure: true
+  - rename:
+      field: json.serviceEventDetails
+      target_field: "aws.cloudtrail.flattened.service_event_details"
+      if: ctx.json.serviceEventDetails != null
   - script:
       lang: painless
       source: |
-        if (ctx.json.serviceEventDetails != null) {
-          ctx.aws.cloudtrail.service_event_details = ctx.json.serviceEventDetails.toString();
+        if (ctx.aws.cloudtrail.flattened.service_event_details != null) {
+          ctx.aws.cloudtrail.service_event_details = ctx.aws.cloudtrail.flattened.service_event_details.toString();
         }
       ignore_failure: true
   - rename:
@@ -225,14 +245,11 @@ processors:
           }
           ctx.related.user.add(userName);
         }
-
-
-
-        if (ctx.json?.requestParameters.userName != null) {
-          addRelatedUser(ctx, ctx.json.requestParameters.userName);
+        if (ctx?.aws?.cloudtrail?.flattened?.request_parameters?.userName != null) {
+          addRelatedUser(ctx, ctx.aws.cloudtrail.flattened.request_parameters.userName);
         }
-        if (ctx.json?.requestParameters.newUserName != null) {
-          addRelatedUser(ctx, ctx.json.requestParameters.newUserName);
+        if (ctx?.aws?.cloudtrail?.flattened?.request_parameters?.newUserName != null) {
+          addRelatedUser(ctx, ctx.aws.cloudtrail.flattened.request_parameters.newUserName);
         }
 
   - script:
@@ -243,18 +260,18 @@ processors:
           return;
         }
         Map aed_map = new HashMap();
-        if (ctx.json?.additionalEventData?.MobileVersion != null) {
-          if (ctx.json.additionalEventData.MobileVersion == 'No') {
+        if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.MobileVersion != null) {
+          if (ctx.aws.cloudtrail.flattened.additional_eventdata.MobileVersion == 'No') {
             aed_map.put("mobile_version", false);
           } else {
             aed_map.put("mobile_version", true);
           }
         }
-        if (ctx.json?.additionalEventData?.LoginTo != null) {
-          aed_map.put("login_to", ctx.json.additionalEventData.LoginTo);
+        if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.LoginTo != null) {
+          aed_map.put("login_to", ctx.aws.cloudtrail.flattened.additional_eventdata.LoginTo);
         }
-        if (ctx.json?.additionalEventData?.MFAUsed != null) {
-          if (ctx.json.additionalEventData.MFAUsed == 'No') {
+        if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.MFAUsed != null) {
+          if (ctx.aws.cloudtrail.flattened.additional_eventdata.MFAUsed == 'No') {
             aed_map.put("mfa_used", false);
           } else {
             aed_map.put("mfa_used", true);
@@ -451,6 +468,12 @@ processors:
           type:
             - user
             - info
+        ListGroupsForUser:
+          category:
+            - iam
+          type:
+            - user
+            - info
         ListGroupPolicies:
           category:
             - iam
@@ -591,10 +614,10 @@ processors:
             return;
         }
 
-        if (ctx.event.action == 'ConsoleLogin' && ctx.json?.responseElements.ConsoleLogin != null) {
-            ctx.event.outcome = Processors.lowercase(ctx.json.responseElements.ConsoleLogin);
+        if (ctx.event.action == 'ConsoleLogin' && ctx?.aws?.cloudtrail?.flattened?.response_elements.ConsoleLogin != null) {
+            ctx.event.outcome = Processors.lowercase(ctx.aws.cloudtrail.flattened.response_elements.ConsoleLogin);
         }
-        
+
         def hm = new HashMap(params.get(ctx.event.action));
         hm.forEach((k, v) -> ctx.event[k] = v);