[core.logging] Add RewriteAppender for filtering LogMeta.

docs/development/core/server/kibana-plugin-core-server.appenderconfigtype.md

@@ -8,5 +8,5 @@
 <b>Signature:</b>
 
 ```typescript
-export declare type AppenderConfigType = ConsoleAppenderConfig | FileAppenderConfig | LegacyAppenderConfig | RollingFileAppenderConfig;
+export declare type AppenderConfigType = ConsoleAppenderConfig | FileAppenderConfig | LegacyAppenderConfig | RewriteAppenderConfig | RollingFileAppenderConfig;
 ```

packages/kbn-logging/src/appenders.ts

@@ -15,6 +15,24 @@ import { LogRecord } from './log_record';
  */
 export interface Appender {
   append(record: LogRecord): void;
+  /**
+   * Appenders can be "attached" to one another so that they are able to act
+   * as a sort of middleware by calling `append` on a different appender.
+   *
+   * As appenders cannot be attached to each other until they are configured,
+   * the `addAppender` method can be used to pass in a newly configured appender
+   * to attach.
+   */
+  addAppender?(appenderRef: string, appender: Appender): void;
+  /**
+   * For appenders which implement `addAppender`, they should declare a list of
+   * `appenderRefs`, which specify the names of the appenders that their configuration
+   * depends on.
+   *
+   * Note that these are the appender key names that the user specifies in their
+   * config, _not_ the names of the appender types themselves.
+   */
+  appenderRefs?: string[];
 }
 
 /**

src/core/server/http/integration_tests/logging.test.ts

@@ -251,7 +251,7 @@ describe('request logging', () => {
           expect(JSON.parse(meta).http.response.headers.bar).toBe('world');
         });
 
-        it('filters sensitive request headers', async () => {
+        it('filters sensitive request headers by default', async () => {
           const { http } = await root.setup();
 
           http.createRouter('/').post(
@@ -283,7 +283,139 @@ describe('request logging', () => {
           expect(JSON.parse(meta).http.request.headers.authorization).toBe('[REDACTED]');
         });
 
-        it('filters sensitive response headers', async () => {
+        it('filters sensitive request headers when RewriteAppender is configured', async () => {
+          root = kbnTestServer.createRoot({
+            logging: {
+              silent: true,
+              appenders: {
+                'test-console': {
+                  type: 'console',
+                  layout: {
+                    type: 'pattern',
+                    pattern: '%level|%logger|%message|%meta',
+                  },
+                },
+                rewrite: {
+                  type: 'rewrite',
+                  appenders: ['test-console'],
+                  policy: {
+                    type: 'meta',
+                    mode: 'update',
+                    properties: [
+                      { path: 'http.request.headers.authorization', value: '[REDACTED]' },
+                    ],
+                  },
+                },
+              },
+              loggers: [
+                {
+                  name: 'http.server.response',
+                  appenders: ['rewrite'],
+                  level: 'debug',
+                },
+              ],
+            },
+            plugins: {
+              initialize: false,
+            },
+          });
+          const { http } = await root.setup();
+
+          http.createRouter('/').post(
+            {
+              path: '/ping',
+              validate: {
+                body: schema.object({ message: schema.string() }),
+              },
+              options: {
+                authRequired: 'optional',
+                body: {
+                  accepts: ['application/json'],
+                },
+                timeout: { payload: 100 },
+              },
+            },
+            (context, req, res) => res.ok({ body: { message: req.body.message } })
+          );
+          await root.start();
+
+          await kbnTestServer.request
+            .post(root, '/ping')
+            .set('content-type', 'application/json')
+            .set('authorization', 'abc')
+            .send({ message: 'hi' })
+            .expect(200);
+          expect(mockConsoleLog).toHaveBeenCalledTimes(1);
+          const [, , , meta] = mockConsoleLog.mock.calls[0][0].split('|');
+          expect(JSON.parse(meta).http.request.headers.authorization).toBe('[REDACTED]');
+        });
+
+        it('filters sensitive response headers by defaut', async () => {
+          const { http } = await root.setup();
+
+          http.createRouter('/').post(
+            {
+              path: '/ping',
+              validate: {
+                body: schema.object({ message: schema.string() }),
+              },
+              options: {
+                authRequired: 'optional',
+                body: {
+                  accepts: ['application/json'],
+                },
+                timeout: { payload: 100 },
+              },
+            },
+            (context, req, res) =>
+              res.ok({ headers: { 'set-cookie': ['123'] }, body: { message: req.body.message } })
+          );
+          await root.start();
+
+          await kbnTestServer.request
+            .post(root, '/ping')
+            .set('Content-Type', 'application/json')
+            .send({ message: 'hi' })
+            .expect(200);
+          expect(mockConsoleLog).toHaveBeenCalledTimes(1);
+          const [, , , meta] = mockConsoleLog.mock.calls[0][0].split('|');
+          expect(JSON.parse(meta).http.response.headers['set-cookie']).toBe('[REDACTED]');
+        });
+
+        it('filters sensitive response headers when RewriteAppender is configured', async () => {
+          root = kbnTestServer.createRoot({
+            logging: {
+              silent: true,
+              appenders: {
+                'test-console': {
+                  type: 'console',
+                  layout: {
+                    type: 'pattern',
+                    pattern: '%level|%logger|%message|%meta',
+                  },
+                },
+                rewrite: {
+                  type: 'rewrite',
+                  appenders: ['test-console'],
+                  policy: {
+                    type: 'meta',
+                    mode: 'update',
+                    properties: [{ path: 'http.response.headers.set-cookie', value: '[REDACTED]' }],
+                  },
+                },
+              },
+              loggers: [
+                {
+                  name: 'http.server.response',
+                  appenders: ['rewrite'],
+                  level: 'debug',
+                },
+              ],
+            },
+            plugins: {
+              initialize: false,
+            },
+          });
           const { http } = await root.setup();
 
           http.createRouter('/').post(

src/core/server/http/logging/get_response_log.ts

@@ -45,7 +45,12 @@ export function getEcsResponseLog(request: Request, log: Logger): LogMeta {
 
   // eslint-disable-next-line @typescript-eslint/naming-convention
   const status_code = isBoom(response) ? response.output.statusCode : response.statusCode;
-  const responseHeaders = isBoom(response) ? response.output.headers : response.headers;
+
+  // shallow clone the headers so they are not mutated if filtered by a RewriteAppender
+  const requestHeaders = { ...request.headers };
+  const responseHeaders = isBoom(response)
+    ? { ...response.output.headers }
+    : { ...response.headers };
 
   // borrowed from the hapi/good implementation
   const responseTime = (request.info.completed || request.info.responded) - request.info.received;
@@ -66,7 +71,7 @@ export function getEcsResponseLog(request: Request, log: Logger): LogMeta {
         mime_type: request.mime,
         referrer: request.info.referrer,
         // @ts-expect-error Headers are not yet part of ECS: https://github.com/elastic/ecs/issues/232.
-        headers: redactSensitiveHeaders(request.headers),
+        headers: redactSensitiveHeaders(requestHeaders),
       },
       response: {
         body: {

src/core/server/logging/README.md

@@ -264,6 +264,127 @@ The maximum number of files to keep. Once this number is reached, oldest files w
 
 The default value is `7`
 
+### Rewrite Appender
+
+*This appender is currently considered experimental and is not intended
+for public consumption. The API is subject to change at any time.*
+
+Similar to log4j's `RewriteAppender`, this appender serves as a sort of middleware,
+modifying the provided log events before passing them along to another
+appender.
+
+```yaml
+logging:
+  appenders:
+    my-rewrite-appender:
+      type: rewrite
+      appenders: [console, file] # name of "destination" appender(s)
+      policy:
+        # ...
+```
+
+The most common use case for the `RewriteAppender` is when you want to
+filter or censor sensitive data that may be contained in a log entry.
+In fact, with a default configuration, Kibana will automatically redact
+any `authorization`, `cookie`, or `set-cookie` headers when logging http
+requests & responses.
+
+To configure additional rewrite rules, you'll need to specify a `RewritePolicy`.
+
+#### Rewrite Policies
+
+Rewrite policies exist to indicate which parts of a log record can be
+modified within the rewrite appender.
+
+**Meta**
+
+The `meta` rewrite policy can read and modify any data contained in the
+`LogMeta` before passing it along to a destination appender.
+
+Meta policies must specify one of three modes, which indicate which action
+to perform on the configured properties:
+- `add` creates a new property at the provided `path`, skipping properties which already exist.
+- `update` updates an existing property at the provided `path` without creating new properties.
+- `remove` removes an existing property at the provided `path`.
+
+The `properties` are listed as a `path` and `value` pair, where `path` is
+the dot-delimited path to the target property in the `LogMeta` object, and
+`value` is the value to add or update in that target property. When using
+the `remove` mode, a `value` is not necessary.
+
+Here's an example of how you would replace any `cookie` header values with `[REDACTED]`:
+
+```yaml
+logging:
+  appenders:
+    my-rewrite-appender:
+      type: rewrite
+      appenders: [console]
+      policy:
+        type: meta # indicates that we want to rewrite the LogMeta
+        mode: update # will update an existing property only
+        properties:
+          - path: "http.request.headers.cookie" # path to property
+            value: "[REDACTED]" # value to replace at path
+```
+
+Rewrite appenders can even be passed to other rewrite appenders to apply
+multiple filter policies/modes, as long as it doesn't create a circular
+reference. Each rewrite appender is applied sequentially (one after the other).
+```yaml
+logging:
+  appenders:
+    remove-stuff:
+      type: rewrite
+      appenders: [add-stuff] # redirect to the next rewrite appender
+      policy:
+        type: meta
+        mode: remove
+        properties:
+          - path: "http.request.headers.authorization"
+          - path: "http.request.headers.cookie"
+          - path: "http.request.headers.set-cookie"
+    add-stuff:
+      type: rewrite
+      appenders: [console] # output to console
+      policy:
+        type: meta
+        mode: add
+        properties:
+          - path: "hello"
+            value: "world" # creates { hello: 'world' } at the LogMeta root
+```
+
+#### Complete Example
+```yaml
+logging:
+  appenders:
+    console:
+      type: console
+      layout:
+        type: pattern
+        highlight: true
+        pattern: "[%date][%level][%logger] %message %meta"
+    file:
+      type: file
+      fileName: ./kibana.log
+      layout:
+        type: json
+    censor:
+      type: rewrite
+      appenders: [console, file]
+      policy:
+        type: meta
+        mode: update
+        properties:
+          - path: "http.request.headers.cookie"
+            value: "[REDACTED]"
+  loggers:
+    - name: http.server.response
+      appenders: [censor] # pass these logs to our rewrite appender
+      level: debug
+```
+
 ## Configuration
 
 As any configuration in the platform, logging configuration is validated against the predefined schema and if there are

src/core/server/logging/appenders/appenders.ts

@@ -17,6 +17,7 @@ import {
 import { Layouts } from '../layouts/layouts';
 import { ConsoleAppender, ConsoleAppenderConfig } from './console/console_appender';
 import { FileAppender, FileAppenderConfig } from './file/file_appender';
+import { RewriteAppender, RewriteAppenderConfig } from './rewrite/rewrite_appender';
 import {
   RollingFileAppender,
   RollingFileAppenderConfig,
@@ -32,6 +33,7 @@ export const appendersSchema = schema.oneOf([
   ConsoleAppender.configSchema,
   FileAppender.configSchema,
   LegacyAppender.configSchema,
+  RewriteAppender.configSchema,
   RollingFileAppender.configSchema,
 ]);
 
@@ -40,6 +42,7 @@ export type AppenderConfigType =
   | ConsoleAppenderConfig
   | FileAppenderConfig
   | LegacyAppenderConfig
+  | RewriteAppenderConfig
   | RollingFileAppenderConfig;
 
 /** @internal */
@@ -57,6 +60,8 @@ export class Appenders {
         return new ConsoleAppender(Layouts.create(config.layout));
       case 'file':
         return new FileAppender(Layouts.create(config.layout), config.fileName);
+      case 'rewrite':
+        return new RewriteAppender(config);
       case 'rolling-file':
         return new RollingFileAppender(config);
       case 'legacy-appender':