diff --git a/docs/development/core/server/kibana-plugin-core-server.appenderconfigtype.md b/docs/development/core/server/kibana-plugin-core-server.appenderconfigtype.md
index a50df950628b3a..f6de959589eca7 100644
--- a/docs/development/core/server/kibana-plugin-core-server.appenderconfigtype.md
+++ b/docs/development/core/server/kibana-plugin-core-server.appenderconfigtype.md
@@ -8,5 +8,5 @@
Signature:
```typescript
-export declare type AppenderConfigType = ConsoleAppenderConfig | FileAppenderConfig | LegacyAppenderConfig | RollingFileAppenderConfig;
+export declare type AppenderConfigType = ConsoleAppenderConfig | FileAppenderConfig | LegacyAppenderConfig | RewriteAppenderConfig | RollingFileAppenderConfig;
```
diff --git a/packages/kbn-logging/src/appenders.ts b/packages/kbn-logging/src/appenders.ts
index 1b128c0f292018..48422db34b3365 100644
--- a/packages/kbn-logging/src/appenders.ts
+++ b/packages/kbn-logging/src/appenders.ts
@@ -15,6 +15,24 @@ import { LogRecord } from './log_record';
*/
export interface Appender {
append(record: LogRecord): void;
+ /**
+ * Appenders can be "attached" to one another so that they are able to act
+ * as a sort of middleware by calling `append` on a different appender.
+ *
+ * As appenders cannot be attached to each other until they are configured,
+ * the `addAppender` method can be used to pass in a newly configured appender
+ * to attach.
+ */
+ addAppender?(appenderRef: string, appender: Appender): void;
+ /**
+ * For appenders which implement `addAppender`, they should declare a list of
+ * `appenderRefs`, which specify the names of the appenders that their configuration
+ * depends on.
+ *
+ * Note that these are the appender key names that the user specifies in their
+ * config, _not_ the names of the appender types themselves.
+ */
+ appenderRefs?: string[];
}
/**
diff --git a/src/core/server/http/integration_tests/logging.test.ts b/src/core/server/http/integration_tests/logging.test.ts
index fcf2cd2ba3372d..62cb699bc49f6b 100644
--- a/src/core/server/http/integration_tests/logging.test.ts
+++ b/src/core/server/http/integration_tests/logging.test.ts
@@ -251,7 +251,7 @@ describe('request logging', () => {
expect(JSON.parse(meta).http.response.headers.bar).toBe('world');
});
- it('filters sensitive request headers', async () => {
+ it('filters sensitive request headers by default', async () => {
const { http } = await root.setup();
http.createRouter('/').post(
@@ -283,7 +283,139 @@ describe('request logging', () => {
expect(JSON.parse(meta).http.request.headers.authorization).toBe('[REDACTED]');
});
- it('filters sensitive response headers', async () => {
+ it('filters sensitive request headers when RewriteAppender is configured', async () => {
+ root = kbnTestServer.createRoot({
+ logging: {
+ silent: true,
+ appenders: {
+ 'test-console': {
+ type: 'console',
+ layout: {
+ type: 'pattern',
+ pattern: '%level|%logger|%message|%meta',
+ },
+ },
+ rewrite: {
+ type: 'rewrite',
+ appenders: ['test-console'],
+ policy: {
+ type: 'meta',
+ mode: 'update',
+ properties: [
+ { path: 'http.request.headers.authorization', value: '[REDACTED]' },
+ ],
+ },
+ },
+ },
+ loggers: [
+ {
+ name: 'http.server.response',
+ appenders: ['rewrite'],
+ level: 'debug',
+ },
+ ],
+ },
+ plugins: {
+ initialize: false,
+ },
+ });
+ const { http } = await root.setup();
+
+ http.createRouter('/').post(
+ {
+ path: '/ping',
+ validate: {
+ body: schema.object({ message: schema.string() }),
+ },
+ options: {
+ authRequired: 'optional',
+ body: {
+ accepts: ['application/json'],
+ },
+ timeout: { payload: 100 },
+ },
+ },
+ (context, req, res) => res.ok({ body: { message: req.body.message } })
+ );
+ await root.start();
+
+ await kbnTestServer.request
+ .post(root, '/ping')
+ .set('content-type', 'application/json')
+ .set('authorization', 'abc')
+ .send({ message: 'hi' })
+ .expect(200);
+ expect(mockConsoleLog).toHaveBeenCalledTimes(1);
+ const [, , , meta] = mockConsoleLog.mock.calls[0][0].split('|');
+ expect(JSON.parse(meta).http.request.headers.authorization).toBe('[REDACTED]');
+ });
+
+ it('filters sensitive response headers by defaut', async () => {
+ const { http } = await root.setup();
+
+ http.createRouter('/').post(
+ {
+ path: '/ping',
+ validate: {
+ body: schema.object({ message: schema.string() }),
+ },
+ options: {
+ authRequired: 'optional',
+ body: {
+ accepts: ['application/json'],
+ },
+ timeout: { payload: 100 },
+ },
+ },
+ (context, req, res) =>
+ res.ok({ headers: { 'set-cookie': ['123'] }, body: { message: req.body.message } })
+ );
+ await root.start();
+
+ await kbnTestServer.request
+ .post(root, '/ping')
+ .set('Content-Type', 'application/json')
+ .send({ message: 'hi' })
+ .expect(200);
+ expect(mockConsoleLog).toHaveBeenCalledTimes(1);
+ const [, , , meta] = mockConsoleLog.mock.calls[0][0].split('|');
+ expect(JSON.parse(meta).http.response.headers['set-cookie']).toBe('[REDACTED]');
+ });
+
+ it('filters sensitive response headers when RewriteAppender is configured', async () => {
+ root = kbnTestServer.createRoot({
+ logging: {
+ silent: true,
+ appenders: {
+ 'test-console': {
+ type: 'console',
+ layout: {
+ type: 'pattern',
+ pattern: '%level|%logger|%message|%meta',
+ },
+ },
+ rewrite: {
+ type: 'rewrite',
+ appenders: ['test-console'],
+ policy: {
+ type: 'meta',
+ mode: 'update',
+ properties: [{ path: 'http.response.headers.set-cookie', value: '[REDACTED]' }],
+ },
+ },
+ },
+ loggers: [
+ {
+ name: 'http.server.response',
+ appenders: ['rewrite'],
+ level: 'debug',
+ },
+ ],
+ },
+ plugins: {
+ initialize: false,
+ },
+ });
const { http } = await root.setup();
http.createRouter('/').post(
diff --git a/src/core/server/http/logging/get_response_log.test.ts b/src/core/server/http/logging/get_response_log.test.ts
index 46c4f1d95e3be0..64241ff44fc6ba 100644
--- a/src/core/server/http/logging/get_response_log.test.ts
+++ b/src/core/server/http/logging/get_response_log.test.ts
@@ -171,6 +171,53 @@ describe('getEcsResponseLog', () => {
});
test('does not mutate original headers', () => {
+ const reqHeaders = { a: 'foo', b: ['hello', 'world'] };
+ const resHeaders = { headers: { c: 'bar' } };
+ const req = createMockHapiRequest({
+ headers: reqHeaders,
+ response: { headers: resHeaders },
+ });
+
+ const responseLog = getEcsResponseLog(req, logger);
+ expect(reqHeaders).toMatchInlineSnapshot(`
+ Object {
+ "a": "foo",
+ "b": Array [
+ "hello",
+ "world",
+ ],
+ }
+ `);
+ expect(resHeaders).toMatchInlineSnapshot(`
+ Object {
+ "headers": Object {
+ "c": "bar",
+ },
+ }
+ `);
+
+ responseLog.http.request.headers.a = 'testA';
+ responseLog.http.request.headers.b[1] = 'testB';
+ responseLog.http.request.headers.c = 'testC';
+ expect(reqHeaders).toMatchInlineSnapshot(`
+ Object {
+ "a": "foo",
+ "b": Array [
+ "hello",
+ "world",
+ ],
+ }
+ `);
+ expect(resHeaders).toMatchInlineSnapshot(`
+ Object {
+ "headers": Object {
+ "c": "bar",
+ },
+ }
+ `);
+ });
+
+ test('does not mutate original headers when redacting sensitive data', () => {
const reqHeaders = { authorization: 'a', cookie: 'b', 'user-agent': 'hi' };
const resHeaders = { headers: { 'content-length': 123, 'set-cookie': 'c' } };
const req = createMockHapiRequest({
diff --git a/src/core/server/http/logging/get_response_log.ts b/src/core/server/http/logging/get_response_log.ts
index f75acde93bf405..57c02e05bebff2 100644
--- a/src/core/server/http/logging/get_response_log.ts
+++ b/src/core/server/http/logging/get_response_log.ts
@@ -18,14 +18,22 @@ const ECS_VERSION = '1.7.0';
const FORBIDDEN_HEADERS = ['authorization', 'cookie', 'set-cookie'];
const REDACTED_HEADER_TEXT = '[REDACTED]';
+type HapiHeaders = Record;
+
// We are excluding sensitive headers by default, until we have a log filtering mechanism.
-function redactSensitiveHeaders(
- headers?: Record
-): Record {
- const result = {} as Record;
+function redactSensitiveHeaders(key: string, value: string | string[]): string | string[] {
+ return FORBIDDEN_HEADERS.includes(key) ? REDACTED_HEADER_TEXT : value;
+}
+
+// Shallow clone the headers so they are not mutated if filtered by a RewriteAppender.
+function cloneAndFilterHeaders(headers?: HapiHeaders) {
+ const result = {} as HapiHeaders;
if (headers) {
for (const key of Object.keys(headers)) {
- result[key] = FORBIDDEN_HEADERS.includes(key) ? REDACTED_HEADER_TEXT : headers[key];
+ result[key] = redactSensitiveHeaders(
+ key,
+ Array.isArray(headers[key]) ? [...headers[key]] : headers[key]
+ );
}
}
return result;
@@ -45,7 +53,11 @@ export function getEcsResponseLog(request: Request, log: Logger): LogMeta {
// eslint-disable-next-line @typescript-eslint/naming-convention
const status_code = isBoom(response) ? response.output.statusCode : response.statusCode;
- const responseHeaders = isBoom(response) ? response.output.headers : response.headers;
+
+ const requestHeaders = cloneAndFilterHeaders(request.headers);
+ const responseHeaders = cloneAndFilterHeaders(
+ isBoom(response) ? (response.output.headers as HapiHeaders) : response.headers
+ );
// borrowed from the hapi/good implementation
const responseTime = (request.info.completed || request.info.responded) - request.info.received;
@@ -66,7 +78,7 @@ export function getEcsResponseLog(request: Request, log: Logger): LogMeta {
mime_type: request.mime,
referrer: request.info.referrer,
// @ts-expect-error Headers are not yet part of ECS: https://github.com/elastic/ecs/issues/232.
- headers: redactSensitiveHeaders(request.headers),
+ headers: requestHeaders,
},
response: {
body: {
@@ -74,7 +86,7 @@ export function getEcsResponseLog(request: Request, log: Logger): LogMeta {
},
status_code,
// @ts-expect-error Headers are not yet part of ECS: https://github.com/elastic/ecs/issues/232.
- headers: redactSensitiveHeaders(responseHeaders),
+ headers: responseHeaders,
// responseTime is a custom non-ECS field
responseTime: !isNaN(responseTime) ? responseTime : undefined,
},
diff --git a/src/core/server/logging/README.mdx b/src/core/server/logging/README.mdx
index 8c093d02315859..1575e67d7b8ee0 100644
--- a/src/core/server/logging/README.mdx
+++ b/src/core/server/logging/README.mdx
@@ -278,6 +278,124 @@ The maximum number of files to keep. Once this number is reached, oldest files w
The default value is `7`
+### Rewrite Appender
+
+*This appender is currently considered experimental and is not intended
+for public consumption. The API is subject to change at any time.*
+
+Similar to log4j's `RewriteAppender`, this appender serves as a sort of middleware,
+modifying the provided log events before passing them along to another
+appender.
+
+```yaml
+logging:
+ appenders:
+ my-rewrite-appender:
+ type: rewrite
+ appenders: [console, file] # name of "destination" appender(s)
+ policy:
+ # ...
+```
+
+The most common use case for the `RewriteAppender` is when you want to
+filter or censor sensitive data that may be contained in a log entry.
+In fact, with a default configuration, Kibana will automatically redact
+any `authorization`, `cookie`, or `set-cookie` headers when logging http
+requests & responses.
+
+To configure additional rewrite rules, you'll need to specify a `RewritePolicy`.
+
+#### Rewrite Policies
+
+Rewrite policies exist to indicate which parts of a log record can be
+modified within the rewrite appender.
+
+**Meta**
+
+The `meta` rewrite policy can read and modify any data contained in the
+`LogMeta` before passing it along to a destination appender.
+
+Meta policies must specify one of three modes, which indicate which action
+to perform on the configured properties:
+- `update` updates an existing property at the provided `path`.
+- `remove` removes an existing property at the provided `path`.
+
+The `properties` are listed as a `path` and `value` pair, where `path` is
+the dot-delimited path to the target property in the `LogMeta` object, and
+`value` is the value to add or update in that target property. When using
+the `remove` mode, a `value` is not necessary.
+
+Here's an example of how you would replace any `cookie` header values with `[REDACTED]`:
+
+```yaml
+logging:
+ appenders:
+ my-rewrite-appender:
+ type: rewrite
+ appenders: [console]
+ policy:
+ type: meta # indicates that we want to rewrite the LogMeta
+ mode: update # will update an existing property only
+ properties:
+ - path: "http.request.headers.cookie" # path to property
+ value: "[REDACTED]" # value to replace at path
+```
+
+Rewrite appenders can even be passed to other rewrite appenders to apply
+multiple filter policies/modes, as long as it doesn't create a circular
+reference. Each rewrite appender is applied sequentially (one after the other).
+```yaml
+logging:
+ appenders:
+ remove-request-headers:
+ type: rewrite
+ appenders: [censor-response-headers] # redirect to the next rewrite appender
+ policy:
+ type: meta
+ mode: remove
+ properties:
+ - path: "http.request.headers" # remove all request headers
+ censor-response-headers:
+ type: rewrite
+ appenders: [console] # output to console
+ policy:
+ type: meta
+ mode: update
+ properties:
+ - path: "http.response.headers.set-cookie"
+ value: "[REDACTED]"
+```
+
+#### Complete Example
+```yaml
+logging:
+ appenders:
+ console:
+ type: console
+ layout:
+ type: pattern
+ highlight: true
+ pattern: "[%date][%level][%logger] %message %meta"
+ file:
+ type: file
+ fileName: ./kibana.log
+ layout:
+ type: json
+ censor:
+ type: rewrite
+ appenders: [console, file]
+ policy:
+ type: meta
+ mode: update
+ properties:
+ - path: "http.request.headers.cookie"
+ value: "[REDACTED]"
+ loggers:
+ - name: http.server.response
+ appenders: [censor] # pass these logs to our rewrite appender
+ level: debug
+```
+
## Configuration
As any configuration in the platform, logging configuration is validated against the predefined schema and if there are
diff --git a/src/core/server/logging/appenders/appenders.ts b/src/core/server/logging/appenders/appenders.ts
index a41a6a2f68fa1b..88df355bd5ebef 100644
--- a/src/core/server/logging/appenders/appenders.ts
+++ b/src/core/server/logging/appenders/appenders.ts
@@ -17,6 +17,7 @@ import {
import { Layouts } from '../layouts/layouts';
import { ConsoleAppender, ConsoleAppenderConfig } from './console/console_appender';
import { FileAppender, FileAppenderConfig } from './file/file_appender';
+import { RewriteAppender, RewriteAppenderConfig } from './rewrite/rewrite_appender';
import {
RollingFileAppender,
RollingFileAppenderConfig,
@@ -32,6 +33,7 @@ export const appendersSchema = schema.oneOf([
ConsoleAppender.configSchema,
FileAppender.configSchema,
LegacyAppender.configSchema,
+ RewriteAppender.configSchema,
RollingFileAppender.configSchema,
]);
@@ -40,6 +42,7 @@ export type AppenderConfigType =
| ConsoleAppenderConfig
| FileAppenderConfig
| LegacyAppenderConfig
+ | RewriteAppenderConfig
| RollingFileAppenderConfig;
/** @internal */
@@ -57,6 +60,8 @@ export class Appenders {
return new ConsoleAppender(Layouts.create(config.layout));
case 'file':
return new FileAppender(Layouts.create(config.layout), config.fileName);
+ case 'rewrite':
+ return new RewriteAppender(config);
case 'rolling-file':
return new RollingFileAppender(config);
case 'legacy-appender':
diff --git a/src/core/server/logging/appenders/rewrite/mocks.ts b/src/core/server/logging/appenders/rewrite/mocks.ts
new file mode 100644
index 00000000000000..a19756e25bf8ea
--- /dev/null
+++ b/src/core/server/logging/appenders/rewrite/mocks.ts
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { RewritePolicy } from './policies/policy';
+
+const createPolicyMock = () => {
+ const mock: jest.Mocked = {
+ rewrite: jest.fn((x) => x),
+ };
+ return mock;
+};
+
+export const rewriteAppenderMocks = {
+ createPolicy: createPolicyMock,
+};
diff --git a/src/core/server/logging/appenders/rewrite/policies/index.ts b/src/core/server/logging/appenders/rewrite/policies/index.ts
new file mode 100644
index 00000000000000..ae3be1e4de916a
--- /dev/null
+++ b/src/core/server/logging/appenders/rewrite/policies/index.ts
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { assertNever } from '@kbn/std';
+import { RewritePolicy } from './policy';
+import { MetaRewritePolicy, MetaRewritePolicyConfig, metaRewritePolicyConfigSchema } from './meta';
+
+export { RewritePolicy };
+
+/**
+ * Available rewrite policies which specify what part of a {@link LogRecord}
+ * can be modified.
+ */
+export type RewritePolicyConfig = MetaRewritePolicyConfig;
+
+export const rewritePolicyConfigSchema = metaRewritePolicyConfigSchema;
+
+export const createRewritePolicy = (config: RewritePolicyConfig): RewritePolicy => {
+ switch (config.type) {
+ case 'meta':
+ return new MetaRewritePolicy(config);
+ default:
+ return assertNever(config.type);
+ }
+};
diff --git a/src/core/server/logging/appenders/rewrite/policies/meta/index.ts b/src/core/server/logging/appenders/rewrite/policies/meta/index.ts
new file mode 100644
index 00000000000000..afdfd6fb709d3e
--- /dev/null
+++ b/src/core/server/logging/appenders/rewrite/policies/meta/index.ts
@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export {
+ MetaRewritePolicy,
+ MetaRewritePolicyConfig,
+ metaRewritePolicyConfigSchema,
+} from './meta_policy';
diff --git a/src/core/server/logging/appenders/rewrite/policies/meta/meta_policy.test.ts b/src/core/server/logging/appenders/rewrite/policies/meta/meta_policy.test.ts
new file mode 100644
index 00000000000000..52b88331a75bee
--- /dev/null
+++ b/src/core/server/logging/appenders/rewrite/policies/meta/meta_policy.test.ts
@@ -0,0 +1,154 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { LogRecord, LogLevel, LogMeta } from '@kbn/logging';
+import { MetaRewritePolicy, MetaRewritePolicyConfig } from './meta_policy';
+
+describe('MetaRewritePolicy', () => {
+ const createPolicy = (
+ mode: MetaRewritePolicyConfig['mode'],
+ properties: MetaRewritePolicyConfig['properties']
+ ) => new MetaRewritePolicy({ type: 'meta', mode, properties });
+
+ const createLogRecord = (meta: LogMeta = {}): LogRecord => ({
+ timestamp: new Date(Date.UTC(2012, 1, 1, 14, 30, 22, 11)),
+ level: LogLevel.Info,
+ context: 'context',
+ message: 'just a log',
+ pid: 42,
+ meta,
+ });
+
+ describe('mode: update', () => {
+ it('updates existing properties in LogMeta', () => {
+ const log = createLogRecord({ a: 'before' });
+ const policy = createPolicy('update', [{ path: 'a', value: 'after' }]);
+ expect(policy.rewrite(log).meta!.a).toBe('after');
+ });
+
+ it('updates nested properties in LogMeta', () => {
+ const log = createLogRecord({ a: 'before a', b: { c: 'before b.c' }, d: [0, 1] });
+ const policy = createPolicy('update', [
+ { path: 'a', value: 'after a' },
+ { path: 'b.c', value: 'after b.c' },
+ { path: 'd[1]', value: 2 },
+ ]);
+ expect(policy.rewrite(log).meta).toMatchInlineSnapshot(`
+ Object {
+ "a": "after a",
+ "b": Object {
+ "c": "after b.c",
+ },
+ "d": Array [
+ 0,
+ 2,
+ ],
+ }
+ `);
+ });
+
+ it('handles string, number, boolean, null', () => {
+ const policy = createPolicy('update', [
+ { path: 'a', value: false },
+ { path: 'b', value: null },
+ { path: 'c', value: 123 },
+ { path: 'd', value: 'hi' },
+ ]);
+ const log = createLogRecord({
+ a: 'a',
+ b: 'b',
+ c: 'c',
+ d: 'd',
+ });
+ expect(policy.rewrite(log).meta).toMatchInlineSnapshot(`
+ Object {
+ "a": false,
+ "b": null,
+ "c": 123,
+ "d": "hi",
+ }
+ `);
+ });
+
+ it(`does not add properties which don't exist yet`, () => {
+ const policy = createPolicy('update', [
+ { path: 'a.b', value: 'foo' },
+ { path: 'a.c', value: 'bar' },
+ ]);
+ const log = createLogRecord({ a: { b: 'existing meta' } });
+ const { meta } = policy.rewrite(log);
+ expect(meta!.a.b).toBe('foo');
+ expect(meta!.a.c).toBeUndefined();
+ });
+
+ it('does not touch anything outside of LogMeta', () => {
+ const policy = createPolicy('update', [{ path: 'a', value: 'bar' }]);
+ const message = Symbol();
+ expect(
+ policy.rewrite(({ message, meta: { a: 'foo' } } as unknown) as LogRecord).message
+ ).toBe(message);
+ expect(policy.rewrite(({ message, meta: { a: 'foo' } } as unknown) as LogRecord))
+ .toMatchInlineSnapshot(`
+ Object {
+ "message": Symbol(),
+ "meta": Object {
+ "a": "bar",
+ },
+ }
+ `);
+ });
+ });
+
+ describe('mode: remove', () => {
+ it('removes existing properties in LogMeta', () => {
+ const log = createLogRecord({ a: 'goodbye' });
+ const policy = createPolicy('remove', [{ path: 'a' }]);
+ expect(policy.rewrite(log).meta!.a).toBeUndefined();
+ });
+
+ it('removes nested properties in LogMeta', () => {
+ const log = createLogRecord({ a: 'a', b: { c: 'b.c' }, d: [0, 1] });
+ const policy = createPolicy('remove', [{ path: 'b.c' }, { path: 'd[1]' }]);
+ expect(policy.rewrite(log).meta).toMatchInlineSnapshot(`
+ Object {
+ "a": "a",
+ "b": Object {},
+ "d": Array [
+ 0,
+ undefined,
+ ],
+ }
+ `);
+ });
+
+ it('has no effect if property does not exist', () => {
+ const log = createLogRecord({ a: 'a' });
+ const policy = createPolicy('remove', [{ path: 'b' }]);
+ expect(policy.rewrite(log).meta).toMatchInlineSnapshot(`
+ Object {
+ "a": "a",
+ }
+ `);
+ });
+
+ it('does not touch anything outside of LogMeta', () => {
+ const policy = createPolicy('remove', [{ path: 'message' }]);
+ const message = Symbol();
+ expect(
+ policy.rewrite(({ message, meta: { message: 'foo' } } as unknown) as LogRecord).message
+ ).toBe(message);
+ expect(policy.rewrite(({ message, meta: { message: 'foo' } } as unknown) as LogRecord))
+ .toMatchInlineSnapshot(`
+ Object {
+ "message": Symbol(),
+ "meta": Object {},
+ }
+ `);
+ });
+ });
+});
diff --git a/src/core/server/logging/appenders/rewrite/policies/meta/meta_policy.ts b/src/core/server/logging/appenders/rewrite/policies/meta/meta_policy.ts
new file mode 100644
index 00000000000000..2215b3489539fa
--- /dev/null
+++ b/src/core/server/logging/appenders/rewrite/policies/meta/meta_policy.ts
@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { schema } from '@kbn/config-schema';
+import { LogRecord } from '@kbn/logging';
+import { set } from '@elastic/safer-lodash-set';
+import { has, unset } from 'lodash';
+import { assertNever } from '@kbn/std';
+import { RewritePolicy } from '../policy';
+
+type MetaRewritePolicyConfigProperties = Array<{
+ path: string;
+ value?: string | number | boolean | null;
+}>;
+
+export interface MetaRewritePolicyConfig {
+ type: 'meta';
+
+ /**
+ * The 'mode' specifies what action to perform on the specified properties.
+ * - 'update' updates an existing property at the provided 'path'.
+ * - 'remove' removes an existing property at the provided 'path'.
+ */
+ mode: 'remove' | 'update';
+
+ /**
+ * The properties to modify.
+ *
+ * @remarks
+ * Each provided 'path' is relative to the record's {@link LogMeta}.
+ * For the 'remove' mode, no 'value' is provided.
+ */
+ properties: MetaRewritePolicyConfigProperties;
+}
+
+export const metaRewritePolicyConfigSchema = schema.object({
+ type: schema.literal('meta'),
+ mode: schema.oneOf([schema.literal('update'), schema.literal('remove')], {
+ defaultValue: 'update',
+ }),
+ properties: schema.arrayOf(
+ schema.object({
+ path: schema.string(),
+ value: schema.maybe(
+ schema.nullable(schema.oneOf([schema.string(), schema.number(), schema.boolean()]))
+ ),
+ })
+ ),
+});
+
+/**
+ * A rewrite policy which can add, remove, or update properties
+ * from a record's {@link LogMeta}.
+ */
+export class MetaRewritePolicy implements RewritePolicy {
+ constructor(private readonly config: MetaRewritePolicyConfig) {}
+
+ rewrite(record: LogRecord): LogRecord {
+ switch (this.config.mode) {
+ case 'update':
+ return this.update(record);
+ case 'remove':
+ return this.remove(record);
+ default:
+ return assertNever(this.config.mode);
+ }
+ }
+
+ private update(record: LogRecord) {
+ for (const { path, value } of this.config.properties) {
+ if (!has(record, `meta.${path}`)) {
+ continue; // don't add properties which don't already exist
+ }
+ set(record, `meta.${path}`, value);
+ }
+ return record;
+ }
+
+ private remove(record: LogRecord) {
+ for (const { path } of this.config.properties) {
+ unset(record, `meta.${path}`);
+ }
+ return record;
+ }
+}
diff --git a/src/core/server/logging/appenders/rewrite/policies/policy.ts b/src/core/server/logging/appenders/rewrite/policies/policy.ts
new file mode 100644
index 00000000000000..f8aef887965fd2
--- /dev/null
+++ b/src/core/server/logging/appenders/rewrite/policies/policy.ts
@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { LogRecord } from '@kbn/logging';
+
+/**
+ * Rewrites a {@link LogRecord} based on the policy's configuration.
+ **/
+export interface RewritePolicy {
+ rewrite(record: LogRecord): LogRecord;
+}
diff --git a/src/core/server/logging/appenders/rewrite/rewrite_appender.test.mocks.ts b/src/core/server/logging/appenders/rewrite/rewrite_appender.test.mocks.ts
new file mode 100644
index 00000000000000..9d29a683057922
--- /dev/null
+++ b/src/core/server/logging/appenders/rewrite/rewrite_appender.test.mocks.ts
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { schema } from '@kbn/config-schema';
+
+export const createRewritePolicyMock = jest.fn();
+jest.doMock('./policies', () => ({
+ rewritePolicyConfigSchema: schema.any(),
+ createRewritePolicy: createRewritePolicyMock,
+}));
+
+export const resetAllMocks = () => {
+ createRewritePolicyMock.mockReset();
+};
diff --git a/src/core/server/logging/appenders/rewrite/rewrite_appender.test.ts b/src/core/server/logging/appenders/rewrite/rewrite_appender.test.ts
new file mode 100644
index 00000000000000..72a54b5012ce54
--- /dev/null
+++ b/src/core/server/logging/appenders/rewrite/rewrite_appender.test.ts
@@ -0,0 +1,137 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { MockedKeys } from '@kbn/utility-types/jest';
+import { createRewritePolicyMock, resetAllMocks } from './rewrite_appender.test.mocks';
+import { rewriteAppenderMocks } from './mocks';
+import { LogLevel, LogRecord, LogMeta, DisposableAppender } from '@kbn/logging';
+import { RewriteAppender, RewriteAppenderConfig } from './rewrite_appender';
+
+// Helper to ensure tuple is typed [A, B] instead of Array
+const toTuple = (a: A, b: B): [A, B] => [a, b];
+
+const createAppenderMock = (name: string) => {
+ const appenderMock: MockedKeys = {
+ append: jest.fn(),
+ dispose: jest.fn(),
+ };
+
+ return toTuple(name, appenderMock);
+};
+
+const createConfig = (appenderNames: string[]): RewriteAppenderConfig => ({
+ type: 'rewrite',
+ appenders: appenderNames,
+ policy: {
+ type: 'meta',
+ mode: 'update',
+ properties: [{ path: 'foo', value: 'bar' }],
+ },
+});
+
+const createLogRecord = (meta: LogMeta = {}): LogRecord => ({
+ timestamp: new Date(),
+ level: LogLevel.Info,
+ context: 'context',
+ message: 'just a log',
+ pid: 42,
+ meta,
+});
+
+describe('RewriteAppender', () => {
+ let policy: ReturnType;
+
+ beforeEach(() => {
+ policy = rewriteAppenderMocks.createPolicy();
+ createRewritePolicyMock.mockReturnValue(policy);
+ });
+
+ afterEach(() => {
+ jest.clearAllMocks();
+ });
+
+ afterAll(() => {
+ resetAllMocks();
+ });
+
+ it('creates a rewrite policy with the provided config', () => {
+ const config = createConfig([]);
+ new RewriteAppender(config);
+ expect(createRewritePolicyMock).toHaveBeenCalledTimes(1);
+ expect(createRewritePolicyMock).toHaveBeenCalledWith(config.policy);
+ });
+
+ describe('#addAppender', () => {
+ it('updates the map of available appenders', () => {
+ const config = createConfig(['mock1']);
+ const appender = new RewriteAppender(config);
+ appender.addAppender(...createAppenderMock('mock1'));
+ expect(() => {
+ appender.append(createLogRecord());
+ }).not.toThrowError();
+ });
+ });
+
+ describe('#append', () => {
+ it('calls the configured appenders with the provided LogRecord', () => {
+ const config = createConfig(['mock1', 'mock2']);
+ const appenderMocks = [createAppenderMock('mock1'), createAppenderMock('mock2')];
+
+ const appender = new RewriteAppender(config);
+ appenderMocks.forEach((mock) => appender.addAppender(...mock));
+
+ const log1 = createLogRecord({ a: 'b' });
+ const log2 = createLogRecord({ c: 'd' });
+
+ appender.append(log1);
+
+ expect(appenderMocks[0][1].append).toHaveBeenCalledTimes(1);
+ expect(appenderMocks[1][1].append).toHaveBeenCalledTimes(1);
+ expect(appenderMocks[0][1].append).toHaveBeenCalledWith(log1);
+ expect(appenderMocks[1][1].append).toHaveBeenCalledWith(log1);
+
+ appender.append(log2);
+
+ expect(appenderMocks[0][1].append).toHaveBeenCalledTimes(2);
+ expect(appenderMocks[1][1].append).toHaveBeenCalledTimes(2);
+ expect(appenderMocks[0][1].append).toHaveBeenCalledWith(log2);
+ expect(appenderMocks[1][1].append).toHaveBeenCalledWith(log2);
+ });
+
+ it('calls `rewrite` on the configured policy', () => {
+ const config = createConfig(['mock1']);
+
+ const appender = new RewriteAppender(config);
+ appender.addAppender(...createAppenderMock('mock1'));
+
+ const log1 = createLogRecord({ a: 'b' });
+ const log2 = createLogRecord({ c: 'd' });
+
+ appender.append(log1);
+
+ expect(policy.rewrite).toHaveBeenCalledTimes(1);
+ expect(policy.rewrite.mock.calls).toEqual([[log1]]);
+
+ appender.append(log2);
+
+ expect(policy.rewrite).toHaveBeenCalledTimes(2);
+ expect(policy.rewrite.mock.calls).toEqual([[log1], [log2]]);
+ });
+
+ it('throws if an appender key cannot be found', () => {
+ const config = createConfig(['oops']);
+ const appender = new RewriteAppender(config);
+
+ expect(() => {
+ appender.append(createLogRecord());
+ }).toThrowErrorMatchingInlineSnapshot(
+ `"Rewrite Appender could not find appender key \\"oops\\". Be sure \`appender.addAppender()\` was called before \`appender.append()\`."`
+ );
+ });
+ });
+});
diff --git a/src/core/server/logging/appenders/rewrite/rewrite_appender.ts b/src/core/server/logging/appenders/rewrite/rewrite_appender.ts
new file mode 100644
index 00000000000000..e54d8ba40ebfcc
--- /dev/null
+++ b/src/core/server/logging/appenders/rewrite/rewrite_appender.ts
@@ -0,0 +1,100 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { schema } from '@kbn/config-schema';
+import { LogRecord, Appender, DisposableAppender } from '@kbn/logging';
+import {
+ createRewritePolicy,
+ rewritePolicyConfigSchema,
+ RewritePolicy,
+ RewritePolicyConfig,
+} from './policies';
+
+export interface RewriteAppenderConfig {
+ type: 'rewrite';
+ /**
+ * The {@link Appender | appender(s)} to pass the log event to after
+ * implementing the specified rewrite policy.
+ */
+ appenders: string[];
+ /**
+ * The {@link RewritePolicy | policy} to use to manipulate the provided data.
+ */
+ policy: RewritePolicyConfig;
+}
+
+/**
+ * Appender that can modify the `LogRecord` instances it receives before passing
+ * them along to another {@link Appender}.
+ * @internal
+ */
+export class RewriteAppender implements DisposableAppender {
+ public static configSchema = schema.object({
+ type: schema.literal('rewrite'),
+ appenders: schema.arrayOf(schema.string(), { defaultValue: [] }),
+ policy: rewritePolicyConfigSchema,
+ });
+
+ private appenders: Map = new Map();
+ private readonly policy: RewritePolicy;
+
+ constructor(private readonly config: RewriteAppenderConfig) {
+ this.policy = createRewritePolicy(config.policy);
+ }
+
+ /**
+ * List of appenders that are dependencies of this appender.
+ *
+ * `addAppender` will throw an error when called with an appender
+ * reference that isn't in this list.
+ */
+ public get appenderRefs() {
+ return this.config.appenders;
+ }
+
+ /**
+ * Appenders can be "attached" to this one so that the RewriteAppender
+ * is able to act as a sort of middleware by calling `append` on other appenders.
+ *
+ * As appenders cannot be attached to each other until they are created,
+ * the `addAppender` method is used to pass in a configured appender.
+ */
+ public addAppender(appenderRef: string, appender: Appender) {
+ if (!this.appenderRefs.includes(appenderRef)) {
+ throw new Error(
+ `addAppender was called with an appender key that is missing from the appenderRefs: "${appenderRef}".`
+ );
+ }
+
+ this.appenders.set(appenderRef, appender);
+ }
+
+ /**
+ * Modifies the `record` and passes it to the specified appender.
+ */
+ public append(record: LogRecord) {
+ const rewrittenRecord = this.policy.rewrite(record);
+ for (const appenderRef of this.appenderRefs) {
+ const appender = this.appenders.get(appenderRef);
+ if (!appender) {
+ throw new Error(
+ `Rewrite Appender could not find appender key "${appenderRef}". ` +
+ 'Be sure `appender.addAppender()` was called before `appender.append()`.'
+ );
+ }
+ appender.append(rewrittenRecord);
+ }
+ }
+
+ /**
+ * Disposes `RewriteAppender`.
+ */
+ public dispose() {
+ this.appenders.clear();
+ }
+}
diff --git a/src/core/server/logging/logging_config.test.ts b/src/core/server/logging/logging_config.test.ts
index 2cb5831a8fb4ce..83f3c139e371af 100644
--- a/src/core/server/logging/logging_config.test.ts
+++ b/src/core/server/logging/logging_config.test.ts
@@ -78,7 +78,6 @@ test('correctly fills in custom `appenders` config.', () => {
type: 'console',
layout: { type: 'pattern', highlight: true },
});
-
expect(configValue.appenders.get('console')).toEqual({
type: 'console',
layout: { type: 'pattern' },
diff --git a/src/core/server/logging/logging_system.test.ts b/src/core/server/logging/logging_system.test.ts
index f68d6c6a97fbc4..8a6fe71bc62220 100644
--- a/src/core/server/logging/logging_system.test.ts
+++ b/src/core/server/logging/logging_system.test.ts
@@ -134,6 +134,76 @@ test('uses `root` logger if context name is not specified.', async () => {
expect(mockConsoleLog.mock.calls).toMatchSnapshot();
});
+test('attaches appenders to appenders that declare refs', async () => {
+ await system.upgrade(
+ config.schema.validate({
+ appenders: {
+ console: {
+ type: 'console',
+ layout: { type: 'pattern', pattern: '[%logger] %message %meta' },
+ },
+ file: {
+ type: 'file',
+ layout: { type: 'pattern', pattern: '[%logger] %message %meta' },
+ fileName: 'path',
+ },
+ rewrite: {
+ type: 'rewrite',
+ appenders: ['console', 'file'],
+ policy: { type: 'meta', mode: 'remove', properties: [{ path: 'b' }] },
+ },
+ },
+ loggers: [{ name: 'tests', level: 'warn', appenders: ['rewrite'] }],
+ })
+ );
+
+ const testLogger = system.get('tests');
+ testLogger.warn('This message goes to a test context.', { a: 'hi', b: 'remove me' });
+
+ expect(mockConsoleLog).toHaveBeenCalledTimes(1);
+ expect(mockConsoleLog.mock.calls[0][0]).toMatchInlineSnapshot(
+ `"[tests] This message goes to a test context. {\\"a\\":\\"hi\\"}"`
+ );
+
+ expect(mockStreamWrite).toHaveBeenCalledTimes(1);
+ expect(mockStreamWrite.mock.calls[0][0]).toMatchInlineSnapshot(`
+ "[tests] This message goes to a test context. {\\"a\\":\\"hi\\"}
+ "
+ `);
+});
+
+test('throws if a circular appender reference is detected', async () => {
+ expect(async () => {
+ await system.upgrade(
+ config.schema.validate({
+ appenders: {
+ console: { type: 'console', layout: { type: 'pattern' } },
+ a: {
+ type: 'rewrite',
+ appenders: ['b'],
+ policy: { type: 'meta', mode: 'remove', properties: [{ path: 'b' }] },
+ },
+ b: {
+ type: 'rewrite',
+ appenders: ['c'],
+ policy: { type: 'meta', mode: 'remove', properties: [{ path: 'b' }] },
+ },
+ c: {
+ type: 'rewrite',
+ appenders: ['console', 'a'],
+ policy: { type: 'meta', mode: 'remove', properties: [{ path: 'b' }] },
+ },
+ },
+ loggers: [{ name: 'tests', level: 'warn', appenders: ['a'] }],
+ })
+ );
+ }).rejects.toThrowErrorMatchingInlineSnapshot(
+ `"Circular appender reference detected: [b -> c -> a -> b]"`
+ );
+
+ expect(mockConsoleLog).toHaveBeenCalledTimes(0);
+});
+
test('`stop()` disposes all appenders.', async () => {
await system.upgrade(
config.schema.validate({
diff --git a/src/core/server/logging/logging_system.ts b/src/core/server/logging/logging_system.ts
index 9ae434aff41d3c..d7c34b48c41012 100644
--- a/src/core/server/logging/logging_system.ts
+++ b/src/core/server/logging/logging_system.ts
@@ -146,6 +146,26 @@ export class LoggingSystem implements LoggerFactory {
return this.getLoggerConfigByContext(config, LoggingConfig.getParentLoggerContext(context));
}
+ /**
+ * Retrieves an appender by the provided key, after first checking that no circular
+ * dependencies exist between appender refs.
+ */
+ private getAppenderByRef(appenderRef: string) {
+ const checkCircularRefs = (key: string, stack: string[]) => {
+ if (stack.includes(key)) {
+ throw new Error(`Circular appender reference detected: [${stack.join(' -> ')} -> ${key}]`);
+ }
+ stack.push(key);
+ const appender = this.appenders.get(key);
+ if (appender?.appenderRefs) {
+ appender.appenderRefs.forEach((ref) => checkCircularRefs(ref, [...stack]));
+ }
+ return appender;
+ };
+
+ return checkCircularRefs(appenderRef, []);
+ }
+
private async applyBaseConfig(newBaseConfig: LoggingConfig) {
const computedConfig = [...this.contextConfigs.values()].reduce(
(baseConfig, contextConfig) => baseConfig.extend(contextConfig),
@@ -167,6 +187,23 @@ export class LoggingSystem implements LoggerFactory {
this.appenders.set(appenderKey, Appenders.create(appenderConfig));
}
+ // Once all appenders have been created, check for any that have explicitly
+ // declared `appenderRefs` dependencies, and look up those dependencies to
+ // attach to the appender. This enables appenders to act as a sort of
+ // middleware and call `append` on each other if needed.
+ for (const [key, appender] of this.appenders) {
+ if (!appender.addAppender || !appender.appenderRefs) {
+ continue;
+ }
+ for (const ref of appender.appenderRefs) {
+ const foundAppender = this.getAppenderByRef(ref);
+ if (!foundAppender) {
+ throw new Error(`Appender "${key}" config contains unknown appender key "${ref}".`);
+ }
+ appender.addAppender(ref, foundAppender);
+ }
+ }
+
for (const [loggerKey, loggerAdapter] of this.loggers) {
loggerAdapter.updateLogger(this.createLogger(loggerKey, computedConfig));
}
diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md
index 2177da84b2b53d..cc1fb05c0c7dd5 100644
--- a/src/core/server/server.api.md
+++ b/src/core/server/server.api.md
@@ -180,10 +180,11 @@ export interface AppCategory {
// Warning: (ae-forgotten-export) The symbol "ConsoleAppenderConfig" needs to be exported by the entry point index.d.ts
// Warning: (ae-forgotten-export) The symbol "FileAppenderConfig" needs to be exported by the entry point index.d.ts
// Warning: (ae-forgotten-export) The symbol "LegacyAppenderConfig" needs to be exported by the entry point index.d.ts
+// Warning: (ae-forgotten-export) The symbol "RewriteAppenderConfig" needs to be exported by the entry point index.d.ts
// Warning: (ae-forgotten-export) The symbol "RollingFileAppenderConfig" needs to be exported by the entry point index.d.ts
//
// @public (undocumented)
-export type AppenderConfigType = ConsoleAppenderConfig | FileAppenderConfig | LegacyAppenderConfig | RollingFileAppenderConfig;
+export type AppenderConfigType = ConsoleAppenderConfig | FileAppenderConfig | LegacyAppenderConfig | RewriteAppenderConfig | RollingFileAppenderConfig;
// @public @deprecated (undocumented)
export interface AssistanceAPIResponse {