Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[7.x] Deduplication of entries and items before sending to endpoint (#71297) #71349

Merged
merged 1 commit into from
Jul 10, 2020
Merged

[7.x] Deduplication of entries and items before sending to endpoint (#71297) #71349

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 137 additions & 2 deletions x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,139 @@ describe('buildEventTypeSignal', () => {
});
});

test('it should deduplicate exception entries', async () => {
const testEntries: EntriesArray = [
{ field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' },
{ field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' },
{ field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' },
{ field: 'server.ip', operator: 'included', type: 'match', value: '192.168.1.1' },
{
field: 'host.hostname.text',
operator: 'included',
type: 'match_any',
value: ['estc', 'kibana'],
},
];

const expectedEndpointExceptions = {
type: 'simple',
entries: [
{
field: 'server.domain',
operator: 'included',
type: 'exact_caseless',
value: 'DOMAIN',
},
{
field: 'server.ip',
operator: 'included',
type: 'exact_cased',
value: '192.168.1.1',
},
{
field: 'host.hostname',
operator: 'included',
type: 'exact_caseless_any',
value: ['estc', 'kibana'],
},
],
};

const first = getFoundExceptionListItemSchemaMock();
first.data[0].entries = testEntries;
mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);

const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
expect(resp).toEqual({
entries: [expectedEndpointExceptions],
});
});

test('it should not deduplicate exception entries across nested boundaries', async () => {
const testEntries: EntriesArray = [
{
entries: [
{ field: 'nested.field', operator: 'included', type: 'match', value: 'some value' },
],
field: 'some.parentField',
type: 'nested',
},
// Same as above but not inside the nest
{ field: 'nested.field', operator: 'included', type: 'match', value: 'some value' },
];

const expectedEndpointExceptions = {
type: 'simple',
entries: [
{
entries: [
{
field: 'nested.field',
operator: 'included',
type: 'exact_cased',
value: 'some value',
},
],
field: 'some.parentField',
type: 'nested',
},
{
field: 'nested.field',
operator: 'included',
type: 'exact_cased',
value: 'some value',
},
],
};

const first = getFoundExceptionListItemSchemaMock();
first.data[0].entries = testEntries;
mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);

const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
expect(resp).toEqual({
entries: [expectedEndpointExceptions],
});
});

test('it should deduplicate exception items', async () => {
const testEntries: EntriesArray = [
{ field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' },
{ field: 'server.ip', operator: 'included', type: 'match', value: '192.168.1.1' },
];

const expectedEndpointExceptions = {
type: 'simple',
entries: [
{
field: 'server.domain',
operator: 'included',
type: 'exact_caseless',
value: 'DOMAIN',
},
{
field: 'server.ip',
operator: 'included',
type: 'exact_cased',
value: '192.168.1.1',
},
],
};

const first = getFoundExceptionListItemSchemaMock();
first.data[0].entries = testEntries;

// Create a second exception item with the same entries
first.data[1] = getExceptionListItemSchemaMock();
first.data[1].entries = testEntries;
mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);

const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
expect(resp).toEqual({
entries: [expectedEndpointExceptions],
});
});

test('it should ignore unsupported entries', async () => {
// Lists and exists are not supported by the Endpoint
const testEntries: EntriesArray = [
Expand Down Expand Up @@ -178,8 +311,9 @@ describe('buildEventTypeSignal', () => {
});

test('it should convert the exception lists response to the proper endpoint format while paging', async () => {
// The first call returns one exception
// The first call returns two exceptions
const first = getFoundExceptionListItemSchemaMock();
first.data.push(getExceptionListItemSchemaMock());

// The second call returns two exceptions
const second = getFoundExceptionListItemSchemaMock();
Expand All @@ -194,7 +328,8 @@ describe('buildEventTypeSignal', () => {
.mockReturnValueOnce(second)
.mockReturnValueOnce(third);
const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
expect(resp.entries.length).toEqual(3);
// Expect 2 exceptions, the first two calls returned the same exception list items
expect(resp.entries.length).toEqual(2);
});

test('it should handle no exceptions', async () => {
Expand Down
19 changes: 16 additions & 3 deletions x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,10 +97,18 @@ export function translateToEndpointExceptions(
exc: FoundExceptionListItemSchema,
schemaVersion: string
): TranslatedExceptionListItem[] {
const entrySet = new Set();
const entriesFiltered: TranslatedExceptionListItem[] = [];
if (schemaVersion === 'v1') {
return exc.data.map((item) => {
return translateItem(schemaVersion, item);
exc.data.forEach((entry) => {
const translatedItem = translateItem(schemaVersion, entry);
const entryHash = createHash('sha256').update(JSON.stringify(translatedItem)).digest('hex');
if (!entrySet.has(entryHash)) {
entriesFiltered.push(translatedItem);
entrySet.add(entryHash);
}
});
return entriesFiltered;
} else {
throw new Error('unsupported schemaVersion');
}
Expand All @@ -124,12 +132,17 @@ function translateItem(
schemaVersion: string,
item: ExceptionListItemSchema
): TranslatedExceptionListItem {
const itemSet = new Set();
return {
type: item.type,
entries: item.entries.reduce((translatedEntries: TranslatedEntry[], entry) => {
const translatedEntry = translateEntry(schemaVersion, entry);
if (translatedEntry !== undefined && translatedEntryType.is(translatedEntry)) {
translatedEntries.push(translatedEntry);
const itemHash = createHash('sha256').update(JSON.stringify(translatedEntry)).digest('hex');
if (!itemSet.has(itemHash)) {
translatedEntries.push(translatedEntry);
itemSet.add(itemHash);
}
}
return translatedEntries;
}, []),
Expand Down