From dbd5f50ffcce4b078702ecf7d16263f5ca59c089 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Tue, 14 Aug 2018 11:06:18 -0500 Subject: [PATCH 01/42] Watcher: migrate PagerDuty v1 events API to v2 API (#32285) The PagerDuty v1 API is EOL and will stop accepting new accounts shortly. This commit swaps out the watcher use of the v1 API with the new v2 API. It does not change anything about the existing watcher API. Closes #32243 --- .../pagerduty/ExecutablePagerDutyAction.java | 2 +- .../notification/pagerduty/IncidentEvent.java | 120 +++++++++++------ .../pagerduty/IncidentEventContext.java | 79 +++++++++++ .../pagerduty/PagerDutyAccount.java | 4 +- .../pagerduty/PagerDutyActionTests.java | 2 +- .../pagerduty/IncidentEventTests.java | 126 ++++++++++++++++++ .../pagerduty/PagerDutyAccountsTests.java | 9 +- 7 files changed, 296 insertions(+), 46 deletions(-) create mode 100644 x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEventTests.java diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/pagerduty/ExecutablePagerDutyAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/pagerduty/ExecutablePagerDutyAction.java index 224e72e1a3da5..59381dc33362c 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/pagerduty/ExecutablePagerDutyAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/pagerduty/ExecutablePagerDutyAction.java @@ -47,7 +47,7 @@ public Action.Result execute(final String actionId, WatchExecutionContext ctx, P return new PagerDutyAction.Result.Simulated(event); } - SentEvent sentEvent = account.send(event, payload); + SentEvent sentEvent = account.send(event, payload, ctx.id().watchId()); return new PagerDutyAction.Result.Executed(account.getName(), sentEvent); } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEvent.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEvent.java index 0fb1a52d28633..c44fbf36e0b18 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEvent.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEvent.java @@ -24,22 +24,22 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; /** * Official documentation for this can be found at * - * https://developer.pagerduty.com/documentation/howto/manually-trigger-an-incident/ - * https://developer.pagerduty.com/documentation/integration/events/trigger - * https://developer.pagerduty.com/documentation/integration/events/acknowledge - * https://developer.pagerduty.com/documentation/integration/events/resolve + * https://v2.developer.pagerduty.com/docs/send-an-event-events-api-v2 */ public class IncidentEvent implements ToXContentObject { static final String HOST = "events.pagerduty.com"; - static final String PATH = "/generic/2010-04-15/create_event.json"; + static final String PATH = "/v2/enqueue"; + static final String ACCEPT_HEADER = "application/vnd.pagerduty+json;version=2"; final String description; @Nullable final HttpProxy proxy; @@ -93,46 +93,81 @@ public int hashCode() { return result; } - public HttpRequest createRequest(final String serviceKey, final Payload payload) throws IOException { + HttpRequest createRequest(final String serviceKey, final Payload payload, final String watchId) throws IOException { return HttpRequest.builder(HOST, -1) .method(HttpMethod.POST) .scheme(Scheme.HTTPS) .path(PATH) .proxy(proxy) - .jsonBody(new ToXContent() { - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(Fields.SERVICE_KEY.getPreferredName(), serviceKey); - builder.field(Fields.EVENT_TYPE.getPreferredName(), eventType); - builder.field(Fields.DESCRIPTION.getPreferredName(), description); - if (incidentKey != null) { - builder.field(Fields.INCIDENT_KEY.getPreferredName(), incidentKey); - } - if (client != null) { - builder.field(Fields.CLIENT.getPreferredName(), client); - } - if (clientUrl != null) { - builder.field(Fields.CLIENT_URL.getPreferredName(), clientUrl); - } - if (attachPayload) { - builder.startObject(Fields.DETAILS.getPreferredName()); - builder.field(Fields.PAYLOAD.getPreferredName()); - payload.toXContent(builder, params); - builder.endObject(); - } - if (contexts != null && contexts.length > 0) { - builder.startArray(Fields.CONTEXTS.getPreferredName()); - for (IncidentEventContext context : contexts) { - context.toXContent(builder, params); - } - builder.endArray(); - } - return builder; - } - }) + .setHeader("Accept", ACCEPT_HEADER) + .jsonBody((b, p) -> buildAPIXContent(b, p, serviceKey, payload, watchId)) .build(); } + XContentBuilder buildAPIXContent(XContentBuilder builder, Params params, String serviceKey, + Payload payload, String watchId) throws IOException { + builder.field(Fields.ROUTING_KEY.getPreferredName(), serviceKey); + builder.field(Fields.EVENT_ACTION.getPreferredName(), eventType); + if (incidentKey != null) { + builder.field(Fields.DEDUP_KEY.getPreferredName(), incidentKey); + } + + builder.startObject(Fields.PAYLOAD.getPreferredName()); + { + builder.field(Fields.SUMMARY.getPreferredName(), description); + + if (attachPayload && payload != null) { + builder.startObject(Fields.CUSTOM_DETAILS.getPreferredName()); + { + builder.field(Fields.PAYLOAD.getPreferredName(), payload, params); + } + builder.endObject(); + } + + if (watchId != null) { + builder.field(Fields.SOURCE.getPreferredName(), watchId); + } else { + builder.field(Fields.SOURCE.getPreferredName(), "watcher"); + } + // TODO externalize this into something user editable + builder.field(Fields.SEVERITY.getPreferredName(), "critical"); + } + builder.endObject(); + + if (client != null) { + builder.field(Fields.CLIENT.getPreferredName(), client); + } + if (clientUrl != null) { + builder.field(Fields.CLIENT_URL.getPreferredName(), clientUrl); + } + + if (contexts != null && contexts.length > 0) { + toXContentV2Contexts(builder, params, contexts); + } + + return builder; + } + + /** + * Turns the V1 API contexts into 2 distinct lists, images and links. The V2 API has separated these out into 2 top level fields. + */ + private void toXContentV2Contexts(XContentBuilder builder, ToXContent.Params params, + IncidentEventContext[] contexts) throws IOException { + // contexts can be either links or images, and the v2 api needs them separate + Map> groups = Arrays.stream(contexts) + .collect(Collectors.groupingBy(iec -> iec.type)); + + List links = groups.getOrDefault(IncidentEventContext.Type.LINK, Collections.emptyList()); + if (links.isEmpty() == false) { + builder.array(Fields.LINKS.getPreferredName(), links.toArray()); + } + + List images = groups.getOrDefault(IncidentEventContext.Type.IMAGE, Collections.emptyList()); + if (images.isEmpty() == false) { + builder.array(Fields.IMAGES.getPreferredName(), images.toArray()); + } + } + @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(); @@ -445,8 +480,15 @@ interface Fields { // we need to keep this for BWC ParseField CONTEXT_DEPRECATED = new ParseField("context"); - ParseField SERVICE_KEY = new ParseField("service_key"); ParseField PAYLOAD = new ParseField("payload"); - ParseField DETAILS = new ParseField("details"); + ParseField ROUTING_KEY = new ParseField("routing_key"); + ParseField EVENT_ACTION = new ParseField("event_action"); + ParseField DEDUP_KEY = new ParseField("dedup_key"); + ParseField SUMMARY = new ParseField("summary"); + ParseField SOURCE = new ParseField("source"); + ParseField SEVERITY = new ParseField("severity"); + ParseField LINKS = new ParseField("links"); + ParseField IMAGES = new ParseField("images"); + ParseField CUSTOM_DETAILS = new ParseField("custom_details"); } } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEventContext.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEventContext.java index d43829346b626..cd9924ae9dca7 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEventContext.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEventContext.java @@ -92,6 +92,85 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder.endObject(); } + public static IncidentEventContext parse(XContentParser parser) throws IOException { + Type type = null; + String href = null; + String text = null; + String src = null; + String alt = null; + + String currentFieldName = null; + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (Strings.hasLength(currentFieldName)) { + if (XField.TYPE.match(currentFieldName, parser.getDeprecationHandler())) { + try { + type = Type.valueOf(parser.text().toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + String msg = "could not parse trigger incident event context. unknown context type [{}]"; + throw new ElasticsearchParseException(msg, parser.text()); + } + } else { + if (XField.HREF.match(currentFieldName, parser.getDeprecationHandler())) { + href = parser.text(); + } else if (XField.TEXT.match(currentFieldName, parser.getDeprecationHandler())) { + text = parser.text(); + } else if (XField.SRC.match(currentFieldName, parser.getDeprecationHandler())) { + src = parser.text(); + } else if (XField.ALT.match(currentFieldName, parser.getDeprecationHandler())) { + alt = parser.text(); + } else { + String msg = "could not parse trigger incident event context. unknown field [{}]"; + throw new ElasticsearchParseException(msg, currentFieldName); + } + } + } + } + + return createAndValidateTemplate(type, href, src, alt, text); + } + + private static IncidentEventContext createAndValidateTemplate(Type type, String href, String src, String alt, + String text) { + if (type == null) { + throw new ElasticsearchParseException("could not parse trigger incident event context. missing required field [{}]", + XField.TYPE.getPreferredName()); + } + + switch (type) { + case LINK: + if (href == null) { + throw new ElasticsearchParseException("could not parse trigger incident event context. missing required field " + + "[{}] for [{}] context", XField.HREF.getPreferredName(), Type.LINK.name().toLowerCase(Locale.ROOT)); + } + if (src != null) { + throw new ElasticsearchParseException("could not parse trigger incident event context. unexpected field [{}] for " + + "[{}] context", XField.SRC.getPreferredName(), Type.LINK.name().toLowerCase(Locale.ROOT)); + } + if (alt != null) { + throw new ElasticsearchParseException("could not parse trigger incident event context. unexpected field [{}] for " + + "[{}] context", XField.ALT.getPreferredName(), Type.LINK.name().toLowerCase(Locale.ROOT)); + } + return link(href, text); + case IMAGE: + if (src == null) { + throw new ElasticsearchParseException("could not parse trigger incident event context. missing required field " + + "[{}] for [{}] context", XField.SRC.getPreferredName(), Type.IMAGE.name().toLowerCase(Locale.ROOT)); + } + if (text != null) { + throw new ElasticsearchParseException("could not parse trigger incident event context. unexpected field [{}] for " + + "[{}] context", XField.TEXT.getPreferredName(), Type.IMAGE.name().toLowerCase(Locale.ROOT)); + } + return image(src, href, alt); + default: + throw new ElasticsearchParseException("could not parse trigger incident event context. unknown context type [{}]", + type); + } + } + + public static class Template implements ToXContentObject { final Type type; diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyAccount.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyAccount.java index 5cf1a77f9711a..b2498a749d7b2 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyAccount.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyAccount.java @@ -48,8 +48,8 @@ public IncidentEventDefaults getDefaults() { return eventDefaults; } - public SentEvent send(IncidentEvent event, Payload payload) throws IOException { - HttpRequest request = event.createRequest(serviceKey, payload); + public SentEvent send(IncidentEvent event, Payload payload, String watchId) throws IOException { + HttpRequest request = event.createRequest(serviceKey, payload, watchId); HttpResponse response = httpClient.execute(request); return SentEvent.responded(event, request, response); } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/pagerduty/PagerDutyActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/pagerduty/PagerDutyActionTests.java index 6f57ccd82d930..07a55c628ec1c 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/pagerduty/PagerDutyActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/pagerduty/PagerDutyActionTests.java @@ -111,7 +111,7 @@ public void testExecute() throws Exception { when(response.status()).thenReturn(200); HttpRequest request = mock(HttpRequest.class); SentEvent sentEvent = SentEvent.responded(event, request, response); - when(account.send(event, payload)).thenReturn(sentEvent); + when(account.send(event, payload, wid.watchId())).thenReturn(sentEvent); when(service.getAccount(accountName)).thenReturn(account); Action.Result result = executable.execute("_id", ctx, payload); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEventTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEventTests.java new file mode 100644 index 0000000000000..3638d5f85d929 --- /dev/null +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEventTests.java @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.watcher.notification.pagerduty; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.yaml.ObjectPath; +import org.elasticsearch.xpack.core.watcher.watch.Payload; +import org.elasticsearch.xpack.watcher.common.http.HttpProxy; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.equalTo; + +public class IncidentEventTests extends ESTestCase { + + public void testPagerDutyXContent() throws IOException { + + String serviceKey = randomAlphaOfLength(3); + boolean attachPayload = randomBoolean(); + Payload payload = null; + if (attachPayload) { + payload = new Payload.Simple(Collections.singletonMap(randomAlphaOfLength(3), randomAlphaOfLength(3))); + } + String watchId = randomAlphaOfLength(3); + String description = randomAlphaOfLength(3); + String eventType = randomAlphaOfLength(3); + String incidentKey = rarely() ? null : randomAlphaOfLength(3); + String client = rarely() ? null : randomAlphaOfLength(3); + String clientUrl = rarely() ? null : randomAlphaOfLength(3); + String account = rarely() ? null : randomAlphaOfLength(3); + + IncidentEventContext[] contexts = null; + List links = new ArrayList<>(); + List images = new ArrayList<>(); + + if (randomBoolean()) { + int numContexts = randomIntBetween(0, 3); + contexts = new IncidentEventContext[numContexts]; + for (int i = 0; i < numContexts; i++) { + if (randomBoolean()) { + contexts[i] = IncidentEventContext.link("href", "text"); + links.add(contexts[i]); + } else { + contexts[i] = IncidentEventContext.image("src", "href", "alt"); + images.add(contexts[i]); + } + } + } + + HttpProxy proxy = rarely() ? null : HttpProxy.NO_PROXY; + + IncidentEvent event = new IncidentEvent(description, eventType, incidentKey, client, clientUrl, account, + attachPayload, contexts, proxy); + + XContentBuilder jsonBuilder = jsonBuilder(); + jsonBuilder.startObject(); // since its a snippet + event.buildAPIXContent(jsonBuilder, ToXContent.EMPTY_PARAMS, serviceKey, payload, watchId); + jsonBuilder.endObject(); + XContentParser parser = createParser(jsonBuilder); + parser.nextToken(); + + ObjectPath objectPath = ObjectPath.createFromXContent(jsonBuilder.contentType().xContent(), BytesReference.bytes(jsonBuilder)); + + String actualServiceKey = objectPath.evaluate(IncidentEvent.Fields.ROUTING_KEY.getPreferredName()); + String actualWatchId = objectPath.evaluate(IncidentEvent.Fields.PAYLOAD.getPreferredName() + + "." + IncidentEvent.Fields.SOURCE.getPreferredName()); + if (actualWatchId == null) { + actualWatchId = "watcher"; // hardcoded if the SOURCE is null + } + String actualDescription = objectPath.evaluate(IncidentEvent.Fields.PAYLOAD.getPreferredName() + + "." + IncidentEvent.Fields.SUMMARY.getPreferredName()); + String actualEventType = objectPath.evaluate(IncidentEvent.Fields.EVENT_ACTION.getPreferredName()); + String actualIncidentKey = objectPath.evaluate(IncidentEvent.Fields.DEDUP_KEY.getPreferredName()); + String actualClient = objectPath.evaluate(IncidentEvent.Fields.CLIENT.getPreferredName()); + String actualClientUrl = objectPath.evaluate(IncidentEvent.Fields.CLIENT_URL.getPreferredName()); + String actualSeverity = objectPath.evaluate(IncidentEvent.Fields.PAYLOAD.getPreferredName() + + "." + IncidentEvent.Fields.SEVERITY.getPreferredName()); + Map payloadDetails = objectPath.evaluate("payload.custom_details.payload"); + Payload actualPayload = null; + + if (payloadDetails != null) { + actualPayload = new Payload.Simple(payloadDetails); + } + + List actualLinks = new ArrayList<>(); + List> linkMap = (List>) objectPath.evaluate(IncidentEvent.Fields.LINKS.getPreferredName()); + if (linkMap != null) { + for (Map iecValue : linkMap) { + actualLinks.add(IncidentEventContext.link(iecValue.get("href"), iecValue.get("text"))); + } + } + + List actualImages = new ArrayList<>(); + List> imgMap = (List>) objectPath.evaluate(IncidentEvent.Fields.IMAGES.getPreferredName()); + if (imgMap != null) { + for (Map iecValue : imgMap) { + actualImages.add(IncidentEventContext.image(iecValue.get("src"), iecValue.get("href"), iecValue.get("alt"))); + } + } + + // assert the actuals were the same as expected + assertThat(serviceKey, equalTo(actualServiceKey)); + assertThat(eventType, equalTo(actualEventType)); + assertThat(incidentKey, equalTo(actualIncidentKey)); + assertThat(description, equalTo(actualDescription)); + assertThat(watchId, equalTo(actualWatchId)); + assertThat("critical", equalTo(actualSeverity)); + assertThat(client, equalTo(actualClient)); + assertThat(clientUrl, equalTo(actualClientUrl)); + assertThat(links, equalTo(actualLinks)); + assertThat(images, equalTo(actualImages)); + assertThat(payload, equalTo(actualPayload)); + } +} diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyAccountsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyAccountsTests.java index d70badc4bec22..1e88c69614270 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyAccountsTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyAccountsTests.java @@ -24,6 +24,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -48,7 +49,7 @@ public void testProxy() throws Exception { HttpProxy proxy = new HttpProxy("localhost", 8080); IncidentEvent event = new IncidentEvent("foo", null, null, null, null, account.getName(), true, null, proxy); - account.send(event, Payload.EMPTY); + account.send(event, Payload.EMPTY, null); HttpRequest request = argumentCaptor.getValue(); assertThat(request.proxy(), is(proxy)); @@ -72,11 +73,13 @@ public void testContextIsSentCorrect() throws Exception { "https://www.elastic.co/products/x-pack/alerting", "X-Pack-Alerting website link with log") }; IncidentEvent event = new IncidentEvent("foo", null, null, null, null, account.getName(), true, contexts, HttpProxy.NO_PROXY); - account.send(event, Payload.EMPTY); + account.send(event, Payload.EMPTY, null); HttpRequest request = argumentCaptor.getValue(); ObjectPath source = ObjectPath.createFromXContent(JsonXContent.jsonXContent, new BytesArray(request.body())); - assertThat(source.evaluate("contexts"), notNullValue()); + assertThat(source.evaluate("contexts"), nullValue()); + assertThat(source.evaluate("links"), notNullValue()); + assertThat(source.evaluate("images"), notNullValue()); } private void addAccountSettings(String name, Settings.Builder builder) { From 08144257dff7068270902727d4e4972653b4f149 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Tue, 14 Aug 2018 11:06:46 -0500 Subject: [PATCH 02/42] Watcher: Remove extraneous auth classes (#32300) The auth.basic package was an example of a single implementation interface that leaked into many different classes. In order to clean this up, the HttpAuth interface, factories, and Registries all were removed and the single implementation, BasicAuth, was substituted in all cases. This removes some dependenies between Auth and the Templates, which can now use static methods on BasicAuth. BasicAuth was also moved into the http package and all of the other classes were removed. --- .../elasticsearch/xpack/watcher/Watcher.java | 22 ++------ .../actions/webhook/WebhookAction.java | 5 +- .../actions/webhook/WebhookActionFactory.java | 8 +-- .../http/{auth/basic => }/BasicAuth.java | 27 ++++++---- .../xpack/watcher/common/http/HttpClient.java | 16 +++--- .../watcher/common/http/HttpRequest.java | 29 ++++------- .../common/http/HttpRequestTemplate.java | 27 ++++------ .../common/http/auth/ApplicableHttpAuth.java | 51 ------------------- .../watcher/common/http/auth/HttpAuth.java | 14 ----- .../common/http/auth/HttpAuthFactory.java | 25 --------- .../common/http/auth/HttpAuthRegistry.java | 50 ------------------ .../http/auth/basic/ApplicableBasicAuth.java | 43 ---------------- .../http/auth/basic/BasicAuthFactory.java | 35 ------------- .../xpack/watcher/input/http/HttpInput.java | 4 +- .../watcher/input/http/HttpInputFactory.java | 8 +-- .../HttpEmailAttachementParser.java | 7 +-- .../email/attachment/ReportingAttachment.java | 10 ++-- .../attachment/ReportingAttachmentParser.java | 24 +++------ .../notification/jira/JiraAccount.java | 2 +- .../actions/email/EmailActionTests.java | 9 +--- .../jira/ExecutableJiraActionTests.java | 9 ++-- .../actions/webhook/WebhookActionTests.java | 11 ++-- .../webhook/WebhookHttpsIntegrationTests.java | 2 +- .../webhook/WebhookIntegrationTests.java | 2 +- .../watcher/common/http/HttpClientTests.java | 26 ++++------ .../http/HttpConnectionTimeoutTests.java | 11 ++-- .../common/http/HttpReadTimeoutTests.java | 11 ++-- .../common/http/HttpRequestTemplateTests.java | 19 ++----- .../watcher/common/http/HttpRequestTests.java | 18 ++----- .../watcher/input/chain/ChainInputTests.java | 2 +- .../watcher/input/http/HttpInputTests.java | 10 ++-- .../HttpEmailAttachementParserTests.java | 9 +--- .../ReportingAttachmentParserTests.java | 26 +++------- .../notification/jira/JiraIssueTests.java | 19 +++---- .../HttpSecretsIntegrationTests.java | 15 ++++-- .../xpack/watcher/watch/WatchTests.java | 7 +-- 36 files changed, 139 insertions(+), 474 deletions(-) rename x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/{auth/basic => }/BasicAuth.java (84%) delete mode 100644 x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/ApplicableHttpAuth.java delete mode 100644 x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuth.java delete mode 100644 x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuthFactory.java delete mode 100644 x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuthRegistry.java delete mode 100644 x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/ApplicableBasicAuth.java delete mode 100644 x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/BasicAuthFactory.java diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java index add12623d6a6c..89f6fcb25129b 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java @@ -92,12 +92,7 @@ import org.elasticsearch.xpack.watcher.actions.webhook.WebhookAction; import org.elasticsearch.xpack.watcher.actions.webhook.WebhookActionFactory; import org.elasticsearch.xpack.watcher.common.http.HttpClient; -import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate; import org.elasticsearch.xpack.watcher.common.http.HttpSettings; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthFactory; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory; import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine; import org.elasticsearch.xpack.watcher.condition.ArrayCompareCondition; import org.elasticsearch.xpack.watcher.condition.CompareCondition; @@ -283,12 +278,7 @@ public Collection createComponents(Client client, ClusterService cluster new WatcherIndexTemplateRegistry(settings, clusterService, threadPool, client); // http client - Map httpAuthFactories = new HashMap<>(); - httpAuthFactories.put(BasicAuth.TYPE, new BasicAuthFactory(cryptoService)); - // TODO: add more auth types, or remove this indirection - HttpAuthRegistry httpAuthRegistry = new HttpAuthRegistry(httpAuthFactories); - HttpRequestTemplate.Parser httpTemplateParser = new HttpRequestTemplate.Parser(httpAuthRegistry); - httpClient = new HttpClient(settings, httpAuthRegistry, getSslService()); + httpClient = new HttpClient(settings, getSslService(), cryptoService); // notification EmailService emailService = new EmailService(settings, cryptoService, clusterService.getClusterSettings()); @@ -305,11 +295,9 @@ public Collection createComponents(Client client, ClusterService cluster TextTemplateEngine templateEngine = new TextTemplateEngine(settings, scriptService); Map emailAttachmentParsers = new HashMap<>(); - emailAttachmentParsers.put(HttpEmailAttachementParser.TYPE, new HttpEmailAttachementParser(httpClient, httpTemplateParser, - templateEngine)); + emailAttachmentParsers.put(HttpEmailAttachementParser.TYPE, new HttpEmailAttachementParser(httpClient, templateEngine)); emailAttachmentParsers.put(DataAttachmentParser.TYPE, new DataAttachmentParser()); - emailAttachmentParsers.put(ReportingAttachmentParser.TYPE, new ReportingAttachmentParser(settings, httpClient, templateEngine, - httpAuthRegistry)); + emailAttachmentParsers.put(ReportingAttachmentParser.TYPE, new ReportingAttachmentParser(settings, httpClient, templateEngine)); EmailAttachmentsParser emailAttachmentsParser = new EmailAttachmentsParser(emailAttachmentParsers); // conditions @@ -329,7 +317,7 @@ public Collection createComponents(Client client, ClusterService cluster // actions final Map actionFactoryMap = new HashMap<>(); actionFactoryMap.put(EmailAction.TYPE, new EmailActionFactory(settings, emailService, templateEngine, emailAttachmentsParser)); - actionFactoryMap.put(WebhookAction.TYPE, new WebhookActionFactory(settings, httpClient, httpTemplateParser, templateEngine)); + actionFactoryMap.put(WebhookAction.TYPE, new WebhookActionFactory(settings, httpClient, templateEngine)); actionFactoryMap.put(IndexAction.TYPE, new IndexActionFactory(settings, client)); actionFactoryMap.put(LoggingAction.TYPE, new LoggingActionFactory(settings, templateEngine)); actionFactoryMap.put(HipChatAction.TYPE, new HipChatActionFactory(settings, templateEngine, hipChatService)); @@ -343,7 +331,7 @@ public Collection createComponents(Client client, ClusterService cluster final Map inputFactories = new HashMap<>(); inputFactories.put(SearchInput.TYPE, new SearchInputFactory(settings, client, xContentRegistry, scriptService)); inputFactories.put(SimpleInput.TYPE, new SimpleInputFactory(settings)); - inputFactories.put(HttpInput.TYPE, new HttpInputFactory(settings, httpClient, templateEngine, httpTemplateParser)); + inputFactories.put(HttpInput.TYPE, new HttpInputFactory(settings, httpClient, templateEngine)); inputFactories.put(NoneInput.TYPE, new NoneInputFactory(settings)); inputFactories.put(TransformInput.TYPE, new TransformInputFactory(settings, transformRegistry)); final InputRegistry inputRegistry = new InputRegistry(settings, inputFactories); diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookAction.java index 8d20d91f4dca7..febbae99d46fa 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookAction.java @@ -55,10 +55,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return requestTemplate.toXContent(builder, params); } - public static WebhookAction parse(String watchId, String actionId, XContentParser parser, - HttpRequestTemplate.Parser requestParser) throws IOException { + public static WebhookAction parse(String watchId, String actionId, XContentParser parser) throws IOException { try { - HttpRequestTemplate request = requestParser.parse(parser); + HttpRequestTemplate request = HttpRequestTemplate.Parser.parse(parser); return new WebhookAction(request); } catch (ElasticsearchParseException pe) { throw new ElasticsearchParseException("could not parse [{}] action [{}/{}]. failed parsing http request template", pe, TYPE, diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionFactory.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionFactory.java index 6de16006432b7..5a7a1069932d9 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionFactory.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionFactory.java @@ -10,7 +10,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.core.watcher.actions.ActionFactory; import org.elasticsearch.xpack.watcher.common.http.HttpClient; -import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate; import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine; import java.io.IOException; @@ -18,21 +17,18 @@ public class WebhookActionFactory extends ActionFactory { private final HttpClient httpClient; - private final HttpRequestTemplate.Parser requestTemplateParser; private final TextTemplateEngine templateEngine; - public WebhookActionFactory(Settings settings, HttpClient httpClient, HttpRequestTemplate.Parser requestTemplateParser, - TextTemplateEngine templateEngine) { + public WebhookActionFactory(Settings settings, HttpClient httpClient, TextTemplateEngine templateEngine) { super(Loggers.getLogger(ExecutableWebhookAction.class, settings)); this.httpClient = httpClient; - this.requestTemplateParser = requestTemplateParser; this.templateEngine = templateEngine; } @Override public ExecutableWebhookAction parseExecutable(String watchId, String actionId, XContentParser parser) throws IOException { - return new ExecutableWebhookAction(WebhookAction.parse(watchId, actionId, parser, requestTemplateParser), + return new ExecutableWebhookAction(WebhookAction.parse(watchId, actionId, parser), actionLogger, httpClient, templateEngine); } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/BasicAuth.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/BasicAuth.java similarity index 84% rename from x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/BasicAuth.java rename to x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/BasicAuth.java index 2d316735efd77..0b40df4262f91 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/BasicAuth.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/BasicAuth.java @@ -3,22 +3,22 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.watcher.common.http.auth.basic; +package org.elasticsearch.xpack.watcher.common.http; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.core.watcher.common.secret.Secret; import org.elasticsearch.xpack.core.watcher.crypto.CryptoService; import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherParams; import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherXContentParser; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuth; import java.io.IOException; import java.util.Objects; -public class BasicAuth implements HttpAuth { +public class BasicAuth implements ToXContentObject { public static final String TYPE = "basic"; @@ -34,11 +34,6 @@ public BasicAuth(String username, Secret password) { this.password = password; } - @Override - public String type() { - return TYPE; - } - public String getUsername() { return username; } @@ -74,7 +69,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder.endObject(); } - public static BasicAuth parse(XContentParser parser) throws IOException { + public static BasicAuth parseInner(XContentParser parser) throws IOException { String username = null; Secret password = null; @@ -103,6 +98,20 @@ public static BasicAuth parse(XContentParser parser) throws IOException { return new BasicAuth(username, password); } + public static BasicAuth parse(XContentParser parser) throws IOException { + String type = null; + XContentParser.Token token; + BasicAuth auth = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + type = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT && type != null) { + auth = BasicAuth.parseInner(parser); + } + } + return auth; + } + interface Field { ParseField USERNAME = new ParseField("username"); ParseField PASSWORD = new ParseField("password"); diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java index 97d7779346fae..547e3b7197f4a 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java @@ -10,6 +10,8 @@ import org.apache.http.HttpHost; import org.apache.http.NameValuePair; import org.apache.http.auth.AuthScope; +import org.apache.http.auth.Credentials; +import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.AuthCache; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.config.RequestConfig; @@ -42,8 +44,7 @@ import org.elasticsearch.xpack.core.common.socket.SocketAccess; import org.elasticsearch.xpack.core.ssl.SSLConfiguration; import org.elasticsearch.xpack.core.ssl.SSLService; -import org.elasticsearch.xpack.watcher.common.http.auth.ApplicableHttpAuth; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry; +import org.elasticsearch.xpack.core.watcher.crypto.CryptoService; import javax.net.ssl.HostnameVerifier; import java.io.ByteArrayOutputStream; @@ -66,20 +67,20 @@ public class HttpClient extends AbstractComponent implements Closeable { // you are querying a remote Elasticsearch cluster private static final int MAX_CONNECTIONS = 500; - private final HttpAuthRegistry httpAuthRegistry; private final CloseableHttpClient client; private final HttpProxy settingsProxy; private final TimeValue defaultConnectionTimeout; private final TimeValue defaultReadTimeout; private final ByteSizeValue maxResponseSize; + private final CryptoService cryptoService; - public HttpClient(Settings settings, HttpAuthRegistry httpAuthRegistry, SSLService sslService) { + public HttpClient(Settings settings, SSLService sslService, CryptoService cryptoService) { super(settings); - this.httpAuthRegistry = httpAuthRegistry; this.defaultConnectionTimeout = HttpSettings.CONNECTION_TIMEOUT.get(settings); this.defaultReadTimeout = HttpSettings.READ_TIMEOUT.get(settings); this.maxResponseSize = HttpSettings.MAX_HTTP_RESPONSE_SIZE.get(settings); this.settingsProxy = getProxyFromSettings(); + this.cryptoService = cryptoService; HttpClientBuilder clientBuilder = HttpClientBuilder.create(); @@ -139,9 +140,10 @@ public HttpResponse execute(HttpRequest request) throws IOException { HttpClientContext localContext = HttpClientContext.create(); // auth if (request.auth() != null) { - ApplicableHttpAuth applicableAuth = httpAuthRegistry.createApplicable(request.auth); CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - applicableAuth.apply(credentialsProvider, new AuthScope(request.host, request.port)); + Credentials credentials = new UsernamePasswordCredentials(request.auth().username, + new String(request.auth().password.text(cryptoService))); + credentialsProvider.setCredentials(new AuthScope(request.host, request.port), credentials); localContext.setCredentialsProvider(credentialsProvider); // preemptive auth, no need to wait for a 401 first diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java index 3754cc440eb64..6bcfc9e5e7d5f 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java @@ -21,8 +21,6 @@ import org.elasticsearch.xpack.core.watcher.support.WatcherUtils; import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherParams; import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherXContentParser; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuth; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -50,7 +48,7 @@ public class HttpRequest implements ToXContentObject { @Nullable final String path; final Map params; final Map headers; - @Nullable final HttpAuth auth; + @Nullable final BasicAuth auth; @Nullable final String body; @Nullable final TimeValue connectionTimeout; @Nullable final TimeValue readTimeout; @@ -58,7 +56,7 @@ public class HttpRequest implements ToXContentObject { public HttpRequest(String host, int port, @Nullable Scheme scheme, @Nullable HttpMethod method, @Nullable String path, @Nullable Map params, @Nullable Map headers, - @Nullable HttpAuth auth, @Nullable String body, @Nullable TimeValue connectionTimeout, + @Nullable BasicAuth auth, @Nullable String body, @Nullable TimeValue connectionTimeout, @Nullable TimeValue readTimeout, @Nullable HttpProxy proxy) { this.host = host; this.port = port; @@ -102,7 +100,7 @@ public Map headers() { return headers; } - public HttpAuth auth() { + public BasicAuth auth() { return auth; } @@ -166,7 +164,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params toX } if (auth != null) { builder.startObject(Field.AUTH.getPreferredName()) - .field(auth.type(), auth, toXContentParams) + .field(BasicAuth.TYPE, auth, toXContentParams) .endObject(); } if (body != null) { @@ -234,7 +232,7 @@ public String toString() { sb.append("], "); } if (auth != null) { - sb.append("auth=[").append(auth.type()).append("], "); + sb.append("auth=[").append(BasicAuth.TYPE).append("], "); } sb.append("connection_timeout=[").append(connectionTimeout).append("], "); sb.append("read_timeout=[").append(readTimeout).append("], "); @@ -254,14 +252,7 @@ static Builder builder() { } public static class Parser { - - private final HttpAuthRegistry httpAuthRegistry; - - public Parser(HttpAuthRegistry httpAuthRegistry) { - this.httpAuthRegistry = httpAuthRegistry; - } - - public HttpRequest parse(XContentParser parser) throws IOException { + public static HttpRequest parse(XContentParser parser) throws IOException { Builder builder = new Builder(); XContentParser.Token token; String currentFieldName = null; @@ -275,7 +266,7 @@ public HttpRequest parse(XContentParser parser) throws IOException { throw new ElasticsearchParseException("could not parse http request. could not parse [{}] field", currentFieldName); } } else if (Field.AUTH.match(currentFieldName, parser.getDeprecationHandler())) { - builder.auth(httpAuthRegistry.parse(parser)); + builder.auth(BasicAuth.parse(parser)); } else if (HttpRequest.Field.CONNECTION_TIMEOUT.match(currentFieldName, parser.getDeprecationHandler())) { builder.connectionTimeout(TimeValue.timeValueMillis(parser.longValue())); } else if (HttpRequest.Field.CONNECTION_TIMEOUT_HUMAN.match(currentFieldName, parser.getDeprecationHandler())) { @@ -302,7 +293,7 @@ public HttpRequest parse(XContentParser parser) throws IOException { builder.setHeaders((Map) WatcherUtils.flattenModel(parser.map())); } else if (Field.PARAMS.match(currentFieldName, parser.getDeprecationHandler())) { builder.setParams((Map) WatcherUtils.flattenModel(parser.map())); - } else if (Field.BODY.match(currentFieldName, parser.getDeprecationHandler())) { + } else if (Field.BODY.match(currentFieldName, parser.getDeprecationHandler())) { builder.body(parser.text()); } else { throw new ElasticsearchParseException("could not parse http request. unexpected object field [{}]", @@ -360,7 +351,7 @@ public static class Builder { private String path; private Map params = new HashMap<>(); private Map headers = new HashMap<>(); - private HttpAuth auth; + private BasicAuth auth; private String body; private TimeValue connectionTimeout; private TimeValue readTimeout; @@ -421,7 +412,7 @@ public Builder setHeader(String key, String value) { return this; } - public Builder auth(HttpAuth auth) { + public Builder auth(BasicAuth auth) { this.auth = auth; return this; } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTemplate.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTemplate.java index e511b30ef5934..8ca87395248b1 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTemplate.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTemplate.java @@ -18,8 +18,6 @@ import org.elasticsearch.rest.RestUtils; import org.elasticsearch.script.ScriptType; import org.elasticsearch.xpack.core.watcher.support.WatcherDateTimeUtils; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuth; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry; import org.elasticsearch.xpack.watcher.common.text.TextTemplate; import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine; @@ -42,14 +40,14 @@ public class HttpRequestTemplate implements ToXContentObject { private final TextTemplate path; private final Map params; private final Map headers; - private final HttpAuth auth; + private final BasicAuth auth; private final TextTemplate body; @Nullable private final TimeValue connectionTimeout; @Nullable private final TimeValue readTimeout; @Nullable private final HttpProxy proxy; public HttpRequestTemplate(String host, int port, @Nullable Scheme scheme, @Nullable HttpMethod method, @Nullable TextTemplate path, - Map params, Map headers, HttpAuth auth, + Map params, Map headers, BasicAuth auth, TextTemplate body, @Nullable TimeValue connectionTimeout, @Nullable TimeValue readTimeout, @Nullable HttpProxy proxy) { this.host = host; @@ -94,7 +92,7 @@ public Map headers() { return headers; } - public HttpAuth auth() { + public BasicAuth auth() { return auth; } @@ -185,7 +183,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par } if (auth != null) { builder.startObject(HttpRequest.Field.AUTH.getPreferredName()) - .field(auth.type(), auth, params) + .field(BasicAuth.TYPE, auth, params) .endObject(); } if (body != null) { @@ -261,14 +259,7 @@ static Builder builder() { } public static class Parser { - - private final HttpAuthRegistry httpAuthRegistry; - - public Parser(HttpAuthRegistry httpAuthRegistry) { - this.httpAuthRegistry = httpAuthRegistry; - } - - public HttpRequestTemplate parse(XContentParser parser) throws IOException { + public static HttpRequestTemplate parse(XContentParser parser) throws IOException { assert parser.currentToken() == XContentParser.Token.START_OBJECT; Builder builder = new Builder(); @@ -312,8 +303,8 @@ public HttpRequestTemplate parse(XContentParser parser) throws IOException { } } else if (token == XContentParser.Token.START_OBJECT) { if (HttpRequest.Field.AUTH.match(currentFieldName, parser.getDeprecationHandler())) { - builder.auth(httpAuthRegistry.parse(parser)); - } else { + builder.auth(BasicAuth.parse(parser)); + } else { throw new ElasticsearchParseException("could not parse http request template. unexpected object field [{}]", currentFieldName); } @@ -387,7 +378,7 @@ public static class Builder { private TextTemplate path; private final Map params = new HashMap<>(); private final Map headers = new HashMap<>(); - private HttpAuth auth; + private BasicAuth auth; private TextTemplate body; private TimeValue connectionTimeout; private TimeValue readTimeout; @@ -444,7 +435,7 @@ public Builder putHeader(String key, TextTemplate value) { return this; } - public Builder auth(HttpAuth auth) { + public Builder auth(BasicAuth auth) { this.auth = auth; return this; } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/ApplicableHttpAuth.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/ApplicableHttpAuth.java deleted file mode 100644 index cecc7bfda79ed..0000000000000 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/ApplicableHttpAuth.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.watcher.common.http.auth; - -import org.apache.http.auth.AuthScope; -import org.apache.http.client.CredentialsProvider; -import org.elasticsearch.common.xcontent.ToXContentObject; -import org.elasticsearch.common.xcontent.XContentBuilder; - -import java.io.IOException; -import java.net.HttpURLConnection; - -public abstract class ApplicableHttpAuth implements ToXContentObject { - - protected final Auth auth; - - public ApplicableHttpAuth(Auth auth) { - this.auth = auth; - } - - public final String type() { - return auth.type(); - } - - public abstract void apply(HttpURLConnection connection); - - public abstract void apply(CredentialsProvider credsProvider, AuthScope authScope); - - @Override - public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return auth.toXContent(builder, params); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - ApplicableHttpAuth that = (ApplicableHttpAuth) o; - - return auth.equals(that.auth); - } - - @Override - public int hashCode() { - return auth.hashCode(); - } -} diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuth.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuth.java deleted file mode 100644 index 0909a33ab7312..0000000000000 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuth.java +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.watcher.common.http.auth; - -import org.elasticsearch.common.xcontent.ToXContentObject; - -public interface HttpAuth extends ToXContentObject { - - String type(); - -} diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuthFactory.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuthFactory.java deleted file mode 100644 index 7667e568b0ad3..0000000000000 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuthFactory.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.watcher.common.http.auth; - -import org.elasticsearch.common.xcontent.XContentParser; - -import java.io.IOException; - -public abstract class HttpAuthFactory> { - - public abstract String type(); - - public abstract Auth parse(XContentParser parser) throws IOException; - - public abstract AAuth createApplicable(Auth auth); - - public AAuth parseApplicable(XContentParser parser) throws IOException { - Auth auth = parse(parser); - return createApplicable(auth); - } - -} diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuthRegistry.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuthRegistry.java deleted file mode 100644 index edf584231d636..0000000000000 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuthRegistry.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.watcher.common.http.auth; - -import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.xcontent.XContentParser; - -import java.io.IOException; -import java.util.Map; - -import static org.elasticsearch.xpack.core.watcher.support.Exceptions.illegalArgument; - -public class HttpAuthRegistry { - - private final Map factories; - - public HttpAuthRegistry(Map factories) { - this.factories = factories; - } - - public HttpAuth parse(XContentParser parser) throws IOException { - String type = null; - XContentParser.Token token; - HttpAuth auth = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - type = parser.currentName(); - } else if (token == XContentParser.Token.START_OBJECT && type != null) { - HttpAuthFactory factory = factories.get(type); - if (factory == null) { - throw new ElasticsearchParseException("unknown http auth type [{}]", type); - } - auth = factory.parse(parser); - } - } - return auth; - } - - public > AA createApplicable(A auth) { - HttpAuthFactory factory = factories.get(auth.type()); - if (factory == null) { - throw illegalArgument("unknown http auth type [{}]", auth.type()); - } - return (AA) factory.createApplicable(auth); - } - -} diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/ApplicableBasicAuth.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/ApplicableBasicAuth.java deleted file mode 100644 index 86639a59a1242..0000000000000 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/ApplicableBasicAuth.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.watcher.common.http.auth.basic; - -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.elasticsearch.xpack.core.watcher.crypto.CryptoService; -import org.elasticsearch.xpack.watcher.common.http.auth.ApplicableHttpAuth; - -import java.net.HttpURLConnection; -import java.nio.charset.StandardCharsets; -import java.util.Base64; - -public class ApplicableBasicAuth extends ApplicableHttpAuth { - - private final String basicAuth; - private final CryptoService cryptoService; - - public ApplicableBasicAuth(BasicAuth auth, CryptoService service) { - super(auth); - basicAuth = headerValue(auth.username, auth.password.text(service)); - this.cryptoService = service; - } - - public static String headerValue(String username, char[] password) { - return "Basic " + Base64.getEncoder().encodeToString((username + ":" + new String(password)).getBytes(StandardCharsets.UTF_8)); - } - - public void apply(HttpURLConnection connection) { - connection.setRequestProperty("Authorization", basicAuth); - } - - @Override - public void apply(CredentialsProvider credsProvider, AuthScope authScope) { - credsProvider.setCredentials(authScope, - new UsernamePasswordCredentials(auth.username, new String(auth.password.text(cryptoService)))); - } - -} diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/BasicAuthFactory.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/BasicAuthFactory.java deleted file mode 100644 index 838c53be7b19a..0000000000000 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/BasicAuthFactory.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.watcher.common.http.auth.basic; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.xpack.core.watcher.crypto.CryptoService; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthFactory; - -import java.io.IOException; - -public class BasicAuthFactory extends HttpAuthFactory { - - private final CryptoService cryptoService; - - public BasicAuthFactory(@Nullable CryptoService cryptoService) { - this.cryptoService = cryptoService; - } - - public String type() { - return BasicAuth.TYPE; - } - - public BasicAuth parse(XContentParser parser) throws IOException { - return BasicAuth.parse(parser); - } - - @Override - public ApplicableBasicAuth createApplicable(BasicAuth auth) { - return new ApplicableBasicAuth(auth, cryptoService); - } -} diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/HttpInput.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/HttpInput.java index 3f3acdc0b3235..4a0763f7d6b69 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/HttpInput.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/HttpInput.java @@ -70,7 +70,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - public static HttpInput parse(String watchId, XContentParser parser, HttpRequestTemplate.Parser requestParser) throws IOException { + public static HttpInput parse(String watchId, XContentParser parser) throws IOException { Set extract = null; HttpRequestTemplate request = null; HttpContentType expectedResponseBodyType = null; @@ -82,7 +82,7 @@ public static HttpInput parse(String watchId, XContentParser parser, HttpRequest currentFieldName = parser.currentName(); } else if (Field.REQUEST.match(currentFieldName, parser.getDeprecationHandler())) { try { - request = requestParser.parse(parser); + request = HttpRequestTemplate.Parser.parse(parser); } catch (ElasticsearchParseException pe) { throw new ElasticsearchParseException("could not parse [{}] input for watch [{}]. failed to parse http request " + "template", pe, TYPE, watchId); diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/HttpInputFactory.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/HttpInputFactory.java index 2cb26ab1d6707..7a68a77ddd21b 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/HttpInputFactory.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/HttpInputFactory.java @@ -9,7 +9,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.watcher.common.http.HttpClient; -import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate; import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine; import org.elasticsearch.xpack.watcher.input.InputFactory; @@ -19,14 +18,11 @@ public final class HttpInputFactory extends InputFactory emailAttachmentParsers = new HashMap<>(); emailAttachmentParsers.put(HttpEmailAttachementParser.TYPE, new HttpEmailAttachementParser(httpClient, - new HttpRequestTemplate.Parser(registry), new MockTextTemplateEngine())); + new MockTextTemplateEngine())); emailAttachmentParsers.put(DataAttachmentParser.TYPE, new DataAttachmentParser()); emailAttachmentParser = new EmailAttachmentsParser(emailAttachmentParsers); } @@ -511,10 +508,8 @@ public void testThatOneFailedEmailAttachmentResultsInActionFailure() throws Exce .thenReturn(new HttpResponse(403)); // setup email attachment parsers - HttpRequestTemplate.Parser httpRequestTemplateParser = new HttpRequestTemplate.Parser(registry); Map attachmentParsers = new HashMap<>(); - attachmentParsers.put(HttpEmailAttachementParser.TYPE, new HttpEmailAttachementParser(httpClient, httpRequestTemplateParser, - engine)); + attachmentParsers.put(HttpEmailAttachementParser.TYPE, new HttpEmailAttachementParser(httpClient, engine)); EmailAttachmentsParser emailAttachmentsParser = new EmailAttachmentsParser(attachmentParsers); XContentBuilder builder = jsonBuilder().startObject() diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/jira/ExecutableJiraActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/jira/ExecutableJiraActionTests.java index cb434e62df5f9..aa03bf4545e5b 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/jira/ExecutableJiraActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/jira/ExecutableJiraActionTests.java @@ -17,8 +17,7 @@ import org.elasticsearch.xpack.watcher.common.http.HttpProxy; import org.elasticsearch.xpack.watcher.common.http.HttpRequest; import org.elasticsearch.xpack.watcher.common.http.HttpResponse; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuth; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth; +import org.elasticsearch.xpack.watcher.common.http.BasicAuth; import org.elasticsearch.xpack.watcher.common.text.TextTemplate; import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine; import org.elasticsearch.xpack.watcher.notification.jira.JiraAccount; @@ -93,11 +92,9 @@ public void testProxy() throws Exception { assertThat(request.port(), is(port)); assertThat(request.path(), is(JiraAccount.DEFAULT_PATH)); - HttpAuth httpAuth = request.auth(); - assertThat(httpAuth.type(), is("basic")); + BasicAuth httpAuth = request.auth(); - BasicAuth basicAuth = (BasicAuth) httpAuth; - assertThat(basicAuth.getUsername(), is(user)); + assertThat(httpAuth.getUsername(), is(user)); } public void testExecutionWithNoDefaults() throws Exception { diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionTests.java index 09ca57c1708f7..755e3dffe5c90 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionTests.java @@ -28,8 +28,6 @@ import org.elasticsearch.xpack.watcher.common.http.HttpRequest; import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate; import org.elasticsearch.xpack.watcher.common.http.HttpResponse; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory; import org.elasticsearch.xpack.watcher.common.text.TextTemplate; import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine; import org.elasticsearch.xpack.watcher.execution.TriggeredExecutionContext; @@ -47,7 +45,6 @@ import java.io.IOException; import java.util.Map; -import static java.util.Collections.singletonMap; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.CoreMatchers.instanceOf; @@ -73,7 +70,6 @@ public class WebhookActionTests extends ESTestCase { private static final String TEST_PATH_STRING = "/testPath"; private TextTemplateEngine templateEngine; - private HttpAuthRegistry authRegistry; private TextTemplate testBody; private TextTemplate testPath; @@ -82,7 +78,6 @@ public void init() throws Exception { templateEngine = new MockTextTemplateEngine(); testBody = new TextTemplate(TEST_BODY_STRING); testPath = new TextTemplate(TEST_PATH_STRING); - authRegistry = new HttpAuthRegistry(singletonMap("basic", new BasicAuthFactory(null))); } public void testExecute() throws Exception { @@ -213,14 +208,14 @@ public void testParserFailure() throws Exception { } private WebhookActionFactory webhookFactory(HttpClient client) { - return new WebhookActionFactory(Settings.EMPTY, client, new HttpRequestTemplate.Parser(authRegistry), templateEngine); + return new WebhookActionFactory(Settings.EMPTY, client, templateEngine); } public void testThatSelectingProxyWorks() throws Exception { Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build()); - try (HttpClient httpClient = new HttpClient(Settings.EMPTY, authRegistry, - new SSLService(environment.settings(), environment)); MockWebServer proxyServer = new MockWebServer()) { + try (HttpClient httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null); + MockWebServer proxyServer = new MockWebServer()) { proxyServer.start(); proxyServer.enqueue(new MockResponse().setResponseCode(200).setBody("fullProxiedContent")); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookHttpsIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookHttpsIntegrationTests.java index a69445386d027..0ba733ea4c5de 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookHttpsIntegrationTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookHttpsIntegrationTests.java @@ -18,7 +18,7 @@ import org.elasticsearch.xpack.watcher.common.http.HttpMethod; import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate; import org.elasticsearch.xpack.watcher.common.http.Scheme; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth; +import org.elasticsearch.xpack.watcher.common.http.BasicAuth; import org.elasticsearch.xpack.watcher.common.text.TextTemplate; import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition; import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase; diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookIntegrationTests.java index 151bf4af18939..4988b7a1b9ab3 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookIntegrationTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookIntegrationTests.java @@ -20,7 +20,7 @@ import org.elasticsearch.xpack.watcher.actions.ActionBuilders; import org.elasticsearch.xpack.watcher.common.http.HttpMethod; import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth; +import org.elasticsearch.xpack.watcher.common.http.BasicAuth; import org.elasticsearch.xpack.watcher.common.text.TextTemplate; import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition; import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase; diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java index 03dcd7947155e..882dc5389a493 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java @@ -27,9 +27,6 @@ import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.core.ssl.TestsSSLService; import org.elasticsearch.xpack.core.ssl.VerificationMode; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory; import org.junit.After; import org.junit.Before; @@ -48,7 +45,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicReference; -import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -64,14 +60,12 @@ public class HttpClientTests extends ESTestCase { private MockWebServer webServer = new MockWebServer(); private HttpClient httpClient; - private HttpAuthRegistry authRegistry; private Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build()); @Before public void init() throws Exception { - authRegistry = new HttpAuthRegistry(singletonMap(BasicAuth.TYPE, new BasicAuthFactory(null))); webServer.start(); - httpClient = new HttpClient(Settings.EMPTY, authRegistry, new SSLService(environment.settings(), environment)); + httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null); } @After @@ -185,7 +179,7 @@ public void testHttps() throws Exception { .setSecureSettings(secureSettings) .build(); } - try (HttpClient client = new HttpClient(settings, authRegistry, new SSLService(settings, environment))) { + try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null)) { secureSettings = new MockSecureSettings(); // We can't use the client created above for the server since it is only a truststore secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode"); @@ -226,7 +220,7 @@ public void testHttpsDisableHostnameVerification() throws Exception { } settings = builder.build(); } - try (HttpClient client = new HttpClient(settings, authRegistry, new SSLService(settings, environment))) { + try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null)) { MockSecureSettings secureSettings = new MockSecureSettings(); // We can't use the client created above for the server since it only defines a truststore secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode-no-subjaltname"); @@ -253,7 +247,7 @@ public void testHttpsClientAuth() throws Exception { .build(); TestsSSLService sslService = new TestsSSLService(settings, environment); - try (HttpClient client = new HttpClient(settings, authRegistry, sslService)) { + try (HttpClient client = new HttpClient(settings, sslService, null)) { testSslMockWebserver(client, sslService.sslContext(), true); } } @@ -301,7 +295,7 @@ public void testHttpResponseWithAnyStatusCodeCanReturnBody() throws Exception { @Network public void testHttpsWithoutTruststore() throws Exception { - try (HttpClient client = new HttpClient(Settings.EMPTY, authRegistry, new SSLService(Settings.EMPTY, environment))) { + try (HttpClient client = new HttpClient(Settings.EMPTY, new SSLService(Settings.EMPTY, environment), null)) { // Known server with a valid cert from a commercial CA HttpRequest.Builder request = HttpRequest.builder("www.elastic.co", 443).scheme(Scheme.HTTPS); HttpResponse response = client.execute(request.build()); @@ -325,7 +319,7 @@ public void testThatProxyCanBeConfigured() throws Exception { .method(HttpMethod.GET) .path("/"); - try (HttpClient client = new HttpClient(settings, authRegistry, new SSLService(settings, environment))) { + try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null)) { HttpResponse response = client.execute(requestBuilder.build()); assertThat(response.status(), equalTo(200)); assertThat(response.body().utf8ToString(), equalTo("fullProxiedContent")); @@ -406,7 +400,7 @@ public void testProxyCanHaveDifferentSchemeThanRequest() throws Exception { .scheme(Scheme.HTTP) .path("/"); - try (HttpClient client = new HttpClient(settings, authRegistry, new SSLService(settings, environment))) { + try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null)) { HttpResponse response = client.execute(requestBuilder.build()); assertThat(response.status(), equalTo(200)); assertThat(response.body().utf8ToString(), equalTo("fullProxiedContent")); @@ -434,7 +428,7 @@ public void testThatProxyCanBeOverriddenByRequest() throws Exception { .proxy(new HttpProxy("localhost", proxyServer.getPort(), Scheme.HTTP)) .path("/"); - try (HttpClient client = new HttpClient(settings, authRegistry, new SSLService(settings, environment))) { + try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null)) { HttpResponse response = client.execute(requestBuilder.build()); assertThat(response.status(), equalTo(200)); assertThat(response.body().utf8ToString(), equalTo("fullProxiedContent")); @@ -455,7 +449,7 @@ public void testThatProxyConfigurationRequiresHostAndPort() { } IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> new HttpClient(settings.build(), authRegistry, new SSLService(settings.build(), environment))); + () -> new HttpClient(settings.build(), new SSLService(settings.build(), environment), null)); assertThat(e.getMessage(), containsString("HTTP proxy requires both settings: [xpack.http.proxy.host] and [xpack.http.proxy.port]")); } @@ -554,7 +548,7 @@ public void testMaxHttpResponseSize() throws Exception { HttpRequest.Builder requestBuilder = HttpRequest.builder("localhost", webServer.getPort()).method(HttpMethod.GET).path("/"); - try (HttpClient client = new HttpClient(settings, authRegistry, new SSLService(environment.settings(), environment))) { + try (HttpClient client = new HttpClient(settings, new SSLService(environment.settings(), environment), null)) { IOException e = expectThrows(IOException.class, () -> client.execute(requestBuilder.build())); assertThat(e.getMessage(), startsWith("Maximum limit of")); } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpConnectionTimeoutTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpConnectionTimeoutTests.java index 8ac2bef16e8d8..21efe5b2b94ea 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpConnectionTimeoutTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpConnectionTimeoutTests.java @@ -13,11 +13,9 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.junit.annotations.Network; import org.elasticsearch.xpack.core.ssl.SSLService; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.lessThan; -import static org.mockito.Mockito.mock; public class HttpConnectionTimeoutTests extends ESTestCase { // setting an unroutable IP to simulate a connection timeout @@ -26,8 +24,7 @@ public class HttpConnectionTimeoutTests extends ESTestCase { @Network public void testDefaultTimeout() throws Exception { Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build()); - HttpClient httpClient = new HttpClient(Settings.EMPTY, mock(HttpAuthRegistry.class), - new SSLService(environment.settings(), environment)); + HttpClient httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null); HttpRequest request = HttpRequest.builder(UNROUTABLE_IP, 12345) .method(HttpMethod.POST) @@ -52,8 +49,7 @@ public void testDefaultTimeout() throws Exception { public void testDefaultTimeoutCustom() throws Exception { Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build()); HttpClient httpClient = new HttpClient(Settings.builder() - .put("xpack.http.default_connection_timeout", "5s").build() - , mock(HttpAuthRegistry.class), new SSLService(environment.settings(), environment)); + .put("xpack.http.default_connection_timeout", "5s").build(), new SSLService(environment.settings(), environment), null); HttpRequest request = HttpRequest.builder(UNROUTABLE_IP, 12345) .method(HttpMethod.POST) @@ -78,8 +74,7 @@ public void testDefaultTimeoutCustom() throws Exception { public void testTimeoutCustomPerRequest() throws Exception { Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build()); HttpClient httpClient = new HttpClient(Settings.builder() - .put("xpack.http.default_connection_timeout", "10s").build() - , mock(HttpAuthRegistry.class), new SSLService(environment.settings(), environment)); + .put("xpack.http.default_connection_timeout", "10s").build(), new SSLService(environment.settings(), environment), null); HttpRequest request = HttpRequest.builder(UNROUTABLE_IP, 12345) .connectionTimeout(TimeValue.timeValueSeconds(5)) diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpReadTimeoutTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpReadTimeoutTests.java index fa5a53f4e1da0..bc328dc586e9e 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpReadTimeoutTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpReadTimeoutTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.test.http.MockResponse; import org.elasticsearch.test.http.MockWebServer; import org.elasticsearch.xpack.core.ssl.SSLService; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry; import org.junit.After; import org.junit.Before; @@ -21,7 +20,6 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.lessThan; -import static org.mockito.Mockito.mock; public class HttpReadTimeoutTests extends ESTestCase { @@ -45,8 +43,7 @@ public void testDefaultTimeout() throws Exception { .path("/") .build(); - try (HttpClient httpClient = new HttpClient(Settings.EMPTY, mock(HttpAuthRegistry.class), - new SSLService(environment.settings(), environment))) { + try (HttpClient httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null)) { long start = System.nanoTime(); expectThrows(SocketTimeoutException.class, () -> httpClient.execute(request)); @@ -68,8 +65,7 @@ public void testDefaultTimeoutCustom() throws Exception { .build(); try (HttpClient httpClient = new HttpClient(Settings.builder() - .put("xpack.http.default_read_timeout", "3s").build() - , mock(HttpAuthRegistry.class), new SSLService(environment.settings(), environment))) { + .put("xpack.http.default_read_timeout", "3s").build(), new SSLService(environment.settings(), environment), null)) { long start = System.nanoTime(); expectThrows(SocketTimeoutException.class, () -> httpClient.execute(request)); @@ -92,8 +88,7 @@ public void testTimeoutCustomPerRequest() throws Exception { .build(); try (HttpClient httpClient = new HttpClient(Settings.builder() - .put("xpack.http.default_read_timeout", "10s").build() - , mock(HttpAuthRegistry.class), new SSLService(environment.settings(), environment))) { + .put("xpack.http.default_read_timeout", "10s").build(), new SSLService(environment.settings(), environment), null)) { long start = System.nanoTime(); expectThrows(SocketTimeoutException.class, () -> httpClient.execute(request)); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTemplateTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTemplateTests.java index 1f57c812ebbe4..dcbc19f4d5070 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTemplateTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTemplateTests.java @@ -14,22 +14,17 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherParams; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory; import org.elasticsearch.xpack.watcher.common.text.TextTemplate; import org.elasticsearch.xpack.watcher.test.MockTextTemplateEngine; import java.util.Collections; import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.is; -import static org.mockito.Mockito.mock; public class HttpRequestTemplateTests extends ESTestCase { @@ -87,8 +82,7 @@ public void testProxyParsing() throws Exception { XContentParser xContentParser = createParser(xContentBuilder); xContentParser.nextToken(); - HttpRequestTemplate.Parser parser = new HttpRequestTemplate.Parser(mock(HttpAuthRegistry.class)); - HttpRequestTemplate parsedTemplate = parser.parse(xContentParser); + HttpRequestTemplate parsedTemplate = HttpRequestTemplate.Parser.parse(xContentParser); assertThat(parsedTemplate.proxy().getPort(), is(proxyPort)); assertThat(parsedTemplate.proxy().getHost(), is(proxyHost)); } @@ -134,14 +128,10 @@ public void testParseSelfGenerated() throws Exception { HttpRequestTemplate template = builder.build(); - HttpAuthRegistry registry = new HttpAuthRegistry(singletonMap(BasicAuth.TYPE, - new BasicAuthFactory(null))); - HttpRequestTemplate.Parser parser = new HttpRequestTemplate.Parser(registry); - XContentBuilder xContentBuilder = template.toXContent(jsonBuilder(), WatcherParams.builder().hideSecrets(false).build()); XContentParser xContentParser = createParser(xContentBuilder); xContentParser.nextToken(); - HttpRequestTemplate parsed = parser.parse(xContentParser); + HttpRequestTemplate parsed = HttpRequestTemplate.Parser.parse(xContentParser); assertEquals(template, parsed); } @@ -194,13 +184,12 @@ private void assertThatManualBuilderEqualsParsingFromUrl(String url, HttpRequest XContentParser urlContentParser = createParser(urlContentBuilder); urlContentParser.nextToken(); - HttpRequestTemplate.Parser parser = new HttpRequestTemplate.Parser(mock(HttpAuthRegistry.class)); - HttpRequestTemplate urlParsedTemplate = parser.parse(urlContentParser); + HttpRequestTemplate urlParsedTemplate = HttpRequestTemplate.Parser.parse(urlContentParser); XContentBuilder xContentBuilder = builder.build().toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS); XContentParser xContentParser = createParser(xContentBuilder); xContentParser.nextToken(); - HttpRequestTemplate parsedTemplate = parser.parse(xContentParser); + HttpRequestTemplate parsedTemplate = HttpRequestTemplate.Parser.parse(xContentParser); assertThat(parsedTemplate, is(urlParsedTemplate)); } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTests.java index 223735cad54e4..0d1541577a58b 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTests.java @@ -13,11 +13,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherParams; import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherXContentParser; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory; -import static java.util.Collections.singletonMap; import static org.elasticsearch.common.xcontent.XContentFactory.cborBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.smileBuilder; @@ -25,7 +21,6 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; -import static org.mockito.Mockito.mock; public class HttpRequestTests extends ESTestCase { @@ -129,14 +124,12 @@ public void testXContentSerialization() throws Exception { try (XContentBuilder xContentBuilder = randomFrom(jsonBuilder(), smileBuilder(), yamlBuilder(), cborBuilder())) { httpRequest.toXContent(xContentBuilder, WatcherParams.builder().hideSecrets(false).build()); - HttpAuthRegistry registry = new HttpAuthRegistry(singletonMap(BasicAuth.TYPE, new BasicAuthFactory(null))); - HttpRequest.Parser httpRequestParser = new HttpRequest.Parser(registry); - + try (XContentParser parser = createParser(xContentBuilder)) { assertNull(parser.currentToken()); parser.nextToken(); - - HttpRequest parsedRequest = httpRequestParser.parse(parser); + + HttpRequest parsedRequest = HttpRequest.Parser.parse(parser); assertEquals(httpRequest, parsedRequest); } } @@ -161,14 +154,13 @@ private void assertThatManualBuilderEqualsParsingFromUrl(String url, HttpRequest XContentParser urlContentParser = createParser(urlContentBuilder); urlContentParser.nextToken(); - HttpRequest.Parser parser = new HttpRequest.Parser(mock(HttpAuthRegistry.class)); - HttpRequest urlParsedRequest = parser.parse(urlContentParser); + HttpRequest urlParsedRequest = HttpRequest.Parser.parse(urlContentParser); WatcherParams params = WatcherParams.builder().hideSecrets(false).build(); XContentBuilder xContentBuilder = builder.build().toXContent(jsonBuilder(), params); XContentParser xContentParser = createParser(xContentBuilder); xContentParser.nextToken(); - HttpRequest parsedRequest = parser.parse(xContentParser); + HttpRequest parsedRequest = HttpRequest.Parser.parse(xContentParser); assertThat(parsedRequest, is(urlParsedRequest)); } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/chain/ChainInputTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/chain/ChainInputTests.java index cc19cef7b4768..2e1e7858f16d9 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/chain/ChainInputTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/chain/ChainInputTests.java @@ -22,7 +22,7 @@ import org.elasticsearch.xpack.core.watcher.input.Input; import org.elasticsearch.xpack.core.watcher.watch.Payload; import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth; +import org.elasticsearch.xpack.watcher.common.http.BasicAuth; import org.elasticsearch.xpack.watcher.condition.ScriptCondition; import org.elasticsearch.xpack.watcher.input.InputFactory; import org.elasticsearch.xpack.watcher.input.InputRegistry; diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/http/HttpInputTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/http/HttpInputTests.java index 545773b4f9293..d134b04ea431f 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/http/HttpInputTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/http/HttpInputTests.java @@ -28,10 +28,7 @@ import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate; import org.elasticsearch.xpack.watcher.common.http.HttpResponse; import org.elasticsearch.xpack.watcher.common.http.Scheme; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuth; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory; +import org.elasticsearch.xpack.watcher.common.http.BasicAuth; import org.elasticsearch.xpack.watcher.common.text.TextTemplate; import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine; import org.elasticsearch.xpack.watcher.input.InputBuilders; @@ -73,8 +70,7 @@ public class HttpInputTests extends ESTestCase { public void init() throws Exception { httpClient = mock(HttpClient.class); templateEngine = mock(TextTemplateEngine.class); - HttpAuthRegistry registry = new HttpAuthRegistry(singletonMap("basic", new BasicAuthFactory(null))); - httpParser = new HttpInputFactory(Settings.EMPTY, httpClient, templateEngine, new HttpRequestTemplate.Parser(registry)); + httpParser = new HttpInputFactory(Settings.EMPTY, httpClient, templateEngine); } public void testExecute() throws Exception { @@ -158,7 +154,7 @@ public void testParser() throws Exception { randomBoolean() ? new MapBuilder().put("a", new TextTemplate("b")).map() : null; Map headers = randomBoolean() ? new MapBuilder().put("c", new TextTemplate("d")).map() : null; - HttpAuth auth = randomBoolean() ? new BasicAuth("username", "password".toCharArray()) : null; + BasicAuth auth = randomBoolean() ? new BasicAuth("username", "password".toCharArray()) : null; HttpRequestTemplate.Builder requestBuilder = HttpRequestTemplate.builder(host, port) .scheme(scheme) .method(httpMethod) diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/HttpEmailAttachementParserTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/HttpEmailAttachementParserTests.java index ef71a1157437b..2b0e632b0cac7 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/HttpEmailAttachementParserTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/HttpEmailAttachementParserTests.java @@ -19,9 +19,6 @@ import org.elasticsearch.xpack.watcher.common.http.HttpRequest; import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate; import org.elasticsearch.xpack.watcher.common.http.HttpResponse; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory; import org.elasticsearch.xpack.watcher.test.MockTextTemplateEngine; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -34,7 +31,6 @@ import java.util.Map; import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Collections.singletonMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.mockExecutionContextBuilder; import static org.hamcrest.Matchers.hasSize; @@ -45,20 +41,17 @@ public class HttpEmailAttachementParserTests extends ESTestCase { - private HttpRequestTemplate.Parser httpRequestTemplateParser; private HttpClient httpClient; private EmailAttachmentsParser emailAttachmentsParser; private Map attachmentParsers; @Before public void init() throws Exception { - HttpAuthRegistry authRegistry = new HttpAuthRegistry(singletonMap(BasicAuth.TYPE, new BasicAuthFactory(null))); - httpRequestTemplateParser = new HttpRequestTemplate.Parser(authRegistry); httpClient = mock(HttpClient.class); attachmentParsers = new HashMap<>(); attachmentParsers.put(HttpEmailAttachementParser.TYPE, - new HttpEmailAttachementParser(httpClient, httpRequestTemplateParser, new MockTextTemplateEngine())); + new HttpEmailAttachementParser(httpClient, new MockTextTemplateEngine())); emailAttachmentsParser = new EmailAttachmentsParser(attachmentParsers); } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParserTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParserTests.java index e8678f5c40e91..97907bbef8a59 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParserTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParserTests.java @@ -8,7 +8,6 @@ import com.fasterxml.jackson.core.io.JsonEOFException; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -24,11 +23,7 @@ import org.elasticsearch.xpack.watcher.common.http.HttpProxy; import org.elasticsearch.xpack.watcher.common.http.HttpRequest; import org.elasticsearch.xpack.watcher.common.http.HttpResponse; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuth; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthFactory; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory; +import org.elasticsearch.xpack.watcher.common.http.BasicAuth; import org.elasticsearch.xpack.watcher.common.text.TextTemplate; import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine; import org.elasticsearch.xpack.watcher.notification.email.Attachment; @@ -69,19 +64,13 @@ public class ReportingAttachmentParserTests extends ESTestCase { private Map attachmentParsers = new HashMap<>(); private EmailAttachmentsParser emailAttachmentsParser; private ReportingAttachmentParser reportingAttachmentParser; - private HttpAuthRegistry authRegistry; private MockTextTemplateEngine templateEngine = new MockTextTemplateEngine(); private String dashboardUrl = "http://www.example.org/ovb/api/reporting/generate/dashboard/My-Dashboard"; @Before public void init() throws Exception { httpClient = mock(HttpClient.class); - - Map factories = MapBuilder.newMapBuilder() - .put("basic", new BasicAuthFactory(null)) - .immutableMap(); - authRegistry = new HttpAuthRegistry(factories); - reportingAttachmentParser = new ReportingAttachmentParser(Settings.EMPTY, httpClient, templateEngine, authRegistry); + reportingAttachmentParser = new ReportingAttachmentParser(Settings.EMPTY, httpClient, templateEngine); attachmentParsers.put(ReportingAttachmentParser.TYPE, reportingAttachmentParser); emailAttachmentsParser = new EmailAttachmentsParser(attachmentParsers); @@ -113,7 +102,7 @@ public void testSerializationWorks() throws Exception { builder.field("inline", true); } - HttpAuth auth = null; + BasicAuth auth = null; boolean withAuth = randomBoolean(); boolean isPasswordEncrypted = randomBoolean(); if (withAuth) { @@ -318,9 +307,8 @@ public void testWithBasicAuth() throws Exception { assertThat(allRequests, hasSize(3)); for (HttpRequest request : allRequests) { assertThat(request.auth(), is(notNullValue())); - assertThat(request.auth().type(), is("basic")); assertThat(request.auth(), instanceOf(BasicAuth.class)); - BasicAuth basicAuth = (BasicAuth) request.auth(); + BasicAuth basicAuth = request.auth(); assertThat(basicAuth.getUsername(), is("foo")); } } @@ -351,7 +339,7 @@ public void testPollingDefaultCanBeOverriddenBySettings() throws Exception { .put(ReportingAttachmentParser.RETRIES_SETTING.getKey(), retries) .build(); - reportingAttachmentParser = new ReportingAttachmentParser(settings, httpClient, templateEngine, authRegistry); + reportingAttachmentParser = new ReportingAttachmentParser(settings, httpClient, templateEngine); expectThrows(ElasticsearchException.class, () -> reportingAttachmentParser.toAttachment(createWatchExecutionContext(), Payload.EMPTY, attachment)); @@ -374,7 +362,7 @@ public String render(TextTemplate textTemplate, Map model) { ReportingAttachment attachment = new ReportingAttachment("foo", "http://www.example.org/REPLACEME", randomBoolean(), TimeValue.timeValueMillis(1), 10, new BasicAuth("foo", "bar".toCharArray()), null); reportingAttachmentParser = new ReportingAttachmentParser(Settings.EMPTY, httpClient, - replaceHttpWithHttpsTemplateEngine, authRegistry); + replaceHttpWithHttpsTemplateEngine); reportingAttachmentParser.toAttachment(createWatchExecutionContext(), Payload.EMPTY, attachment); ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(HttpRequest.class); @@ -391,7 +379,7 @@ public void testRetrySettingCannotBeNegative() throws Exception { Settings invalidSettings = Settings.builder().put("xpack.notification.reporting.retries", -10).build(); e = expectThrows(IllegalArgumentException.class, - () -> new ReportingAttachmentParser(invalidSettings, httpClient, templateEngine, authRegistry)); + () -> new ReportingAttachmentParser(invalidSettings, httpClient, templateEngine)); assertThat(e.getMessage(), is("Failed to parse value [-10] for setting [xpack.notification.reporting.retries] must be >= 0")); } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/jira/JiraIssueTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/jira/JiraIssueTests.java index 8e615d76050a7..e3c4d79f6d0e5 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/jira/JiraIssueTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/jira/JiraIssueTests.java @@ -14,14 +14,11 @@ import org.elasticsearch.xpack.watcher.common.http.HttpMethod; import org.elasticsearch.xpack.watcher.common.http.HttpRequest; import org.elasticsearch.xpack.watcher.common.http.HttpResponse; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory; +import org.elasticsearch.xpack.watcher.common.http.BasicAuth; import java.util.HashMap; import java.util.Map; -import static java.util.Collections.singletonMap; import static org.elasticsearch.common.xcontent.XContentFactory.cborBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.smileBuilder; @@ -43,19 +40,19 @@ public void testToXContent() throws Exception { Map parsedFields = null; Map parsedResult = null; - + HttpRequest parsedRequest = null; HttpResponse parsedResponse = null; String parsedAccount = null; String parsedReason = null; - + try (XContentParser parser = createParser(builder)) { assertNull(parser.currentToken()); parser.nextToken(); - + XContentParser.Token token = parser.currentToken(); assertThat(token, is(XContentParser.Token.START_OBJECT)); - + String currentFieldName = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -65,9 +62,7 @@ public void testToXContent() throws Exception { } else if ("result".equals(currentFieldName)) { parsedResult = parser.map(); } else if ("request".equals(currentFieldName)) { - HttpAuthRegistry registry = new HttpAuthRegistry(singletonMap(BasicAuth.TYPE, new BasicAuthFactory(null))); - HttpRequest.Parser httpRequestParser = new HttpRequest.Parser(registry); - parsedRequest = httpRequestParser.parse(parser); + parsedRequest = HttpRequest.Parser.parse(parser); } else if ("response".equals(currentFieldName)) { parsedResponse = HttpResponse.parse(parser); } else if ("fields".equals(currentFieldName)) { @@ -79,7 +74,7 @@ public void testToXContent() throws Exception { } } } - + assertThat(parsedAccount, equalTo(issue.getAccount())); assertThat(parsedFields, equalTo(issue.getFields())); if (issue.successful()) { diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java index 9b7050dc30d79..12a48230e4417 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java @@ -22,9 +22,8 @@ import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse; import org.elasticsearch.xpack.core.watcher.trigger.TriggerEvent; import org.elasticsearch.xpack.core.watcher.watch.Watch; +import org.elasticsearch.xpack.watcher.common.http.BasicAuth; import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.ApplicableBasicAuth; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth; import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition; import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase; import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTriggerEvent; @@ -32,6 +31,8 @@ import org.junit.After; import org.junit.Before; +import java.nio.charset.StandardCharsets; +import java.util.Base64; import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; @@ -154,7 +155,7 @@ public void testHttpInput() throws Exception { assertThat(webServer.requests(), hasSize(1)); assertThat(webServer.requests().get(0).getHeader("Authorization"), - is(ApplicableBasicAuth.headerValue(USERNAME, PASSWORD.toCharArray()))); + is(headerValue(USERNAME, PASSWORD.toCharArray()))); // now trigger the by the scheduler and make sure that the password is also correctly transmitted webServer.enqueue(new MockResponse().setResponseCode(200).setBody( @@ -162,7 +163,7 @@ public void testHttpInput() throws Exception { timeWarp().trigger("_id"); assertThat(webServer.requests(), hasSize(2)); assertThat(webServer.requests().get(1).getHeader("Authorization"), - is(ApplicableBasicAuth.headerValue(USERNAME, PASSWORD.toCharArray()))); + is(headerValue(USERNAME, PASSWORD.toCharArray()))); } @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/30094") @@ -252,6 +253,10 @@ public void testWebhookAction() throws Exception { assertThat(webServer.requests(), hasSize(1)); assertThat(webServer.requests().get(0).getHeader("Authorization"), - is(ApplicableBasicAuth.headerValue(USERNAME, PASSWORD.toCharArray()))); + is(headerValue(USERNAME, PASSWORD.toCharArray()))); + } + + private String headerValue(String username, char[] password) { + return "Basic " + Base64.getEncoder().encodeToString((username + ":" + new String(password)).getBytes(StandardCharsets.UTF_8)); } } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java index c7b8cf8c069b4..20d23982e20da 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java @@ -57,8 +57,6 @@ import org.elasticsearch.xpack.watcher.common.http.HttpClient; import org.elasticsearch.xpack.watcher.common.http.HttpMethod; import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate; -import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry; -import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory; import org.elasticsearch.xpack.watcher.common.text.TextTemplate; import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine; import org.elasticsearch.xpack.watcher.condition.AlwaysConditionTests; @@ -152,7 +150,6 @@ public class WatchTests extends ESTestCase { private EmailService emailService; private TextTemplateEngine templateEngine; private HtmlSanitizer htmlSanitizer; - private HttpAuthRegistry authRegistry; private XPackLicenseState licenseState; private Logger logger; private Settings settings = Settings.EMPTY; @@ -167,7 +164,6 @@ public void init() throws Exception { templateEngine = mock(TextTemplateEngine.class); htmlSanitizer = mock(HtmlSanitizer.class); licenseState = mock(XPackLicenseState.class); - authRegistry = new HttpAuthRegistry(singletonMap("basic", new BasicAuthFactory(null))); logger = Loggers.getLogger(WatchTests.class); searchTemplateService = mock(WatcherSearchTemplateService.class); } @@ -546,8 +542,7 @@ private ActionRegistry registry(List actions, ConditionRegistry c parsers.put(IndexAction.TYPE, new IndexActionFactory(settings, client)); break; case WebhookAction.TYPE: - parsers.put(WebhookAction.TYPE, new WebhookActionFactory(settings, httpClient, - new HttpRequestTemplate.Parser(authRegistry), templateEngine)); + parsers.put(WebhookAction.TYPE, new WebhookActionFactory(settings, httpClient, templateEngine)); break; } } From a5ec145851818ef2decba7460a51df24dc614e80 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Tue, 14 Aug 2018 11:07:58 -0500 Subject: [PATCH 03/42] Watcher: Remove unused hipchat render method (#32211) The HipChatMessage#render is no longer used, and instead the HipChatAccount#render is used in the ExecutableHipChatAction. Only a test that validated the HttpProxy used this render method still. This commit cleans it up. --- .../notification/hipchat/HipChatMessage.java | 22 ------------------- .../hipchat/HipChatAccountsTests.java | 9 +------- 2 files changed, 1 insertion(+), 30 deletions(-) diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatMessage.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatMessage.java index ae08bb85e9dfd..10eaba535edf1 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatMessage.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatMessage.java @@ -13,14 +13,12 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.watcher.common.text.TextTemplate; -import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Objects; public class HipChatMessage implements ToXContentObject { @@ -181,26 +179,6 @@ public int hashCode() { return Objects.hash(body, rooms, users, from, format, color, notify); } - public HipChatMessage render(TextTemplateEngine engine, Map model) { - String body = engine.render(this.body, model); - String[] rooms = null; - if (this.rooms != null) { - rooms = new String[this.rooms.length]; - for (int i = 0; i < this.rooms.length; i++) { - rooms[i] = engine.render(this.rooms[i], model); - } - } - String[] users = null; - if (this.users != null) { - users = new String[this.users.length]; - for (int i = 0; i < this.users.length; i++) { - users[i] = engine.render(this.users[i], model); - } - } - Color color = this.color == null ? null : Color.resolve(engine.render(this.color, model), null); - return new HipChatMessage(body, rooms, users, from, format, color, notify); - } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatAccountsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatAccountsTests.java index b793bf9202382..795f5aaacc7c9 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatAccountsTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatAccountsTests.java @@ -12,12 +12,9 @@ import org.elasticsearch.xpack.watcher.common.http.HttpProxy; import org.elasticsearch.xpack.watcher.common.http.HttpRequest; import org.elasticsearch.xpack.watcher.common.http.HttpResponse; -import org.elasticsearch.xpack.watcher.common.text.TextTemplate; -import org.elasticsearch.xpack.watcher.test.MockTextTemplateEngine; import org.junit.Before; import org.mockito.ArgumentCaptor; -import java.util.HashMap; import java.util.HashSet; import static org.hamcrest.Matchers.is; @@ -40,11 +37,7 @@ public void testProxy() throws Exception { new HashSet<>(HipChatService.getSettings()))); HipChatAccount account = service.getAccount("account1"); - HipChatMessage.Template template = new HipChatMessage.Template.Builder(new TextTemplate("foo")) - .addRooms(new TextTemplate("room")) - .setFrom("from") - .build(); - HipChatMessage hipChatMessage = template.render(new MockTextTemplateEngine(), new HashMap<>()); + HipChatMessage hipChatMessage = new HipChatMessage("body", new String[]{"rooms"}, null, "from", null, null, null); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(HttpRequest.class); when(httpClient.execute(argumentCaptor.capture())).thenReturn(new HttpResponse(200)); From e9622abdaccf1499da3fc321ae7fb0063025f5f3 Mon Sep 17 00:00:00 2001 From: Jay Modi Date: Wed, 15 Aug 2018 15:26:00 -0600 Subject: [PATCH 04/42] Move CharArrays to core lib (#32851) This change cleans up some methods in the CharArrays class from x-pack, which includes the unification of char[] to utf8 and utf8 to char[] conversions that intentionally do not use strings. There was previously an implementation in x-pack and in the reloading of secure settings. The method from the reloading of secure settings was adopted as it handled more scenarios related to the backing byte and char buffers that were used to perform the conversions. The cleaned up class is moved into libs/core to allow it to be used by requests that will be migrated to the high level rest client. Relates #32332 --- .../org/elasticsearch/common/CharArrays.java | 150 ++++++++++++++++++ .../elasticsearch/common/CharArraysTests.java | 72 +++++++++ .../NodesReloadSecureSettingsRequest.java | 66 +------- .../action/token/CreateTokenRequest.java | 2 +- .../action/user/ChangePasswordRequest.java | 2 +- .../security/action/user/PutUserRequest.java | 2 +- .../core/security/authc/support/BCrypt.java | 9 +- .../security/authc/support/CharArrays.java | 101 ------------ .../core/security/authc/support/Hasher.java | 1 + .../authc/support/UsernamePasswordToken.java | 14 +- .../xpack/core/ssl/PemUtils.java | 2 +- .../core/watcher/crypto/CryptoService.java | 2 +- .../ldap/ActiveDirectorySessionFactory.java | 2 +- .../authc/ldap/LdapSessionFactory.java | 2 +- .../ldap/LdapUserSearchSessionFactory.java | 2 +- .../authc/ldap/PoolingSessionFactory.java | 2 +- .../esnative/ESNativeMigrateToolTests.java | 2 +- .../example/realm/CustomRealm.java | 2 +- 18 files changed, 257 insertions(+), 178 deletions(-) create mode 100644 libs/core/src/main/java/org/elasticsearch/common/CharArrays.java create mode 100644 libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/CharArrays.java diff --git a/libs/core/src/main/java/org/elasticsearch/common/CharArrays.java b/libs/core/src/main/java/org/elasticsearch/common/CharArrays.java new file mode 100644 index 0000000000000..907874ca5735b --- /dev/null +++ b/libs/core/src/main/java/org/elasticsearch/common/CharArrays.java @@ -0,0 +1,150 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Objects; + +/** + * Helper class similar to Arrays to handle conversions for Char arrays + */ +public final class CharArrays { + + private CharArrays() {} + + /** + * Decodes the provided byte[] to a UTF-8 char[]. This is done while avoiding + * conversions to String. The provided byte[] is not modified by this method, so + * the caller needs to take care of clearing the value if it is sensitive. + */ + public static char[] utf8BytesToChars(byte[] utf8Bytes) { + final ByteBuffer byteBuffer = ByteBuffer.wrap(utf8Bytes); + final CharBuffer charBuffer = StandardCharsets.UTF_8.decode(byteBuffer); + final char[] chars; + if (charBuffer.hasArray()) { + // there is no guarantee that the char buffers backing array is the right size + // so we need to make a copy + chars = Arrays.copyOfRange(charBuffer.array(), charBuffer.position(), charBuffer.limit()); + Arrays.fill(charBuffer.array(), (char) 0); // clear sensitive data + } else { + final int length = charBuffer.limit() - charBuffer.position(); + chars = new char[length]; + charBuffer.get(chars); + // if the buffer is not read only we can reset and fill with 0's + if (charBuffer.isReadOnly() == false) { + charBuffer.clear(); // reset + for (int i = 0; i < charBuffer.limit(); i++) { + charBuffer.put((char) 0); + } + } + } + return chars; + } + + /** + * Encodes the provided char[] to a UTF-8 byte[]. This is done while avoiding + * conversions to String. The provided char[] is not modified by this method, so + * the caller needs to take care of clearing the value if it is sensitive. + */ + public static byte[] toUtf8Bytes(char[] chars) { + final CharBuffer charBuffer = CharBuffer.wrap(chars); + final ByteBuffer byteBuffer = StandardCharsets.UTF_8.encode(charBuffer); + final byte[] bytes; + if (byteBuffer.hasArray()) { + // there is no guarantee that the byte buffers backing array is the right size + // so we need to make a copy + bytes = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.position(), byteBuffer.limit()); + Arrays.fill(byteBuffer.array(), (byte) 0); // clear sensitive data + } else { + final int length = byteBuffer.limit() - byteBuffer.position(); + bytes = new byte[length]; + byteBuffer.get(bytes); + // if the buffer is not read only we can reset and fill with 0's + if (byteBuffer.isReadOnly() == false) { + byteBuffer.clear(); // reset + for (int i = 0; i < byteBuffer.limit(); i++) { + byteBuffer.put((byte) 0); + } + } + } + return bytes; + } + + /** + * Tests if a char[] contains a sequence of characters that match the prefix. This is like + * {@link String#startsWith(String)} but does not require conversion of the char[] to a string. + */ + public static boolean charsBeginsWith(String prefix, char[] chars) { + if (chars == null || prefix == null) { + return false; + } + + if (prefix.length() > chars.length) { + return false; + } + + for (int i = 0; i < prefix.length(); i++) { + if (chars[i] != prefix.charAt(i)) { + return false; + } + } + + return true; + } + + /** + * Constant time equality check of char arrays to avoid potential timing attacks. + */ + public static boolean constantTimeEquals(char[] a, char[] b) { + Objects.requireNonNull(a, "char arrays must not be null for constantTimeEquals"); + Objects.requireNonNull(b, "char arrays must not be null for constantTimeEquals"); + if (a.length != b.length) { + return false; + } + + int equals = 0; + for (int i = 0; i < a.length; i++) { + equals |= a[i] ^ b[i]; + } + + return equals == 0; + } + + /** + * Constant time equality check of strings to avoid potential timing attacks. + */ + public static boolean constantTimeEquals(String a, String b) { + Objects.requireNonNull(a, "strings must not be null for constantTimeEquals"); + Objects.requireNonNull(b, "strings must not be null for constantTimeEquals"); + if (a.length() != b.length()) { + return false; + } + + int equals = 0; + for (int i = 0; i < a.length(); i++) { + equals |= a.charAt(i) ^ b.charAt(i); + } + + return equals == 0; + } +} diff --git a/libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java b/libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java new file mode 100644 index 0000000000000..64b1ecd1f8a2d --- /dev/null +++ b/libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common; + +import org.elasticsearch.test.ESTestCase; + +import java.nio.charset.StandardCharsets; + +public class CharArraysTests extends ESTestCase { + + public void testCharsToBytes() { + final String originalValue = randomUnicodeOfCodepointLengthBetween(0, 32); + final byte[] expectedBytes = originalValue.getBytes(StandardCharsets.UTF_8); + final char[] valueChars = originalValue.toCharArray(); + + final byte[] convertedBytes = CharArrays.toUtf8Bytes(valueChars); + assertArrayEquals(expectedBytes, convertedBytes); + } + + public void testBytesToUtf8Chars() { + final String originalValue = randomUnicodeOfCodepointLengthBetween(0, 32); + final byte[] bytes = originalValue.getBytes(StandardCharsets.UTF_8); + final char[] expectedChars = originalValue.toCharArray(); + + final char[] convertedChars = CharArrays.utf8BytesToChars(bytes); + assertArrayEquals(expectedChars, convertedChars); + } + + public void testCharsBeginsWith() { + assertFalse(CharArrays.charsBeginsWith(randomAlphaOfLength(4), null)); + assertFalse(CharArrays.charsBeginsWith(null, null)); + assertFalse(CharArrays.charsBeginsWith(null, randomAlphaOfLength(4).toCharArray())); + assertFalse(CharArrays.charsBeginsWith(randomAlphaOfLength(2), randomAlphaOfLengthBetween(3, 8).toCharArray())); + + final String prefix = randomAlphaOfLengthBetween(2, 4); + assertTrue(CharArrays.charsBeginsWith(prefix, prefix.toCharArray())); + final char[] prefixedValue = prefix.concat(randomAlphaOfLengthBetween(1, 12)).toCharArray(); + assertTrue(CharArrays.charsBeginsWith(prefix, prefixedValue)); + + final String modifiedPrefix = randomBoolean() ? prefix.substring(1) : prefix.substring(0, prefix.length() - 1); + final char[] nonMatchingValue = modifiedPrefix.concat(randomAlphaOfLengthBetween(0, 12)).toCharArray(); + assertFalse(CharArrays.charsBeginsWith(prefix, nonMatchingValue)); + assertTrue(CharArrays.charsBeginsWith(modifiedPrefix, nonMatchingValue)); + } + + public void testConstantTimeEquals() { + final String value = randomAlphaOfLengthBetween(0, 32); + assertTrue(CharArrays.constantTimeEquals(value, value)); + assertTrue(CharArrays.constantTimeEquals(value.toCharArray(), value.toCharArray())); + + final String other = randomAlphaOfLengthBetween(1, 32); + assertFalse(CharArrays.constantTimeEquals(value, other)); + assertFalse(CharArrays.constantTimeEquals(value.toCharArray(), other.toCharArray())); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java index 50df7b1bb26e0..5320470d366e1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java @@ -22,14 +22,12 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.nodes.BaseNodesRequest; +import org.elasticsearch.common.CharArrays; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.SecureString; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import static org.elasticsearch.action.ValidateActions.addValidationError; @@ -83,7 +81,7 @@ public void readFrom(StreamInput in) throws IOException { super.readFrom(in); final byte[] passwordBytes = in.readByteArray(); try { - this.secureSettingsPassword = new SecureString(utf8BytesToChars(passwordBytes)); + this.secureSettingsPassword = new SecureString(CharArrays.utf8BytesToChars(passwordBytes)); } finally { Arrays.fill(passwordBytes, (byte) 0); } @@ -92,69 +90,11 @@ public void readFrom(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - final byte[] passwordBytes = charsToUtf8Bytes(this.secureSettingsPassword.getChars()); + final byte[] passwordBytes = CharArrays.toUtf8Bytes(this.secureSettingsPassword.getChars()); try { out.writeByteArray(passwordBytes); } finally { Arrays.fill(passwordBytes, (byte) 0); } } - - /** - * Encodes the provided char[] to a UTF-8 byte[]. This is done while avoiding - * conversions to String. The provided char[] is not modified by this method, so - * the caller needs to take care of clearing the value if it is sensitive. - */ - private static byte[] charsToUtf8Bytes(char[] chars) { - final CharBuffer charBuffer = CharBuffer.wrap(chars); - final ByteBuffer byteBuffer = StandardCharsets.UTF_8.encode(charBuffer); - final byte[] bytes; - if (byteBuffer.hasArray()) { - // there is no guarantee that the byte buffers backing array is the right size - // so we need to make a copy - bytes = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.position(), byteBuffer.limit()); - Arrays.fill(byteBuffer.array(), (byte) 0); // clear sensitive data - } else { - final int length = byteBuffer.limit() - byteBuffer.position(); - bytes = new byte[length]; - byteBuffer.get(bytes); - // if the buffer is not read only we can reset and fill with 0's - if (byteBuffer.isReadOnly() == false) { - byteBuffer.clear(); // reset - for (int i = 0; i < byteBuffer.limit(); i++) { - byteBuffer.put((byte) 0); - } - } - } - return bytes; - } - - /** - * Decodes the provided byte[] to a UTF-8 char[]. This is done while avoiding - * conversions to String. The provided byte[] is not modified by this method, so - * the caller needs to take care of clearing the value if it is sensitive. - */ - public static char[] utf8BytesToChars(byte[] utf8Bytes) { - final ByteBuffer byteBuffer = ByteBuffer.wrap(utf8Bytes); - final CharBuffer charBuffer = StandardCharsets.UTF_8.decode(byteBuffer); - final char[] chars; - if (charBuffer.hasArray()) { - // there is no guarantee that the char buffers backing array is the right size - // so we need to make a copy - chars = Arrays.copyOfRange(charBuffer.array(), charBuffer.position(), charBuffer.limit()); - Arrays.fill(charBuffer.array(), (char) 0); // clear sensitive data - } else { - final int length = charBuffer.limit() - charBuffer.position(); - chars = new char[length]; - charBuffer.get(chars); - // if the buffer is not read only we can reset and fill with 0's - if (charBuffer.isReadOnly() == false) { - charBuffer.clear(); // reset - for (int i = 0; i < charBuffer.limit(); i++) { - charBuffer.put((char) 0); - } - } - } - return chars; - } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequest.java index 5956e1a661345..fdb46711c0c59 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequest.java @@ -15,7 +15,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.xpack.core.security.authc.support.CharArrays; +import org.elasticsearch.common.CharArrays; import java.io.IOException; import java.util.Arrays; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequest.java index f84b133d984b6..b78b81c060080 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequest.java @@ -12,7 +12,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.core.security.authc.support.CharArrays; +import org.elasticsearch.common.CharArrays; import java.io.IOException; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequest.java index f37072b9cf0fc..e704259396a34 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequest.java @@ -8,12 +8,12 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.common.CharArrays; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.core.security.authc.support.CharArrays; import java.io.IOException; import java.util.Map; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/BCrypt.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/BCrypt.java index ceb93dc4c853c..a93476bbdc8da 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/BCrypt.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/BCrypt.java @@ -14,6 +14,7 @@ // ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF // OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +import org.elasticsearch.common.CharArrays; import org.elasticsearch.common.settings.SecureString; import java.security.SecureRandom; @@ -54,7 +55,7 @@ * String stronger_salt = BCrypt.gensalt(12)
* *

- * The amount of work increases exponentially (2**log_rounds), so + * The amount of work increases exponentially (2**log_rounds), so * each increment is twice as much work. The default log_rounds is * 10, and the valid range is 4 to 30. * @@ -689,7 +690,11 @@ public static String hashpw(SecureString password, String salt) { // the next lines are the SecureString replacement for the above commented-out section if (minor >= 'a') { - try (SecureString secureString = new SecureString(CharArrays.concat(password.getChars(), "\000".toCharArray()))) { + final char[] suffix = "\000".toCharArray(); + final char[] result = new char[password.length() + suffix.length]; + System.arraycopy(password.getChars(), 0, result, 0, password.length()); + System.arraycopy(suffix, 0, result, password.length(), suffix.length); + try (SecureString secureString = new SecureString(result)) { passwordb = CharArrays.toUtf8Bytes(secureString.getChars()); } } else { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/CharArrays.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/CharArrays.java deleted file mode 100644 index 26df90c31a2de..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/CharArrays.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.core.security.authc.support; - -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; - -/** - * Helper class similar to Arrays to handle conversions for Char arrays - */ -public class CharArrays { - - public static char[] utf8BytesToChars(byte[] utf8Bytes) { - ByteBuffer byteBuffer = ByteBuffer.wrap(utf8Bytes); - CharBuffer charBuffer = StandardCharsets.UTF_8.decode(byteBuffer); - char[] chars = Arrays.copyOfRange(charBuffer.array(), charBuffer.position(), charBuffer.limit()); - byteBuffer.clear(); - charBuffer.clear(); - return chars; - } - - /** - * Like String.indexOf for for an array of chars - */ - static int indexOf(char[] array, char ch) { - for (int i = 0; (i < array.length); i++) { - if (array[i] == ch) { - return i; - } - } - return -1; - } - - /** - * Converts the provided char[] to a UTF-8 byte[]. The provided char[] is not modified by this - * method, so the caller needs to take care of clearing the value if it is sensitive. - */ - public static byte[] toUtf8Bytes(char[] chars) { - CharBuffer charBuffer = CharBuffer.wrap(chars); - ByteBuffer byteBuffer = StandardCharsets.UTF_8.encode(charBuffer); - byte[] bytes = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.position(), byteBuffer.limit()); - Arrays.fill(byteBuffer.array(), (byte) 0); // clear sensitive data - return bytes; - } - - public static boolean charsBeginsWith(String prefix, char[] chars) { - if (chars == null || prefix == null) { - return false; - } - - if (prefix.length() > chars.length) { - return false; - } - - for (int i = 0; i < prefix.length(); i++) { - if (chars[i] != prefix.charAt(i)) { - return false; - } - } - - return true; - } - - public static boolean constantTimeEquals(char[] a, char[] b) { - if (a.length != b.length) { - return false; - } - - int equals = 0; - for (int i = 0; i < a.length; i++) { - equals |= a[i] ^ b[i]; - } - - return equals == 0; - } - - public static boolean constantTimeEquals(String a, String b) { - if (a.length() != b.length()) { - return false; - } - - int equals = 0; - for (int i = 0; i < a.length(); i++) { - equals |= a.charAt(i) ^ b.charAt(i); - } - - return equals == 0; - } - - public static char[] concat(char[] a, char[] b) { - final char[] result = new char[a.length + b.length]; - System.arraycopy(a, 0, result, 0, a.length); - System.arraycopy(b, 0, result, a.length, b.length); - return result; - } -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java index d12547bd90645..492622b2c519c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.core.security.authc.support; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.CharArrays; import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.settings.SecureString; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/UsernamePasswordToken.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/UsernamePasswordToken.java index d8e58c29d237b..1349303600884 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/UsernamePasswordToken.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/UsernamePasswordToken.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.security.authc.support; +import org.elasticsearch.common.CharArrays; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -107,7 +108,7 @@ private static UsernamePasswordToken extractToken(String headerValue) { throw authenticationError("invalid basic authentication header encoding", e); } - int i = CharArrays.indexOf(userpasswd, ':'); + int i = indexOfColon(userpasswd); if (i < 0) { throw authenticationError("invalid basic authentication header value"); } @@ -121,4 +122,15 @@ public static void putTokenHeader(ThreadContext context, UsernamePasswordToken t context.putHeader(BASIC_AUTH_HEADER, basicAuthHeaderValue(token.username, token.password)); } + /** + * Like String.indexOf for for an array of chars + */ + private static int indexOfColon(char[] array) { + for (int i = 0; (i < array.length); i++) { + if (array[i] == ':') { + return i; + } + } + return -1; + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PemUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PemUtils.java index d959c017e0a35..a3814a76a3e6e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PemUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PemUtils.java @@ -7,7 +7,7 @@ package org.elasticsearch.xpack.core.ssl; import org.elasticsearch.common.hash.MessageDigests; -import org.elasticsearch.xpack.core.security.authc.support.CharArrays; +import org.elasticsearch.common.CharArrays; import java.io.BufferedReader; import java.io.IOException; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/crypto/CryptoService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/crypto/CryptoService.java index b1f3a32769ec9..a25e79ffdf66f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/crypto/CryptoService.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/crypto/CryptoService.java @@ -13,7 +13,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.core.watcher.WatcherField; import org.elasticsearch.xpack.core.security.SecurityField; -import org.elasticsearch.xpack.core.security.authc.support.CharArrays; +import org.elasticsearch.common.CharArrays; import javax.crypto.BadPaddingException; import javax.crypto.Cipher; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java index d175e1b229312..8107d7488188b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java @@ -32,7 +32,7 @@ import org.elasticsearch.xpack.core.security.authc.ldap.ActiveDirectorySessionFactorySettings; import org.elasticsearch.xpack.core.security.authc.ldap.PoolingSessionFactorySettings; import org.elasticsearch.xpack.core.security.authc.ldap.support.LdapSearchScope; -import org.elasticsearch.xpack.core.security.authc.support.CharArrays; +import org.elasticsearch.common.CharArrays; import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.security.authc.ldap.support.LdapMetaDataResolver; import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapSessionFactory.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapSessionFactory.java index 36d14aa67c0de..70b2f0015cf7a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapSessionFactory.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapSessionFactory.java @@ -19,7 +19,7 @@ import org.elasticsearch.xpack.core.security.authc.RealmSettings; import org.elasticsearch.xpack.core.security.authc.ldap.LdapSessionFactorySettings; import org.elasticsearch.xpack.core.security.authc.ldap.SearchGroupsResolverSettings; -import org.elasticsearch.xpack.core.security.authc.support.CharArrays; +import org.elasticsearch.common.CharArrays; import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.security.authc.ldap.support.LdapMetaDataResolver; import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactory.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactory.java index 2ec87888d8c13..a3541ec2759b3 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactory.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactory.java @@ -24,7 +24,7 @@ import org.elasticsearch.xpack.core.security.authc.ldap.LdapUserSearchSessionFactorySettings; import org.elasticsearch.xpack.core.security.authc.ldap.SearchGroupsResolverSettings; import org.elasticsearch.xpack.core.security.authc.ldap.support.LdapSearchScope; -import org.elasticsearch.xpack.core.security.authc.support.CharArrays; +import org.elasticsearch.common.CharArrays; import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession; import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession.GroupsResolver; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/PoolingSessionFactory.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/PoolingSessionFactory.java index 367bd525036e2..986fa1900e7c8 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/PoolingSessionFactory.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/PoolingSessionFactory.java @@ -25,7 +25,7 @@ import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.RealmSettings; import org.elasticsearch.xpack.core.security.authc.ldap.PoolingSessionFactorySettings; -import org.elasticsearch.xpack.core.security.authc.support.CharArrays; +import org.elasticsearch.common.CharArrays; import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.security.authc.ldap.support.LdapMetaDataResolver; import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java index fbb76100c798a..3119640dc640b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java @@ -14,7 +14,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.test.NativeRealmIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; -import org.elasticsearch.xpack.core.security.authc.support.CharArrays; +import org.elasticsearch.common.CharArrays; import org.elasticsearch.xpack.core.security.client.SecurityClient; import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.BeforeClass; diff --git a/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/realm/CustomRealm.java b/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/realm/CustomRealm.java index af3fb160e133f..c6502c05d252f 100644 --- a/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/realm/CustomRealm.java +++ b/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/realm/CustomRealm.java @@ -12,7 +12,7 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; import org.elasticsearch.xpack.core.security.authc.Realm; import org.elasticsearch.xpack.core.security.authc.RealmConfig; -import org.elasticsearch.xpack.core.security.authc.support.CharArrays; +import org.elasticsearch.common.CharArrays; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.protocol.xpack.security.User; From bfcbe9e14a5151d90a99d021644903d3ba87318a Mon Sep 17 00:00:00 2001 From: markharwood Date: Thu, 16 Aug 2018 10:21:37 +0100 Subject: [PATCH 05/42] Docs enhancement: added reference to cluster-level setting `search.default_allow_partial_results` (#32810) Closes #32809 --- docs/reference/search/request-body.asciidoc | 3 ++- docs/reference/search/uri-request.asciidoc | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc index 2a51d705d83ec..e7c9b593af372 100644 --- a/docs/reference/search/request-body.asciidoc +++ b/docs/reference/search/request-body.asciidoc @@ -90,7 +90,8 @@ And here is a sample response: Set to `false` to return an overall failure if the request would produce partial results. Defaults to true, which will allow partial results in the case of timeouts - or partial failures. + or partial failures. This default can be controlled using the cluster-level setting + `search.default_allow_partial_results`. `terminate_after`:: diff --git a/docs/reference/search/uri-request.asciidoc b/docs/reference/search/uri-request.asciidoc index a90f32bb3cd36..279bc0c0384c1 100644 --- a/docs/reference/search/uri-request.asciidoc +++ b/docs/reference/search/uri-request.asciidoc @@ -125,5 +125,6 @@ more details on the different types of search that can be performed. |`allow_partial_search_results` |Set to `false` to return an overall failure if the request would produce partial results. Defaults to true, which will allow partial results in the case of timeouts -or partial failures.. +or partial failures. This default can be controlled using the cluster-level setting +`search.default_allow_partial_results`. |======================================================================= From 708aff9b27ffeac655e7b395f73f74df7779020a Mon Sep 17 00:00:00 2001 From: Ed Savage <32410745+edsavage@users.noreply.github.com> Date: Wed, 15 Aug 2018 13:13:32 +0100 Subject: [PATCH 06/42] [ML] Removing old per-partition normalization code (#32816) [ML] Removing old per-partition normalization code Per-partition normalization is an old, undocumented feature that was never used by clients. It has been superseded by per-partition maximum scoring. To maintain communication compatibility with nodes prior to 6.5 it is necessary to maintain/cope with the old wire format --- .../core/ml/job/config/AnalysisConfig.java | 60 ++------ .../xpack/core/ml/job/messages/Messages.java | 4 - .../persistence/ElasticsearchMappings.java | 21 +-- .../xpack/core/ml/job/results/Bucket.java | 60 ++++---- .../core/ml/job/results/PartitionScore.java | 131 ------------------ .../ml/job/results/ReservedFieldNames.java | 1 - .../action/GetBucketActionResponseTests.java | 10 -- .../ml/job/config/AnalysisConfigTests.java | 42 +----- .../xpack/ml/MachineLearning.java | 4 +- .../process/autodetect/AutodetectBuilder.java | 5 - .../autodetect/AutodetectProcessManager.java | 2 +- .../normalizer/BucketNormalizable.java | 11 +- .../NativeNormalizerProcessFactory.java | 9 +- .../job/process/normalizer/Normalizable.java | 2 +- .../ml/job/process/normalizer/Normalizer.java | 5 +- .../process/normalizer/NormalizerBuilder.java | 8 +- .../normalizer/NormalizerProcessFactory.java | 3 +- .../PartitionScoreNormalizable.java | 87 ------------ .../job/process/normalizer/ScoresUpdater.java | 26 ++-- .../ShortCircuitingRenormalizer.java | 8 +- .../autodetect/AutodetectBuilderTests.java | 7 +- .../AutoDetectResultProcessorTests.java | 19 --- .../normalizer/BucketNormalizableTests.java | 15 +- .../normalizer/NormalizerBuilderTests.java | 7 +- .../process/normalizer/NormalizerTests.java | 4 +- .../normalizer/ScoresUpdaterTests.java | 29 ++-- .../ShortCircuitingRenormalizerTests.java | 12 +- .../xpack/ml/job/results/BucketTests.java | 49 +------ .../ml/job/results/PartitionScoreTests.java | 54 -------- 29 files changed, 99 insertions(+), 596 deletions(-) delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/PartitionScore.java delete mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/PartitionScoreNormalizable.java delete mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/PartitionScoreTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java index 5badd073c66c7..135ad755359e6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java @@ -64,7 +64,6 @@ public class AnalysisConfig implements ToXContentObject, Writeable { private static final ParseField OVERLAPPING_BUCKETS = new ParseField("overlapping_buckets"); private static final ParseField RESULT_FINALIZATION_WINDOW = new ParseField("result_finalization_window"); private static final ParseField MULTIVARIATE_BY_FIELDS = new ParseField("multivariate_by_fields"); - private static final ParseField USER_PER_PARTITION_NORMALIZATION = new ParseField("use_per_partition_normalization"); public static final String ML_CATEGORY_FIELD = "mlcategory"; public static final Set AUTO_CREATED_FIELDS = new HashSet<>(Collections.singletonList(ML_CATEGORY_FIELD)); @@ -98,7 +97,6 @@ private static ConstructingObjectParser createPars parser.declareBoolean(Builder::setOverlappingBuckets, OVERLAPPING_BUCKETS); parser.declareLong(Builder::setResultFinalizationWindow, RESULT_FINALIZATION_WINDOW); parser.declareBoolean(Builder::setMultivariateByFields, MULTIVARIATE_BY_FIELDS); - parser.declareBoolean(Builder::setUsePerPartitionNormalization, USER_PER_PARTITION_NORMALIZATION); return parser; } @@ -117,12 +115,11 @@ private static ConstructingObjectParser createPars private final Boolean overlappingBuckets; private final Long resultFinalizationWindow; private final Boolean multivariateByFields; - private final boolean usePerPartitionNormalization; private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, List categorizationFilters, CategorizationAnalyzerConfig categorizationAnalyzerConfig, TimeValue latency, String summaryCountFieldName, List detectors, List influencers, Boolean overlappingBuckets, Long resultFinalizationWindow, - Boolean multivariateByFields, boolean usePerPartitionNormalization) { + Boolean multivariateByFields) { this.detectors = detectors; this.bucketSpan = bucketSpan; this.latency = latency; @@ -134,7 +131,6 @@ private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, Lis this.overlappingBuckets = overlappingBuckets; this.resultFinalizationWindow = resultFinalizationWindow; this.multivariateByFields = multivariateByFields; - this.usePerPartitionNormalization = usePerPartitionNormalization; } public AnalysisConfig(StreamInput in) throws IOException { @@ -165,7 +161,12 @@ public AnalysisConfig(StreamInput in) throws IOException { } } - usePerPartitionNormalization = in.readBoolean(); + // BWC for removed per-partition normalization + // Version check is temporarily against the latest to satisfy CI tests + // TODO change to V_6_5_0 after successful backport to 6.x + if (in.getVersion().before(Version.V_7_0_0_alpha1)) { + in.readBoolean(); + } } @Override @@ -195,7 +196,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(false); } - out.writeBoolean(usePerPartitionNormalization); + // BWC for removed per-partition normalization + // Version check is temporarily against the latest to satisfy CI tests + // TODO change to V_6_5_0 after successful backport to 6.x + if (out.getVersion().before(Version.V_7_0_0_alpha1)) { + out.writeBoolean(false); + } } /** @@ -299,10 +305,6 @@ public Boolean getMultivariateByFields() { return multivariateByFields; } - public boolean getUsePerPartitionNormalization() { - return usePerPartitionNormalization; - } - /** * Return the set of fields required by the analysis. * These are the influencer fields, metric field, partition field, @@ -403,9 +405,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (multivariateByFields != null) { builder.field(MULTIVARIATE_BY_FIELDS.getPreferredName(), multivariateByFields); } - if (usePerPartitionNormalization) { - builder.field(USER_PER_PARTITION_NORMALIZATION.getPreferredName(), usePerPartitionNormalization); - } builder.endObject(); return builder; } @@ -416,7 +415,6 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; AnalysisConfig that = (AnalysisConfig) o; return Objects.equals(latency, that.latency) && - usePerPartitionNormalization == that.usePerPartitionNormalization && Objects.equals(bucketSpan, that.bucketSpan) && Objects.equals(categorizationFieldName, that.categorizationFieldName) && Objects.equals(categorizationFilters, that.categorizationFilters) && @@ -434,7 +432,7 @@ public int hashCode() { return Objects.hash( bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig, latency, summaryCountFieldName, detectors, influencers, overlappingBuckets, resultFinalizationWindow, - multivariateByFields, usePerPartitionNormalization + multivariateByFields ); } @@ -453,7 +451,6 @@ public static class Builder { private Boolean overlappingBuckets; private Long resultFinalizationWindow; private Boolean multivariateByFields; - private boolean usePerPartitionNormalization = false; public Builder(List detectors) { setDetectors(detectors); @@ -472,7 +469,6 @@ public Builder(AnalysisConfig analysisConfig) { this.overlappingBuckets = analysisConfig.overlappingBuckets; this.resultFinalizationWindow = analysisConfig.resultFinalizationWindow; this.multivariateByFields = analysisConfig.multivariateByFields; - this.usePerPartitionNormalization = analysisConfig.usePerPartitionNormalization; } public void setDetectors(List detectors) { @@ -535,10 +531,6 @@ public void setMultivariateByFields(Boolean multivariateByFields) { this.multivariateByFields = multivariateByFields; } - public void setUsePerPartitionNormalization(boolean usePerPartitionNormalization) { - this.usePerPartitionNormalization = usePerPartitionNormalization; - } - /** * Checks the configuration is valid *

    @@ -571,16 +563,11 @@ public AnalysisConfig build() { overlappingBuckets = verifyOverlappingBucketsConfig(overlappingBuckets, detectors); - if (usePerPartitionNormalization) { - checkDetectorsHavePartitionFields(detectors); - checkNoInfluencersAreSet(influencers); - } - verifyNoInconsistentNestedFieldNames(); return new AnalysisConfig(bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig, latency, summaryCountFieldName, detectors, influencers, overlappingBuckets, - resultFinalizationWindow, multivariateByFields, usePerPartitionNormalization); + resultFinalizationWindow, multivariateByFields); } private void verifyNoMetricFunctionsWhenSummaryCountFieldNameIsSet() { @@ -704,23 +691,6 @@ private void verifyCategorizationFiltersAreValidRegex() { } } - private static void checkDetectorsHavePartitionFields(List detectors) { - for (Detector detector : detectors) { - if (!Strings.isNullOrEmpty(detector.getPartitionFieldName())) { - return; - } - } - throw ExceptionsHelper.badRequestException(Messages.getMessage( - Messages.JOB_CONFIG_PER_PARTITION_NORMALIZATION_REQUIRES_PARTITION_FIELD)); - } - - private static void checkNoInfluencersAreSet(List influencers) { - if (!influencers.isEmpty()) { - throw ExceptionsHelper.badRequestException(Messages.getMessage( - Messages.JOB_CONFIG_PER_PARTITION_NORMALIZATION_CANNOT_USE_INFLUENCERS)); - } - } - private static boolean isValidRegex(String exp) { try { Pattern.compile(exp); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java index 289839b9342bc..7411115bda358 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java @@ -130,10 +130,6 @@ public final class Messages { "over_field_name cannot be used with function ''{0}''"; public static final String JOB_CONFIG_OVERLAPPING_BUCKETS_INCOMPATIBLE_FUNCTION = "Overlapping buckets cannot be used with function ''{0}''"; - public static final String JOB_CONFIG_PER_PARTITION_NORMALIZATION_CANNOT_USE_INFLUENCERS = - "A job configured with Per-Partition Normalization cannot use influencers"; - public static final String JOB_CONFIG_PER_PARTITION_NORMALIZATION_REQUIRES_PARTITION_FIELD = - "If the job is configured with Per-Partition Normalization enabled a detector must have a partition field"; public static final String JOB_CONFIG_UNKNOWN_FUNCTION = "Unknown function ''{0}''"; public static final String JOB_CONFIG_UPDATE_ANALYSIS_LIMITS_MODEL_MEMORY_LIMIT_CANNOT_BE_DECREASED = "Invalid update value for analysis_limits: model_memory_limit cannot be decreased below current usage; " + diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java index bc420c658d0b5..316417f4b23aa 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java @@ -227,23 +227,6 @@ private static void addResultsMapping(XContentBuilder builder) throws IOExceptio .startObject(Bucket.SCHEDULED_EVENTS.getPreferredName()) .field(TYPE, KEYWORD) .endObject() - .startObject(Bucket.PARTITION_SCORES.getPreferredName()) - .field(TYPE, NESTED) - .startObject(PROPERTIES) - .startObject(AnomalyRecord.PARTITION_FIELD_NAME.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Bucket.INITIAL_ANOMALY_SCORE.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(AnomalyRecord.PROBABILITY.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .endObject() - .endObject() .startObject(Bucket.BUCKET_INFLUENCERS.getPreferredName()) .field(TYPE, NESTED) @@ -328,7 +311,7 @@ private static void addTermFields(XContentBuilder builder, Collection te } private static void addForecastFieldsToMapping(XContentBuilder builder) throws IOException { - + // Forecast Output builder.startObject(Forecast.FORECAST_LOWER.getPreferredName()) .field(TYPE, DOUBLE) @@ -370,7 +353,7 @@ private static void addForecastFieldsToMapping(XContentBuilder builder) throws I .field(TYPE, LONG) .endObject(); } - + /** * AnomalyRecord fields to be added under the 'properties' section of the mapping * @param builder Add properties to this builder diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Bucket.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Bucket.java index 8a88232a559d4..8a7fe2395b4e0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Bucket.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Bucket.java @@ -25,7 +25,6 @@ import java.util.Date; import java.util.List; import java.util.Objects; -import java.util.Optional; /** * Bucket Result POJO @@ -43,7 +42,6 @@ public class Bucket implements ToXContentObject, Writeable { public static final ParseField BUCKET_INFLUENCERS = new ParseField("bucket_influencers"); public static final ParseField BUCKET_SPAN = new ParseField("bucket_span"); public static final ParseField PROCESSING_TIME_MS = new ParseField("processing_time_ms"); - public static final ParseField PARTITION_SCORES = new ParseField("partition_scores"); public static final ParseField SCHEDULED_EVENTS = new ParseField("scheduled_events"); // Used for QueryPage @@ -58,6 +56,19 @@ public class Bucket implements ToXContentObject, Writeable { public static final ConstructingObjectParser STRICT_PARSER = createParser(false); public static final ConstructingObjectParser LENIENT_PARSER = createParser(true); + /* * + * Read and discard the old (prior to 6.5) perPartitionNormalization values + */ + public static Bucket readOldPerPartitionNormalization(StreamInput in) throws IOException { + in.readString(); + in.readString(); + in.readDouble(); + in.readDouble(); + in.readDouble(); + + return null; + } + private static ConstructingObjectParser createParser(boolean ignoreUnknownFields) { ConstructingObjectParser parser = new ConstructingObjectParser<>(RESULT_TYPE_VALUE, ignoreUnknownFields, a -> new Bucket((String) a[0], (Date) a[1], (long) a[2])); @@ -82,8 +93,6 @@ private static ConstructingObjectParser createParser(boolean ignor parser.declareObjectArray(Bucket::setBucketInfluencers, ignoreUnknownFields ? BucketInfluencer.LENIENT_PARSER : BucketInfluencer.STRICT_PARSER, BUCKET_INFLUENCERS); parser.declareLong(Bucket::setProcessingTimeMs, PROCESSING_TIME_MS); - parser.declareObjectArray(Bucket::setPartitionScores, ignoreUnknownFields ? - PartitionScore.LENIENT_PARSER : PartitionScore.STRICT_PARSER, PARTITION_SCORES); parser.declareString((bucket, s) -> {}, Result.RESULT_TYPE); parser.declareStringArray(Bucket::setScheduledEvents, SCHEDULED_EVENTS); @@ -100,7 +109,6 @@ private static ConstructingObjectParser createParser(boolean ignor private boolean isInterim; private List bucketInfluencers = new ArrayList<>(); // Can't use emptyList as might be appended to private long processingTimeMs; - private List partitionScores = Collections.emptyList(); private List scheduledEvents = Collections.emptyList(); public Bucket(String jobId, Date timestamp, long bucketSpan) { @@ -120,7 +128,6 @@ public Bucket(Bucket other) { this.isInterim = other.isInterim; this.bucketInfluencers = new ArrayList<>(other.bucketInfluencers); this.processingTimeMs = other.processingTimeMs; - this.partitionScores = new ArrayList<>(other.partitionScores); this.scheduledEvents = new ArrayList<>(other.scheduledEvents); } @@ -143,7 +150,10 @@ public Bucket(StreamInput in) throws IOException { if (in.getVersion().before(Version.V_5_5_0)) { in.readGenericValue(); } - partitionScores = in.readList(PartitionScore::new); + // bwc for perPartitionNormalization + if (in.getVersion().before(Version.V_6_5_0)) { + in.readList(Bucket::readOldPerPartitionNormalization); + } if (in.getVersion().onOrAfter(Version.V_6_2_0)) { scheduledEvents = in.readList(StreamInput::readString); if (scheduledEvents.isEmpty()) { @@ -174,7 +184,10 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().before(Version.V_5_5_0)) { out.writeGenericValue(Collections.emptyMap()); } - out.writeList(partitionScores); + // bwc for perPartitionNormalization + if (out.getVersion().before(Version.V_6_5_0)) { + out.writeList(Collections.emptyList()); + } if (out.getVersion().onOrAfter(Version.V_6_2_0)) { out.writeStringList(scheduledEvents); } @@ -195,9 +208,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Result.IS_INTERIM.getPreferredName(), isInterim); builder.field(BUCKET_INFLUENCERS.getPreferredName(), bucketInfluencers); builder.field(PROCESSING_TIME_MS.getPreferredName(), processingTimeMs); - if (partitionScores.isEmpty() == false) { - builder.field(PARTITION_SCORES.getPreferredName(), partitionScores); - } + if (scheduledEvents.isEmpty() == false) { builder.field(SCHEDULED_EVENTS.getPreferredName(), scheduledEvents); } @@ -304,14 +315,6 @@ public void addBucketInfluencer(BucketInfluencer bucketInfluencer) { bucketInfluencers.add(bucketInfluencer); } - public List getPartitionScores() { - return partitionScores; - } - - public void setPartitionScores(List scores) { - partitionScores = Objects.requireNonNull(scores); - } - public List getScheduledEvents() { return scheduledEvents; } @@ -320,24 +323,10 @@ public void setScheduledEvents(List scheduledEvents) { this.scheduledEvents = ExceptionsHelper.requireNonNull(scheduledEvents, SCHEDULED_EVENTS.getPreferredName()); } - public double partitionInitialAnomalyScore(String partitionValue) { - Optional first = partitionScores.stream().filter(s -> partitionValue.equals(s.getPartitionFieldValue())) - .findFirst(); - - return first.isPresent() ? first.get().getInitialRecordScore() : 0.0; - } - - public double partitionAnomalyScore(String partitionValue) { - Optional first = partitionScores.stream().filter(s -> partitionValue.equals(s.getPartitionFieldValue())) - .findFirst(); - - return first.isPresent() ? first.get().getRecordScore() : 0.0; - } - @Override public int hashCode() { return Objects.hash(jobId, timestamp, eventCount, initialAnomalyScore, anomalyScore, records, - isInterim, bucketSpan, bucketInfluencers, partitionScores, processingTimeMs, scheduledEvents); + isInterim, bucketSpan, bucketInfluencers, processingTimeMs, scheduledEvents); } /** @@ -360,7 +349,6 @@ public boolean equals(Object other) { && (this.anomalyScore == that.anomalyScore) && (this.initialAnomalyScore == that.initialAnomalyScore) && Objects.equals(this.records, that.records) && Objects.equals(this.isInterim, that.isInterim) && Objects.equals(this.bucketInfluencers, that.bucketInfluencers) - && Objects.equals(this.partitionScores, that.partitionScores) && (this.processingTimeMs == that.processingTimeMs) && Objects.equals(this.scheduledEvents, that.scheduledEvents); } @@ -374,6 +362,6 @@ public boolean equals(Object other) { * @return true if the bucket should be normalized or false otherwise */ public boolean isNormalizable() { - return anomalyScore > 0.0 || partitionScores.stream().anyMatch(s -> s.getRecordScore() > 0); + return anomalyScore > 0.0; } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/PartitionScore.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/PartitionScore.java deleted file mode 100644 index 3d0acc8fde667..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/PartitionScore.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.core.ml.job.results; - -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.ToXContentObject; -import org.elasticsearch.common.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.Objects; - -public class PartitionScore implements ToXContentObject, Writeable { - public static final ParseField PARTITION_SCORE = new ParseField("partition_score"); - - private final String partitionFieldValue; - private final String partitionFieldName; - private final double initialRecordScore; - private double recordScore; - private double probability; - - public static final ConstructingObjectParser STRICT_PARSER = createParser(false); - public static final ConstructingObjectParser LENIENT_PARSER = createParser(true); - - private static ConstructingObjectParser createParser(boolean ignoreUnknownFields) { - ConstructingObjectParser parser = new ConstructingObjectParser<>(PARTITION_SCORE.getPreferredName(), - ignoreUnknownFields, a -> new PartitionScore((String) a[0], (String) a[1], (Double) a[2], (Double) a[3], (Double) a[4])); - - parser.declareString(ConstructingObjectParser.constructorArg(), AnomalyRecord.PARTITION_FIELD_NAME); - parser.declareString(ConstructingObjectParser.constructorArg(), AnomalyRecord.PARTITION_FIELD_VALUE); - parser.declareDouble(ConstructingObjectParser.constructorArg(), AnomalyRecord.INITIAL_RECORD_SCORE); - parser.declareDouble(ConstructingObjectParser.constructorArg(), AnomalyRecord.RECORD_SCORE); - parser.declareDouble(ConstructingObjectParser.constructorArg(), AnomalyRecord.PROBABILITY); - - return parser; - } - - public PartitionScore(String fieldName, String fieldValue, double initialRecordScore, double recordScore, double probability) { - partitionFieldName = fieldName; - partitionFieldValue = fieldValue; - this.initialRecordScore = initialRecordScore; - this.recordScore = recordScore; - this.probability = probability; - } - - public PartitionScore(StreamInput in) throws IOException { - partitionFieldName = in.readString(); - partitionFieldValue = in.readString(); - initialRecordScore = in.readDouble(); - recordScore = in.readDouble(); - probability = in.readDouble(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(partitionFieldName); - out.writeString(partitionFieldValue); - out.writeDouble(initialRecordScore); - out.writeDouble(recordScore); - out.writeDouble(probability); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field(AnomalyRecord.PARTITION_FIELD_NAME.getPreferredName(), partitionFieldName); - builder.field(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName(), partitionFieldValue); - builder.field(AnomalyRecord.INITIAL_RECORD_SCORE.getPreferredName(), initialRecordScore); - builder.field(AnomalyRecord.RECORD_SCORE.getPreferredName(), recordScore); - builder.field(AnomalyRecord.PROBABILITY.getPreferredName(), probability); - builder.endObject(); - return builder; - } - - public double getInitialRecordScore() { - return initialRecordScore; - } - - public double getRecordScore() { - return recordScore; - } - - public void setRecordScore(double recordScore) { - this.recordScore = recordScore; - } - - public String getPartitionFieldName() { - return partitionFieldName; - } - - public String getPartitionFieldValue() { - return partitionFieldValue; - } - - public double getProbability() { - return probability; - } - - public void setProbability(double probability) { - this.probability = probability; - } - - @Override - public int hashCode() { - return Objects.hash(partitionFieldName, partitionFieldValue, probability, initialRecordScore, recordScore); - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - - if (other instanceof PartitionScore == false) { - return false; - } - - PartitionScore that = (PartitionScore) other; - - // id is excluded from the test as it is generated by the datastore - return Objects.equals(this.partitionFieldValue, that.partitionFieldValue) - && Objects.equals(this.partitionFieldName, that.partitionFieldName) && (this.probability == that.probability) - && (this.initialRecordScore == that.initialRecordScore) && (this.recordScore == that.recordScore); - } -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java index fb9a697ac4644..63c4278e541d4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java @@ -81,7 +81,6 @@ public final class ReservedFieldNames { Bucket.EVENT_COUNT.getPreferredName(), Bucket.INITIAL_ANOMALY_SCORE.getPreferredName(), Bucket.PROCESSING_TIME_MS.getPreferredName(), - Bucket.PARTITION_SCORES.getPreferredName(), Bucket.SCHEDULED_EVENTS.getPreferredName(), BucketInfluencer.INITIAL_ANOMALY_SCORE.getPreferredName(), BucketInfluencer.ANOMALY_SCORE.getPreferredName(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetBucketActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetBucketActionResponseTests.java index 4fbb7a9249641..ffd21834e583c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetBucketActionResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetBucketActionResponseTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord; import org.elasticsearch.xpack.core.ml.job.results.Bucket; import org.elasticsearch.xpack.core.ml.job.results.BucketInfluencer; -import org.elasticsearch.xpack.core.ml.job.results.PartitionScore; import java.util.ArrayList; import java.util.Collections; @@ -53,15 +52,6 @@ protected Response createTestInstance() { if (randomBoolean()) { bucket.setInterim(randomBoolean()); } - if (randomBoolean()) { - int size = randomInt(10); - List partitionScores = new ArrayList<>(size); - for (int i = 0; i < size; i++) { - partitionScores.add(new PartitionScore(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20), - randomDouble(), randomDouble(), randomDouble())); - } - bucket.setPartitionScores(partitionScores); - } if (randomBoolean()) { bucket.setProcessingTimeMs(randomLong()); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java index 50c60a31427d6..d691124a90a43 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java @@ -97,11 +97,8 @@ public static AnalysisConfig.Builder createRandomized() { builder.setResultFinalizationWindow(randomNonNegativeLong()); } - boolean usePerPartitionNormalisation = randomBoolean(); - builder.setUsePerPartitionNormalization(usePerPartitionNormalisation); - if (!usePerPartitionNormalisation) { // influencers can't be used with per partition normalisation - builder.setInfluencers(Arrays.asList(generateRandomStringArray(10, 10, false))); - } + builder.setInfluencers(Arrays.asList(generateRandomStringArray(10, 10, false))); + return builder; } @@ -690,40 +687,15 @@ public void testVerify_GivenEmptyCategorizationFilter() { assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_EMPTY), e.getMessage()); } - - public void testCheckDetectorsHavePartitionFields() { - AnalysisConfig.Builder config = createValidConfig(); - config.setUsePerPartitionNormalization(true); - - ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, config::build); - - assertEquals(Messages.getMessage(Messages.JOB_CONFIG_PER_PARTITION_NORMALIZATION_REQUIRES_PARTITION_FIELD), e.getMessage()); - } - public void testCheckDetectorsHavePartitionFields_doesntThrowWhenValid() { AnalysisConfig.Builder config = createValidConfig(); Detector.Builder builder = new Detector.Builder(config.build().getDetectors().get(0)); builder.setPartitionFieldName("pField"); config.build().getDetectors().set(0, builder.build()); - config.setUsePerPartitionNormalization(true); config.build(); } - public void testCheckNoInfluencersAreSet() { - - AnalysisConfig.Builder config = createValidConfig(); - Detector.Builder builder = new Detector.Builder(config.build().getDetectors().get(0)); - builder.setPartitionFieldName("pField"); - config.build().getDetectors().set(0, builder.build()); - config.setInfluencers(Arrays.asList("inf1", "inf2")); - config.setUsePerPartitionNormalization(true); - - ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, config::build); - - assertEquals(Messages.getMessage(Messages.JOB_CONFIG_PER_PARTITION_NORMALIZATION_CANNOT_USE_INFLUENCERS), e.getMessage()); - } - public void testVerify_GivenCategorizationFiltersContainInvalidRegex() { AnalysisConfig.Builder config = createValidCategorizationConfig(); config.setCategorizationFilters(Arrays.asList("foo", "(")); @@ -756,7 +728,7 @@ private static AnalysisConfig.Builder createValidCategorizationConfig() { @Override protected AnalysisConfig mutateInstance(AnalysisConfig instance) { AnalysisConfig.Builder builder = new AnalysisConfig.Builder(instance); - switch (between(0, 11)) { + switch (between(0, 10)) { case 0: List detectors = new ArrayList<>(instance.getDetectors()); Detector.Builder detector = new Detector.Builder(); @@ -832,7 +804,6 @@ protected AnalysisConfig mutateInstance(AnalysisConfig instance) { List influencers = new ArrayList<>(instance.getInfluencers()); influencers.add(randomAlphaOfLengthBetween(5, 10)); builder.setInfluencers(influencers); - builder.setUsePerPartitionNormalization(false); break; case 8: if (instance.getOverlappingBuckets() == null) { @@ -855,13 +826,6 @@ protected AnalysisConfig mutateInstance(AnalysisConfig instance) { builder.setMultivariateByFields(instance.getMultivariateByFields() == false); } break; - case 11: - boolean usePerPartitionNormalization = instance.getUsePerPartitionNormalization() == false; - builder.setUsePerPartitionNormalization(usePerPartitionNormalization); - if (usePerPartitionNormalization) { - builder.setInfluencers(Collections.emptyList()); - } - break; default: throw new AssertionError("Illegal randomisation branch"); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 910d9664240b2..e5847288c3708 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -389,8 +389,8 @@ public Collection createComponents(Client client, ClusterService cluster autodetectProcessFactory = (job, autodetectParams, executorService, onProcessCrash) -> new BlackHoleAutodetectProcess(job.getId()); // factor of 1.0 makes renormalization a no-op - normalizerProcessFactory = (jobId, quantilesState, bucketSpan, perPartitionNormalization, - executorService) -> new MultiplyingNormalizerProcess(settings, 1.0); + normalizerProcessFactory = (jobId, quantilesState, bucketSpan, executorService) -> + new MultiplyingNormalizerProcess(settings, 1.0); } NormalizerFactory normalizerFactory = new NormalizerFactory(normalizerProcessFactory, threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME)); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java index 200cb08512572..0094eba97cecb 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java @@ -61,7 +61,6 @@ public class AutodetectBuilder { public static final String LENGTH_ENCODED_INPUT_ARG = "--lengthEncodedInput"; public static final String MODEL_CONFIG_ARG = "--modelconfig="; public static final String QUANTILES_STATE_PATH_ARG = "--quantilesState="; - public static final String PER_PARTITION_NORMALIZATION = "--perPartitionNormalization"; private static final String CONF_EXTENSION = ".conf"; static final String JOB_ID_ARG = "--jobid="; @@ -207,10 +206,6 @@ List buildAutodetectCommand() { if (Boolean.TRUE.equals(analysisConfig.getMultivariateByFields())) { command.add(MULTIVARIATE_BY_FIELDS_ARG); } - - if (analysisConfig.getUsePerPartitionNormalization()) { - command.add(PER_PARTITION_NORMALIZATION); - } } // Input is always length encoded diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java index 063ab3b49d146..fa05c2e63ee11 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java @@ -499,7 +499,7 @@ AutodetectCommunicator create(JobTask jobTask, AutodetectParams autodetectParams new JobRenormalizedResultsPersister(job.getId(), settings, client), normalizerFactory); ExecutorService renormalizerExecutorService = threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME); Renormalizer renormalizer = new ShortCircuitingRenormalizer(jobId, scoresUpdater, - renormalizerExecutorService, job.getAnalysisConfig().getUsePerPartitionNormalization()); + renormalizerExecutorService); AutodetectProcess process = autodetectProcessFactory.createAutodetectProcess(job, autodetectParams, autoDetectExecutorService, onProcessCrash(jobTask)); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizable.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizable.java index 7ef23cb513b7f..9b54d01de5fa6 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizable.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizable.java @@ -16,12 +16,10 @@ import java.util.stream.Collectors; import static org.elasticsearch.xpack.ml.job.process.normalizer.Normalizable.ChildType.BUCKET_INFLUENCER; -import static org.elasticsearch.xpack.ml.job.process.normalizer.Normalizable.ChildType.PARTITION_SCORE; - public class BucketNormalizable extends Normalizable { - private static final List CHILD_TYPES = Arrays.asList(BUCKET_INFLUENCER, PARTITION_SCORE); + private static final List CHILD_TYPES = Arrays.asList(BUCKET_INFLUENCER); private final Bucket bucket; @@ -117,11 +115,6 @@ public List getChildren(ChildType type) { .map(bi -> new BucketInfluencerNormalizable(bi, getOriginatingIndex())) .collect(Collectors.toList())); break; - case PARTITION_SCORE: - children.addAll(bucket.getPartitionScores().stream() - .map(ps -> new PartitionScoreNormalizable(ps, getOriginatingIndex())) - .collect(Collectors.toList())); - break; default: throw new IllegalArgumentException("Invalid type: " + type); } @@ -135,8 +128,6 @@ public boolean setMaxChildrenScore(ChildType childrenType, double maxScore) { double oldScore = bucket.getAnomalyScore(); bucket.setAnomalyScore(maxScore); return maxScore != oldScore; - case PARTITION_SCORE: - return false; default: throw new IllegalArgumentException("Invalid type: " + childrenType); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcessFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcessFactory.java index c96a3b48fe1d8..60f52d3f44288 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcessFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcessFactory.java @@ -38,20 +38,19 @@ public NativeNormalizerProcessFactory(Environment env, Settings settings, Native @Override public NormalizerProcess createNormalizerProcess(String jobId, String quantilesState, Integer bucketSpan, - boolean perPartitionNormalization, ExecutorService executorService) { + ExecutorService executorService) { ProcessPipes processPipes = new ProcessPipes(env, NAMED_PIPE_HELPER, NormalizerBuilder.NORMALIZE, jobId, true, false, true, true, false, false); - createNativeProcess(jobId, quantilesState, processPipes, bucketSpan, perPartitionNormalization); + createNativeProcess(jobId, quantilesState, processPipes, bucketSpan); return new NativeNormalizerProcess(jobId, settings, processPipes.getLogStream().get(), processPipes.getProcessInStream().get(), processPipes.getProcessOutStream().get(), executorService); } - private void createNativeProcess(String jobId, String quantilesState, ProcessPipes processPipes, Integer bucketSpan, - boolean perPartitionNormalization) { + private void createNativeProcess(String jobId, String quantilesState, ProcessPipes processPipes, Integer bucketSpan) { try { - List command = new NormalizerBuilder(env, jobId, quantilesState, bucketSpan, perPartitionNormalization).build(); + List command = new NormalizerBuilder(env, jobId, quantilesState, bucketSpan).build(); processPipes.addArgs(command); nativeController.startProcess(command); processPipes.connectStreams(PROCESS_STARTUP_TIMEOUT); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizable.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizable.java index 7efadf2961308..0d88372de17e4 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizable.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizable.java @@ -11,7 +11,7 @@ import java.util.Objects; public abstract class Normalizable implements ToXContentObject { - public enum ChildType {BUCKET_INFLUENCER, RECORD, PARTITION_SCORE}; + public enum ChildType {BUCKET_INFLUENCER, RECORD}; private final String indexName; private boolean hadBigNormalizedUpdate; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizer.java index 2d4e2135478f3..22e7d3ba99598 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizer.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizer.java @@ -46,15 +46,14 @@ public Normalizer(String jobId, NormalizerProcessFactory processFactory, Executo * and normalizes the given results. * * @param bucketSpan If null the default is used - * @param perPartitionNormalization Is normalization per partition (rather than per job)? * @param results Will be updated with the normalized results * @param quantilesState The state to be used to seed the system change * normalizer */ - public void normalize(Integer bucketSpan, boolean perPartitionNormalization, + public void normalize(Integer bucketSpan, List results, String quantilesState) { NormalizerProcess process = processFactory.createNormalizerProcess(jobId, quantilesState, bucketSpan, - perPartitionNormalization, executorService); + executorService); NormalizerResultHandler resultsHandler = process.createNormalizedResultsHandler(); Future resultsHandlerFuture = executorService.submit(() -> { try { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilder.java index 5630a75127506..37ffd92a3d72f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilder.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilder.java @@ -29,15 +29,12 @@ public class NormalizerBuilder { private final String jobId; private final String quantilesState; private final Integer bucketSpan; - private final boolean perPartitionNormalization; - public NormalizerBuilder(Environment env, String jobId, String quantilesState, Integer bucketSpan, - boolean perPartitionNormalization) { + public NormalizerBuilder(Environment env, String jobId, String quantilesState, Integer bucketSpan) { this.env = env; this.jobId = jobId; this.quantilesState = quantilesState; this.bucketSpan = bucketSpan; - this.perPartitionNormalization = perPartitionNormalization; } /** @@ -49,9 +46,6 @@ public List build() throws IOException { command.add(NORMALIZE_PATH); addIfNotNull(bucketSpan, AutodetectBuilder.BUCKET_SPAN_ARG, command); command.add(AutodetectBuilder.LENGTH_ENCODED_INPUT_ARG); - if (perPartitionNormalization) { - command.add(AutodetectBuilder.PER_PARTITION_NORMALIZATION); - } if (quantilesState != null) { Path quantilesStateFilePath = AutodetectBuilder.writeNormalizerInitState(jobId, quantilesState, env); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerProcessFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerProcessFactory.java index bdb63b778971e..cf08190a99392 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerProcessFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerProcessFactory.java @@ -17,6 +17,5 @@ public interface NormalizerProcessFactory { * @param executorService Executor service used to start the async tasks a job needs to operate the analytical process * @return The process */ - NormalizerProcess createNormalizerProcess(String jobId, String quantilesState, Integer bucketSpan, boolean perPartitionNormalization, - ExecutorService executorService); + NormalizerProcess createNormalizerProcess(String jobId, String quantilesState, Integer bucketSpan, ExecutorService executorService); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/PartitionScoreNormalizable.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/PartitionScoreNormalizable.java deleted file mode 100644 index 91b2a7a505e35..0000000000000 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/PartitionScoreNormalizable.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.ml.job.process.normalizer; - -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.xpack.core.ml.job.results.PartitionScore; - -import java.io.IOException; -import java.util.Objects; - - -public class PartitionScoreNormalizable extends AbstractLeafNormalizable { - private final PartitionScore score; - - public PartitionScoreNormalizable(PartitionScore score, String indexName) { - super(indexName); - this.score = Objects.requireNonNull(score); - } - - @Override - public String getId() { - throw new UnsupportedOperationException("PartitionScore has no ID as it should not be persisted outside of the owning bucket"); - } - - @Override - public Level getLevel() { - return Level.PARTITION; - } - - @Override - public String getPartitionFieldName() { - return score.getPartitionFieldName(); - } - - @Override - public String getPartitionFieldValue() { - return score.getPartitionFieldValue(); - } - - @Override - public String getPersonFieldName() { - return null; - } - - @Override - public String getPersonFieldValue() { - return null; - } - - @Override - public String getFunctionName() { - return null; - } - - @Override - public String getValueFieldName() { - return null; - } - - @Override - public double getProbability() { - return score.getProbability(); - } - - @Override - public double getNormalizedScore() { - return score.getRecordScore(); - } - - @Override - public void setNormalizedScore(double normalizedScore) { - score.setRecordScore(normalizedScore); - } - - @Override - public void setParentScore(double parentScore) { - // Do nothing as it is not holding the parent score. - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return score.toXContent(builder, params); - } -} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdater.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdater.java index cfb5660c911b5..c2ef2fab7f8a4 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdater.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdater.java @@ -79,12 +79,12 @@ private long getNormalizationWindowOrDefault(Job job) { * Update the anomaly score field on all previously persisted buckets * and all contained records */ - public void update(String quantilesState, long endBucketEpochMs, long windowExtensionMs, boolean perPartitionNormalization) { + public void update(String quantilesState, long endBucketEpochMs, long windowExtensionMs) { Normalizer normalizer = normalizerFactory.create(jobId); int[] counts = {0, 0}; - updateBuckets(normalizer, quantilesState, endBucketEpochMs, windowExtensionMs, counts, perPartitionNormalization); - updateRecords(normalizer, quantilesState, endBucketEpochMs, windowExtensionMs, counts, perPartitionNormalization); - updateInfluencers(normalizer, quantilesState, endBucketEpochMs, windowExtensionMs, counts, perPartitionNormalization); + updateBuckets(normalizer, quantilesState, endBucketEpochMs, windowExtensionMs, counts); + updateRecords(normalizer, quantilesState, endBucketEpochMs, windowExtensionMs, counts); + updateInfluencers(normalizer, quantilesState, endBucketEpochMs, windowExtensionMs, counts); // The updates will have been persisted in batches throughout the renormalization // process - this call just catches any leftovers @@ -94,7 +94,7 @@ public void update(String quantilesState, long endBucketEpochMs, long windowExte } private void updateBuckets(Normalizer normalizer, String quantilesState, long endBucketEpochMs, - long windowExtensionMs, int[] counts, boolean perPartitionNormalization) { + long windowExtensionMs, int[] counts) { BatchedDocumentsIterator> bucketsIterator = jobResultsProvider.newBatchedBucketsIterator(jobId) .timeRange(calcNormalizationWindowStart(endBucketEpochMs, windowExtensionMs), endBucketEpochMs) @@ -114,14 +114,14 @@ private void updateBuckets(Normalizer normalizer, String quantilesState, long en if (current.result.isNormalizable()) { bucketsToRenormalize.add(new BucketNormalizable(current.result, current.index)); if (bucketsToRenormalize.size() >= TARGET_BUCKETS_TO_RENORMALIZE) { - normalizeBuckets(normalizer, bucketsToRenormalize, quantilesState, counts, perPartitionNormalization); + normalizeBuckets(normalizer, bucketsToRenormalize, quantilesState, counts); bucketsToRenormalize.clear(); } } } } if (!bucketsToRenormalize.isEmpty()) { - normalizeBuckets(normalizer, bucketsToRenormalize, quantilesState, counts, perPartitionNormalization); + normalizeBuckets(normalizer, bucketsToRenormalize, quantilesState, counts); } } @@ -130,8 +130,8 @@ private long calcNormalizationWindowStart(long endEpochMs, long windowExtensionM } private void normalizeBuckets(Normalizer normalizer, List normalizableBuckets, - String quantilesState, int[] counts, boolean perPartitionNormalization) { - normalizer.normalize(bucketSpan, perPartitionNormalization, normalizableBuckets, quantilesState); + String quantilesState, int[] counts) { + normalizer.normalize(bucketSpan, normalizableBuckets, quantilesState); for (BucketNormalizable bucketNormalizable : normalizableBuckets) { if (bucketNormalizable.hadBigNormalizedUpdate()) { @@ -144,7 +144,7 @@ private void normalizeBuckets(Normalizer normalizer, List no } private void updateRecords(Normalizer normalizer, String quantilesState, long endBucketEpochMs, - long windowExtensionMs, int[] counts, boolean perPartitionNormalization) { + long windowExtensionMs, int[] counts) { BatchedDocumentsIterator> recordsIterator = jobResultsProvider.newBatchedRecordsIterator(jobId) .timeRange(calcNormalizationWindowStart(endBucketEpochMs, windowExtensionMs), endBucketEpochMs) .includeInterim(false); @@ -160,14 +160,14 @@ private void updateRecords(Normalizer normalizer, String quantilesState, long en List asNormalizables = records.stream() .map(recordResultIndex -> new RecordNormalizable(recordResultIndex.result, recordResultIndex.index)) .collect(Collectors.toList()); - normalizer.normalize(bucketSpan, perPartitionNormalization, asNormalizables, quantilesState); + normalizer.normalize(bucketSpan, asNormalizables, quantilesState); persistChanged(counts, asNormalizables); } } private void updateInfluencers(Normalizer normalizer, String quantilesState, long endBucketEpochMs, - long windowExtensionMs, int[] counts, boolean perPartitionNormalization) { + long windowExtensionMs, int[] counts) { BatchedDocumentsIterator> influencersIterator = jobResultsProvider.newBatchedInfluencersIterator(jobId) .timeRange(calcNormalizationWindowStart(endBucketEpochMs, windowExtensionMs), endBucketEpochMs) .includeInterim(false); @@ -183,7 +183,7 @@ private void updateInfluencers(Normalizer normalizer, String quantilesState, lon List asNormalizables = influencers.stream() .map(influencerResultIndex -> new InfluencerNormalizable(influencerResultIndex.result, influencerResultIndex.index)) .collect(Collectors.toList()); - normalizer.normalize(bucketSpan, perPartitionNormalization, asNormalizables, quantilesState); + normalizer.normalize(bucketSpan, asNormalizables, quantilesState); persistChanged(counts, asNormalizables); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizer.java index 057ed18cd696f..7db66387db8e0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizer.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizer.java @@ -26,7 +26,6 @@ public class ShortCircuitingRenormalizer implements Renormalizer { private final String jobId; private final ScoresUpdater scoresUpdater; private final ExecutorService executorService; - private final boolean isPerPartitionNormalization; private final Deque quantilesDeque = new ConcurrentLinkedDeque<>(); private final Deque latchDeque = new ConcurrentLinkedDeque<>(); /** @@ -34,12 +33,10 @@ public class ShortCircuitingRenormalizer implements Renormalizer { */ private final Semaphore semaphore = new Semaphore(1); - public ShortCircuitingRenormalizer(String jobId, ScoresUpdater scoresUpdater, ExecutorService executorService, - boolean isPerPartitionNormalization) { + public ShortCircuitingRenormalizer(String jobId, ScoresUpdater scoresUpdater, ExecutorService executorService) { this.jobId = jobId; this.scoresUpdater = scoresUpdater; this.executorService = executorService; - this.isPerPartitionNormalization = isPerPartitionNormalization; } @Override @@ -161,8 +158,7 @@ private void doRenormalizations() { jobId, latestBucketTimeMs, earliestBucketTimeMs); windowExtensionMs = 0; } - scoresUpdater.update(latestQuantiles.getQuantileState(), latestBucketTimeMs, windowExtensionMs, - isPerPartitionNormalization); + scoresUpdater.update(latestQuantiles.getQuantileState(), latestBucketTimeMs, windowExtensionMs); latch.countDown(); latch = null; } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilderTests.java index 0f83106441185..325ad52864bfa 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilderTests.java @@ -56,7 +56,6 @@ public void testBuildAutodetectCommand() { acBuilder.setSummaryCountFieldName("summaryField"); acBuilder.setOverlappingBuckets(true); acBuilder.setMultivariateByFields(true); - acBuilder.setUsePerPartitionNormalization(true); job.setAnalysisConfig(acBuilder); DataDescription.Builder dd = new DataDescription.Builder(); @@ -66,7 +65,7 @@ public void testBuildAutodetectCommand() { job.setDataDescription(dd); List command = autodetectBuilder(job.build()).buildAutodetectCommand(); - assertEquals(13, command.size()); + assertEquals(12, command.size()); assertTrue(command.contains(AutodetectBuilder.AUTODETECT_PATH)); assertTrue(command.contains(AutodetectBuilder.BUCKET_SPAN_ARG + "120")); assertTrue(command.contains(AutodetectBuilder.LATENCY_ARG + "360")); @@ -80,8 +79,6 @@ public void testBuildAutodetectCommand() { assertTrue(command.contains(AutodetectBuilder.TIME_FIELD_ARG + "tf")); assertTrue(command.contains(AutodetectBuilder.JOB_ID_ARG + "unit-test-job")); - assertTrue(command.contains(AutodetectBuilder.PER_PARTITION_NORMALIZATION)); - int expectedPersistInterval = 10800 + AutodetectBuilder.calculateStaggeringInterval(job.getId()); assertTrue(command.contains(AutodetectBuilder.PERSIST_INTERVAL_ARG + expectedPersistInterval)); int expectedMaxQuantileInterval = 21600 + AutodetectBuilder.calculateStaggeringInterval(job.getId()); @@ -116,4 +113,4 @@ public void testBuildAutodetectCommand_givenPersistModelState() { private AutodetectBuilder autodetectBuilder(Job job) { return new AutodetectBuilder(job, filesToDelete, logger, env, settings, nativeController, processPipes); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java index 0b68eadfd6587..a3e772b3a9d83 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java @@ -171,25 +171,6 @@ public void testProcessResult_records() { verifyNoMoreInteractions(persister); } - public void testProcessResult_records_isPerPartitionNormalization() { - JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class); - when(persister.bulkPersisterBuilder(JOB_ID)).thenReturn(bulkBuilder); - - AutoDetectResultProcessor.Context context = new AutoDetectResultProcessor.Context("foo", bulkBuilder); - context.deleteInterimRequired = false; - AutodetectResult result = mock(AutodetectResult.class); - AnomalyRecord record1 = new AnomalyRecord("foo", new Date(123), 123); - record1.setPartitionFieldValue("pValue"); - AnomalyRecord record2 = new AnomalyRecord("foo", new Date(123), 123); - record2.setPartitionFieldValue("pValue"); - List records = Arrays.asList(record1, record2); - when(result.getRecords()).thenReturn(records); - processorUnderTest.processResult(context, result); - - verify(bulkBuilder, times(1)).persistRecords(records); - verify(bulkBuilder, never()).executeRequest(); - verifyNoMoreInteractions(persister); - } public void testProcessResult_influencers() { JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizableTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizableTests.java index 4436fcc7026fe..f2cd6572db69d 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizableTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizableTests.java @@ -9,10 +9,8 @@ import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord; import org.elasticsearch.xpack.core.ml.job.results.Bucket; import org.elasticsearch.xpack.core.ml.job.results.BucketInfluencer; -import org.elasticsearch.xpack.core.ml.job.results.PartitionScore; import org.junit.Before; -import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.List; @@ -46,11 +44,6 @@ public void setUpBucket() { AnomalyRecord record2 = new AnomalyRecord("foo", bucket.getTimestamp(), 600); record2.setRecordScore(2.0); bucket.setRecords(Arrays.asList(record1, record2)); - - List partitionScores = new ArrayList<>(); - partitionScores.add(new PartitionScore("pf1", "pv1", 0.3, 0.2, 0.1)); - partitionScores.add(new PartitionScore("pf1", "pv2", 0.5, 0.4, 0.01)); - bucket.setPartitionScores(partitionScores); } public void testIsContainerOnly() { @@ -106,15 +99,11 @@ public void testGetChildren() { BucketNormalizable bn = new BucketNormalizable(bucket, INDEX_NAME); List children = bn.getChildren(); - assertEquals(4, children.size()); + assertEquals(2, children.size()); assertTrue(children.get(0) instanceof BucketInfluencerNormalizable); assertEquals(42.0, children.get(0).getNormalizedScore(), EPSILON); assertTrue(children.get(1) instanceof BucketInfluencerNormalizable); assertEquals(88.0, children.get(1).getNormalizedScore(), EPSILON); - assertTrue(children.get(2) instanceof PartitionScoreNormalizable); - assertEquals(0.2, children.get(2).getNormalizedScore(), EPSILON); - assertTrue(children.get(3) instanceof PartitionScoreNormalizable); - assertEquals(0.4, children.get(3).getNormalizedScore(), EPSILON); } public void testGetChildren_GivenTypeBucketInfluencer() { @@ -132,7 +121,6 @@ public void testSetMaxChildrenScore_GivenDifferentScores() { BucketNormalizable bucketNormalizable = new BucketNormalizable(bucket, INDEX_NAME); assertTrue(bucketNormalizable.setMaxChildrenScore(Normalizable.ChildType.BUCKET_INFLUENCER, 95.0)); - assertFalse(bucketNormalizable.setMaxChildrenScore(Normalizable.ChildType.PARTITION_SCORE, 42.0)); assertEquals(95.0, bucket.getAnomalyScore(), EPSILON); } @@ -141,7 +129,6 @@ public void testSetMaxChildrenScore_GivenSameScores() { BucketNormalizable bucketNormalizable = new BucketNormalizable(bucket, INDEX_NAME); assertFalse(bucketNormalizable.setMaxChildrenScore(Normalizable.ChildType.BUCKET_INFLUENCER, 88.0)); - assertFalse(bucketNormalizable.setMaxChildrenScore(Normalizable.ChildType.PARTITION_SCORE, 2.0)); assertEquals(88.0, bucket.getAnomalyScore(), EPSILON); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilderTests.java index 64e595fd5a043..5d9c183c738e4 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilderTests.java @@ -21,11 +21,10 @@ public void testBuildNormalizerCommand() throws IOException { Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build()); String jobId = "unit-test-job"; - List command = new NormalizerBuilder(env, jobId, null, 300, true).build(); - assertEquals(4, command.size()); + List command = new NormalizerBuilder(env, jobId, null, 300).build(); + assertEquals(3, command.size()); assertTrue(command.contains("./normalize")); assertTrue(command.contains(AutodetectBuilder.BUCKET_SPAN_ARG + "300")); assertTrue(command.contains(AutodetectBuilder.LENGTH_ENCODED_INPUT_ARG)); - assertTrue(command.contains(AutodetectBuilder.PER_PARTITION_NORMALIZATION)); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerTests.java index 1b34226e33640..d06146ad53fdb 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerTests.java @@ -49,7 +49,7 @@ public void testNormalize() throws IOException, InterruptedException { ExecutorService threadpool = Executors.newScheduledThreadPool(1); try { NormalizerProcessFactory processFactory = mock(NormalizerProcessFactory.class); - when(processFactory.createNormalizerProcess(eq(JOB_ID), eq(QUANTILES_STATE), eq(BUCKET_SPAN), eq(false), + when(processFactory.createNormalizerProcess(eq(JOB_ID), eq(QUANTILES_STATE), eq(BUCKET_SPAN), any())).thenReturn(new MultiplyingNormalizerProcess(Settings.EMPTY, FACTOR)); Normalizer normalizer = new Normalizer(JOB_ID, processFactory, threadpool); @@ -58,7 +58,7 @@ public void testNormalize() throws IOException, InterruptedException { bucket.addBucketInfluencer(createTimeBucketInfluencer(bucket.getTimestamp(), 0.07, INITIAL_SCORE)); List asNormalizables = Arrays.asList(new BucketNormalizable(bucket, INDEX_NAME)); - normalizer.normalize(BUCKET_SPAN, false, asNormalizables, QUANTILES_STATE); + normalizer.normalize(BUCKET_SPAN, asNormalizables, QUANTILES_STATE); assertEquals(1, asNormalizables.size()); assertEquals(FACTOR * INITIAL_SCORE, asNormalizables.get(0).getNormalizedScore(), 0.0001); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdaterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdaterTests.java index 8adab79c16fe1..7d4dfd408a400 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdaterTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdaterTests.java @@ -33,7 +33,6 @@ import java.util.List; import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.anyList; import static org.mockito.Matchers.anyListOf; @@ -96,7 +95,7 @@ public void testUpdate_GivenBucketWithZeroScoreAndNoRecords() throws IOException buckets.add(bucket); givenProviderReturnsBuckets(buckets); - scoresUpdater.update(QUANTILES_STATE, 3600, 0, false); + scoresUpdater.update(QUANTILES_STATE, 3600, 0); verifyNormalizerWasInvoked(0); verifyNothingWasUpdated(); @@ -114,7 +113,7 @@ public void testUpdate_GivenTwoBucketsOnlyOneUpdated() throws IOException { givenProviderReturnsBuckets(buckets); givenProviderReturnsRecords(new ArrayDeque<>()); - scoresUpdater.update(QUANTILES_STATE, 3600, 0, false); + scoresUpdater.update(QUANTILES_STATE, 3600, 0); verifyNormalizerWasInvoked(1); verify(jobRenormalizedResultsPersister, times(1)).updateBucket(any()); @@ -130,7 +129,7 @@ public void testUpdate_GivenSingleBucketWithAnomalyScoreAndNoRecords() throws IO givenProviderReturnsBuckets(buckets); givenProviderReturnsRecords(new ArrayDeque<>()); - scoresUpdater.update(QUANTILES_STATE, 3600, 0, false); + scoresUpdater.update(QUANTILES_STATE, 3600, 0); verifyNormalizerWasInvoked(1); verifyBucketWasUpdated(1); @@ -151,7 +150,7 @@ public void testUpdate_GivenSingleBucketAndRecords() throws IOException { givenProviderReturnsBuckets(buckets); givenProviderReturnsRecords(records); - scoresUpdater.update(QUANTILES_STATE, 3600, 0, false); + scoresUpdater.update(QUANTILES_STATE, 3600, 0); verifyNormalizerWasInvoked(2); verify(jobRenormalizedResultsPersister, times(1)).updateBucket(any()); @@ -177,7 +176,7 @@ public void testUpdate_GivenEnoughBucketsForTwoBatchesButOneNormalization() thro givenProviderReturnsBuckets(batch1, batch2); givenProviderReturnsRecords(new ArrayDeque<>()); - scoresUpdater.update(QUANTILES_STATE, 3600, 0, false); + scoresUpdater.update(QUANTILES_STATE, 3600, 0); verifyNormalizerWasInvoked(1); @@ -213,7 +212,7 @@ public void testUpdate_GivenTwoBucketsWithFirstHavingEnoughRecordsToForceSecondN recordIter.requireIncludeInterim(false); when(jobResultsProvider.newBatchedRecordsIterator(JOB_ID)).thenReturn(recordIter); - scoresUpdater.update(QUANTILES_STATE, 3600, 0, false); + scoresUpdater.update(QUANTILES_STATE, 3600, 0); verifyNormalizerWasInvoked(2); } @@ -225,7 +224,7 @@ public void testUpdate_GivenInfluencerWithBigChange() throws IOException { influencers.add(influencer); givenProviderReturnsInfluencers(influencers); - scoresUpdater.update(QUANTILES_STATE, 3600, 0, false); + scoresUpdater.update(QUANTILES_STATE, 3600, 0); verifyNormalizerWasInvoked(1); verify(jobRenormalizedResultsPersister, times(1)).updateResults(any()); @@ -254,7 +253,7 @@ public void testUpdate_GivenShutdown() throws IOException { givenProviderReturnsRecords(records); scoresUpdater.shutdown(); - scoresUpdater.update(QUANTILES_STATE, 3600, 0, false); + scoresUpdater.update(QUANTILES_STATE, 3600, 0); verifyNormalizerWasInvoked(0); verify(jobRenormalizedResultsPersister, never()).updateBucket(any()); @@ -273,7 +272,7 @@ public void testDefaultRenormalizationWindowBasedOnTime() throws IOException { givenProviderReturnsRecords(new ArrayDeque<>()); givenProviderReturnsNoInfluencers(); - scoresUpdater.update(QUANTILES_STATE, 2595600000L, 0, false); + scoresUpdater.update(QUANTILES_STATE, 2595600000L, 0); verifyNormalizerWasInvoked(1); verifyBucketWasUpdated(1); @@ -290,7 +289,7 @@ public void testManualRenormalizationWindow() throws IOException { givenProviderReturnsRecords(new ArrayDeque<>()); givenProviderReturnsNoInfluencers(); - scoresUpdater.update(QUANTILES_STATE, 90000000L, 0, false); + scoresUpdater.update(QUANTILES_STATE, 90000000L, 0); verifyNormalizerWasInvoked(1); verifyBucketWasUpdated(1); @@ -308,7 +307,7 @@ public void testManualRenormalizationWindow_GivenExtension() throws IOException givenProviderReturnsRecords(new ArrayDeque<>()); givenProviderReturnsNoInfluencers(); - scoresUpdater.update(QUANTILES_STATE, 90000000L, 900000, false); + scoresUpdater.update(QUANTILES_STATE, 90000000L, 900000); verifyNormalizerWasInvoked(1); verifyBucketWasUpdated(1); @@ -340,7 +339,7 @@ private void givenNormalizerRaisesBigChangeFlag() { doAnswer(new Answer() { @Override public Void answer(InvocationOnMock invocationOnMock) throws Throwable { - List normalizables = (List) invocationOnMock.getArguments()[2]; + List normalizables = (List) invocationOnMock.getArguments()[1]; for (Normalizable normalizable : normalizables) { normalizable.raiseBigChangeFlag(); for (Normalizable child : normalizable.getChildren()) { @@ -349,7 +348,7 @@ public Void answer(InvocationOnMock invocationOnMock) throws Throwable { } return null; } - }).when(normalizer).normalize(anyInt(), anyBoolean(), anyList(), anyString()); + }).when(normalizer).normalize(anyInt(), anyList(), anyString()); } private void givenProviderReturnsBuckets(Deque batch1, Deque batch2) { @@ -417,7 +416,7 @@ private void givenProviderReturnsInfluencers(Deque influencers) { private void verifyNormalizerWasInvoked(int times) throws IOException { int bucketSpan = job.getAnalysisConfig() == null ? 0 : ((Long) job.getAnalysisConfig().getBucketSpan().seconds()).intValue(); verify(normalizer, times(times)).normalize( - eq(bucketSpan), eq(false), anyListOf(Normalizable.class), + eq(bucketSpan), anyListOf(Normalizable.class), eq(QUANTILES_STATE)); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizerTests.java index 769d3657b0497..e10d7191acc42 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizerTests.java @@ -18,7 +18,6 @@ import static org.hamcrest.Matchers.is; import static org.mockito.Matchers.anyLong; -import static org.mockito.Matchers.eq; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -43,10 +42,7 @@ public void setUpMocks() { public void testNormalize() throws InterruptedException { ExecutorService threadpool = Executors.newScheduledThreadPool(10); try { - boolean isPerPartitionNormalization = randomBoolean(); - - ShortCircuitingRenormalizer renormalizer = new ShortCircuitingRenormalizer(JOB_ID, scoresUpdater, threadpool, - isPerPartitionNormalization); + ShortCircuitingRenormalizer renormalizer = new ShortCircuitingRenormalizer(JOB_ID, scoresUpdater, threadpool); // Blast through many sets of quantiles in quick succession, faster than the normalizer can process them for (int i = 1; i < TEST_SIZE / 2; ++i) { @@ -61,7 +57,7 @@ public void testNormalize() throws InterruptedException { renormalizer.waitUntilIdle(); ArgumentCaptor stateCaptor = ArgumentCaptor.forClass(String.class); - verify(scoresUpdater, atLeastOnce()).update(stateCaptor.capture(), anyLong(), anyLong(), eq(isPerPartitionNormalization)); + verify(scoresUpdater, atLeastOnce()).update(stateCaptor.capture(), anyLong(), anyLong()); List quantilesUsed = stateCaptor.getAllValues(); assertFalse(quantilesUsed.isEmpty()); @@ -91,7 +87,7 @@ public void testNormalize() throws InterruptedException { public void testIsEnabled_GivenNormalizationWindowIsZero() { ScoresUpdater scoresUpdater = mock(ScoresUpdater.class); when(scoresUpdater.getNormalizationWindow()).thenReturn(0L); - ShortCircuitingRenormalizer renormalizer = new ShortCircuitingRenormalizer(JOB_ID, scoresUpdater, null, randomBoolean()); + ShortCircuitingRenormalizer renormalizer = new ShortCircuitingRenormalizer(JOB_ID, scoresUpdater, null); assertThat(renormalizer.isEnabled(), is(false)); } @@ -99,7 +95,7 @@ public void testIsEnabled_GivenNormalizationWindowIsZero() { public void testIsEnabled_GivenNormalizationWindowGreaterThanZero() { ScoresUpdater scoresUpdater = mock(ScoresUpdater.class); when(scoresUpdater.getNormalizationWindow()).thenReturn(1L); - ShortCircuitingRenormalizer renormalizer = new ShortCircuitingRenormalizer(JOB_ID, scoresUpdater, null, randomBoolean()); + ShortCircuitingRenormalizer renormalizer = new ShortCircuitingRenormalizer(JOB_ID, scoresUpdater, null); assertThat(renormalizer.isEnabled(), is(true)); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/BucketTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/BucketTests.java index 966501db43ff6..65343b0a068ac 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/BucketTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/BucketTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecordTests; import org.elasticsearch.xpack.core.ml.job.results.Bucket; import org.elasticsearch.xpack.core.ml.job.results.BucketInfluencer; -import org.elasticsearch.xpack.core.ml.job.results.PartitionScore; import java.io.IOException; import java.util.ArrayList; @@ -61,15 +60,6 @@ public Bucket createTestInstance(String jobId) { if (randomBoolean()) { bucket.setInterim(randomBoolean()); } - if (randomBoolean()) { - int size = randomInt(10); - List partitionScores = new ArrayList<>(size); - for (int i = 0; i < size; i++) { - partitionScores.add(new PartitionScore(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20), randomDouble(), - randomDouble(), randomDouble())); - } - bucket.setPartitionScores(partitionScores); - } if (randomBoolean()) { bucket.setProcessingTimeMs(randomLong()); } @@ -235,15 +225,6 @@ public void testIsNormalizable_GivenAnomalyScoreIsZeroAndRecordCountIsZero() { assertFalse(bucket.isNormalizable()); } - public void testIsNormalizable_GivenAnomalyScoreIsZeroAndPartitionsScoresAreNonZero() { - Bucket bucket = new Bucket("foo", new Date(123), 123); - bucket.addBucketInfluencer(new BucketInfluencer("foo", new Date(123), 123)); - bucket.setAnomalyScore(0.0); - bucket.setPartitionScores(Collections.singletonList(new PartitionScore("n", "v", 50.0, 40.0, 0.01))); - - assertTrue(bucket.isNormalizable()); - } - public void testIsNormalizable_GivenAnomalyScoreIsNonZeroAndRecordCountIsZero() { Bucket bucket = new Bucket("foo", new Date(123), 123); bucket.addBucketInfluencer(new BucketInfluencer("foo", new Date(123), 123)); @@ -260,35 +241,7 @@ public void testIsNormalizable_GivenAnomalyScoreIsNonZeroAndRecordCountIsNonZero assertTrue(bucket.isNormalizable()); } - public void testPartitionAnomalyScore() { - List pScore = new ArrayList<>(); - pScore.add(new PartitionScore("pf", "pv1", 11.0, 10.0, 0.1)); - pScore.add(new PartitionScore("pf", "pv3", 51.0, 50.0, 0.1)); - pScore.add(new PartitionScore("pf", "pv4", 61.0, 60.0, 0.1)); - pScore.add(new PartitionScore("pf", "pv2", 41.0, 40.0, 0.1)); - - Bucket bucket = new Bucket("foo", new Date(123), 123); - bucket.setPartitionScores(pScore); - - double initialAnomalyScore = bucket.partitionInitialAnomalyScore("pv1"); - assertEquals(11.0, initialAnomalyScore, 0.001); - double anomalyScore = bucket.partitionAnomalyScore("pv1"); - assertEquals(10.0, anomalyScore, 0.001); - initialAnomalyScore = bucket.partitionInitialAnomalyScore("pv2"); - assertEquals(41.0, initialAnomalyScore, 0.001); - anomalyScore = bucket.partitionAnomalyScore("pv2"); - assertEquals(40.0, anomalyScore, 0.001); - initialAnomalyScore = bucket.partitionInitialAnomalyScore("pv3"); - assertEquals(51.0, initialAnomalyScore, 0.001); - anomalyScore = bucket.partitionAnomalyScore("pv3"); - assertEquals(50.0, anomalyScore, 0.001); - initialAnomalyScore = bucket.partitionInitialAnomalyScore("pv4"); - assertEquals(61.0, initialAnomalyScore, 0.001); - anomalyScore = bucket.partitionAnomalyScore("pv4"); - assertEquals(60.0, anomalyScore, 0.001); - } - - public void testId() { + public void testId() { Bucket bucket = new Bucket("foo", new Date(123), 60L); assertEquals("foo_bucket_123_60", bucket.getId()); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/PartitionScoreTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/PartitionScoreTests.java deleted file mode 100644 index 74c3934c532dd..0000000000000 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/PartitionScoreTests.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.ml.job.results; - -import org.elasticsearch.common.io.stream.Writeable.Reader; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.test.AbstractSerializingTestCase; -import org.elasticsearch.xpack.core.ml.job.results.PartitionScore; - -import java.io.IOException; - -import static org.hamcrest.Matchers.containsString; - -public class PartitionScoreTests extends AbstractSerializingTestCase { - - @Override - protected PartitionScore createTestInstance() { - return new PartitionScore(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20), randomDouble(), randomDouble(), - randomDouble()); - } - - @Override - protected Reader instanceReader() { - return PartitionScore::new; - } - - @Override - protected PartitionScore doParseInstance(XContentParser parser) { - return PartitionScore.STRICT_PARSER.apply(parser, null); - } - - public void testStrictParser() throws IOException { - String json = "{\"partition_field_name\":\"field_1\", \"partition_field_value\":\"x\", \"initial_record_score\": 3," + - " \"record_score\": 3, \"probability\": 0.001, \"foo\":\"bar\"}"; - try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> PartitionScore.STRICT_PARSER.apply(parser, null)); - - assertThat(e.getMessage(), containsString("unknown field [foo]")); - } - } - - public void testLenientParser() throws IOException { - String json = "{\"partition_field_name\":\"field_1\", \"partition_field_value\":\"x\", \"initial_record_score\": 3," + - " \"record_score\": 3, \"probability\": 0.001, \"foo\":\"bar\"}"; - try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { - PartitionScore.LENIENT_PARSER.apply(parser, null); - } - } -} From ffd48ce55c5b4d7f139f9e0942b9f039b5c68e5b Mon Sep 17 00:00:00 2001 From: datosh Date: Thu, 16 Aug 2018 11:34:41 +0200 Subject: [PATCH 07/42] [DOCS] Clarify sentence in network-host.asciidoc (#32429) --- docs/reference/setup/important-settings/network-host.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/setup/important-settings/network-host.asciidoc b/docs/reference/setup/important-settings/network-host.asciidoc index 7e29e73123d8d..1788bfebc66b5 100644 --- a/docs/reference/setup/important-settings/network-host.asciidoc +++ b/docs/reference/setup/important-settings/network-host.asciidoc @@ -9,7 +9,7 @@ location on a single node. This can be useful for testing Elasticsearch's ability to form clusters, but it is not a configuration recommended for production. -In order to communicate and to form a cluster with nodes on other servers, your +In order to form a cluster with nodes on other servers, your node will need to bind to a non-loopback address. While there are many <>, usually all you need to configure is `network.host`: From e923e4813f84f126f859f49f11a128d4589e31f5 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 16 Aug 2018 11:53:01 +0200 Subject: [PATCH 08/42] CharArraysTests: Fix test bug. --- .../test/java/org/elasticsearch/common/CharArraysTests.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java b/libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java index 64b1ecd1f8a2d..9283283ab0861 100644 --- a/libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java +++ b/libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java @@ -55,7 +55,10 @@ public void testCharsBeginsWith() { assertTrue(CharArrays.charsBeginsWith(prefix, prefixedValue)); final String modifiedPrefix = randomBoolean() ? prefix.substring(1) : prefix.substring(0, prefix.length() - 1); - final char[] nonMatchingValue = modifiedPrefix.concat(randomAlphaOfLengthBetween(0, 12)).toCharArray(); + char[] nonMatchingValue; + do { + nonMatchingValue = modifiedPrefix.concat(randomAlphaOfLengthBetween(0, 12)).toCharArray(); + } while (new String(nonMatchingValue).startsWith(prefix)); assertFalse(CharArrays.charsBeginsWith(prefix, nonMatchingValue)); assertTrue(CharArrays.charsBeginsWith(modifiedPrefix, nonMatchingValue)); } From 08936d4977f1025ea5ab7ed141ff1d96b70a5192 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Thu, 16 Aug 2018 11:07:20 +0100 Subject: [PATCH 09/42] Mutes test in DuelScrollIT Due to https://github.com/elastic/elasticsearch/issues/32682 --- .../test/java/org/elasticsearch/search/scroll/DuelScrollIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java b/server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java index 31fcfa7155cc0..1ddd11e5d0f7d 100644 --- a/server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java +++ b/server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java @@ -21,6 +21,7 @@ import com.carrotsearch.hppc.IntHashSet; import com.carrotsearch.randomizedtesting.generators.RandomPicks; + import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; @@ -256,6 +257,7 @@ private void testDuelIndexOrder(SearchType searchType, boolean trackScores, int } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32682") public void testDuelIndexOrderQueryThenFetch() throws Exception { final SearchType searchType = RandomPicks.randomFrom(random(), Arrays.asList(SearchType.QUERY_THEN_FETCH, SearchType.DFS_QUERY_THEN_FETCH)); From 226aad1c239699690cab3f1f3369e6b1046c033b Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 16 Aug 2018 12:27:21 +0200 Subject: [PATCH 10/42] AwaitFix AckIT. Relates #32767 --- server/src/test/java/org/elasticsearch/cluster/ack/AckIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/cluster/ack/AckIT.java b/server/src/test/java/org/elasticsearch/cluster/ack/AckIT.java index 2cd8a2c27c714..df97854cc35b0 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ack/AckIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/ack/AckIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster.ack; +import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; @@ -50,6 +51,7 @@ import static org.hamcrest.Matchers.notNullValue; @ClusterScope(minNumDataNodes = 2) +@AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/32767") public class AckIT extends ESIntegTestCase { @Override From c08dd01bfa45036efa1adef6b9be2ca85285ca08 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 16 Aug 2018 12:13:19 +0100 Subject: [PATCH 11/42] [ML] BWC changes for backport of PR #32816 --- .../xpack/core/ml/job/config/AnalysisConfig.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java index 135ad755359e6..9068ffda4de55 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java @@ -162,9 +162,8 @@ public AnalysisConfig(StreamInput in) throws IOException { } // BWC for removed per-partition normalization - // Version check is temporarily against the latest to satisfy CI tests - // TODO change to V_6_5_0 after successful backport to 6.x - if (in.getVersion().before(Version.V_7_0_0_alpha1)) { + // TODO Remove in 7.0.0 + if (in.getVersion().before(Version.V_6_5_0)) { in.readBoolean(); } } @@ -197,9 +196,8 @@ public void writeTo(StreamOutput out) throws IOException { } // BWC for removed per-partition normalization - // Version check is temporarily against the latest to satisfy CI tests - // TODO change to V_6_5_0 after successful backport to 6.x - if (out.getVersion().before(Version.V_7_0_0_alpha1)) { + // TODO Remove in 7.0.0 + if (out.getVersion().before(Version.V_6_5_0)) { out.writeBoolean(false); } } From 1b5faaf3fb33d9b61083e42ed76b20156dc8ce3b Mon Sep 17 00:00:00 2001 From: javanna Date: Thu, 16 Aug 2018 13:37:09 +0200 Subject: [PATCH 12/42] [DOCS] fix sentence in monitoring.asciidoc Relates to #32097 --- .../en/security/tribe-clients-integrations/monitoring.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/docs/en/security/tribe-clients-integrations/monitoring.asciidoc b/x-pack/docs/en/security/tribe-clients-integrations/monitoring.asciidoc index 67bffadfb296b..aad11ebe707e0 100644 --- a/x-pack/docs/en/security/tribe-clients-integrations/monitoring.asciidoc +++ b/x-pack/docs/en/security/tribe-clients-integrations/monitoring.asciidoc @@ -2,7 +2,7 @@ === Monitoring and Security <> consists of two components: an agent -that you install on on each {es} and Logstash node, and a Monitoring UI +that you install on each {es} and Logstash node, and a Monitoring UI in {kib}. The monitoring agent collects and indexes metrics from the nodes and you visualize the data through the Monitoring dashboards in {kib}. The agent can index data on the same {es} cluster, or send it to an external From 973dc58f60b37b416a9dc050d11471a3d30be634 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Thu, 16 Aug 2018 07:24:05 -0400 Subject: [PATCH 13/42] Remove passphrase support from reload settings API (#32889) We do not support passphrases on the secure settings storage (the keystore). Yet, we added support for this in the API layer. This commit removes this support so that we are not limited in our future options, or have to make a breaking change. --- .../NodesReloadSecureSettingsRequest.java | 68 +------------ ...desReloadSecureSettingsRequestBuilder.java | 49 ---------- ...nsportNodesReloadSecureSettingsAction.java | 6 +- .../RestReloadSecureSettingsAction.java | 11 +-- .../action/admin/ReloadSecureSettingsIT.java | 98 ++----------------- 5 files changed, 16 insertions(+), 216 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java index 5320470d366e1..fb3e6ac71adf3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java @@ -19,82 +19,22 @@ package org.elasticsearch.action.admin.cluster.node.reload; - -import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.nodes.BaseNodesRequest; -import org.elasticsearch.common.CharArrays; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.settings.SecureString; - -import java.io.IOException; -import java.util.Arrays; - -import static org.elasticsearch.action.ValidateActions.addValidationError; /** - * Request for a reload secure settings action + * Request for a reload secure settings action. */ public class NodesReloadSecureSettingsRequest extends BaseNodesRequest { - /** - * The password which is broadcasted to all nodes, but is never stored on - * persistent storage. The password is used to reread and decrypt the contents - * of the node's keystore (backing the implementation of - * {@code SecureSettings}). - */ - private SecureString secureSettingsPassword; - public NodesReloadSecureSettingsRequest() { } /** - * Reload secure settings only on certain nodes, based on the nodes ids - * specified. If none are passed, secure settings will be reloaded on all the - * nodes. + * Reload secure settings only on certain nodes, based on the nodes IDs specified. If none are passed, secure settings will be reloaded + * on all the nodes. */ - public NodesReloadSecureSettingsRequest(String... nodesIds) { + public NodesReloadSecureSettingsRequest(final String... nodesIds) { super(nodesIds); } - @Override - public ActionRequestValidationException validate() { - ActionRequestValidationException validationException = null; - if (secureSettingsPassword == null) { - validationException = addValidationError("secure settings password cannot be null (use empty string instead)", - validationException); - } - return validationException; - } - - public SecureString secureSettingsPassword() { - return secureSettingsPassword; - } - - public NodesReloadSecureSettingsRequest secureStorePassword(SecureString secureStorePassword) { - this.secureSettingsPassword = secureStorePassword; - return this; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - final byte[] passwordBytes = in.readByteArray(); - try { - this.secureSettingsPassword = new SecureString(CharArrays.utf8BytesToChars(passwordBytes)); - } finally { - Arrays.fill(passwordBytes, (byte) 0); - } - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - final byte[] passwordBytes = CharArrays.toUtf8Bytes(this.secureSettingsPassword.getChars()); - try { - out.writeByteArray(passwordBytes); - } finally { - Arrays.fill(passwordBytes, (byte) 0); - } - } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequestBuilder.java index b5f2f73e56f51..c8250455e6ba3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequestBuilder.java @@ -19,19 +19,8 @@ package org.elasticsearch.action.admin.cluster.node.reload; -import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Objects; /** * Builder for the reload secure settings nodes request @@ -39,46 +28,8 @@ public class NodesReloadSecureSettingsRequestBuilder extends NodesOperationRequestBuilder { - public static final String SECURE_SETTINGS_PASSWORD_FIELD_NAME = "secure_settings_password"; - public NodesReloadSecureSettingsRequestBuilder(ElasticsearchClient client, NodesReloadSecureSettingsAction action) { super(client, action, new NodesReloadSecureSettingsRequest()); } - public NodesReloadSecureSettingsRequestBuilder setSecureStorePassword(SecureString secureStorePassword) { - request.secureStorePassword(secureStorePassword); - return this; - } - - public NodesReloadSecureSettingsRequestBuilder source(BytesReference source, XContentType xContentType) throws IOException { - Objects.requireNonNull(xContentType); - // EMPTY is ok here because we never call namedObject - try (InputStream stream = source.streamInput(); - XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, - LoggingDeprecationHandler.INSTANCE, stream)) { - XContentParser.Token token; - token = parser.nextToken(); - if (token != XContentParser.Token.START_OBJECT) { - throw new ElasticsearchParseException("expected an object, but found token [{}]", token); - } - token = parser.nextToken(); - if (token != XContentParser.Token.FIELD_NAME || false == SECURE_SETTINGS_PASSWORD_FIELD_NAME.equals(parser.currentName())) { - throw new ElasticsearchParseException("expected a field named [{}], but found [{}]", SECURE_SETTINGS_PASSWORD_FIELD_NAME, - token); - } - token = parser.nextToken(); - if (token != XContentParser.Token.VALUE_STRING) { - throw new ElasticsearchParseException("expected field [{}] to be of type string, but found [{}] instead", - SECURE_SETTINGS_PASSWORD_FIELD_NAME, token); - } - final String password = parser.text(); - setSecureStorePassword(new SecureString(password.toCharArray())); - token = parser.nextToken(); - if (token != XContentParser.Token.END_OBJECT) { - throw new ElasticsearchParseException("expected end of object, but found token [{}]", token); - } - } - return this; - } - } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java index cb870e58d3187..a4de42597303b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.KeyStoreWrapper; -import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.PluginsService; @@ -83,16 +82,13 @@ protected NodesReloadSecureSettingsResponse.NodeResponse newNodeResponse() { @Override protected NodesReloadSecureSettingsResponse.NodeResponse nodeOperation(NodeRequest nodeReloadRequest) { - final NodesReloadSecureSettingsRequest request = nodeReloadRequest.request; - final SecureString secureSettingsPassword = request.secureSettingsPassword(); try (KeyStoreWrapper keystore = KeyStoreWrapper.load(environment.configFile())) { // reread keystore from config file if (keystore == null) { return new NodesReloadSecureSettingsResponse.NodeResponse(clusterService.localNode(), new IllegalStateException("Keystore is missing")); } - // decrypt the keystore using the password from the request - keystore.decrypt(secureSettingsPassword.getChars()); + keystore.decrypt(new char[0]); // add the keystore to the original node settings object final Settings settingsWithKeystore = Settings.builder() .put(environment.settings(), false) diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java index 0697871ea5d1c..2251615d678fb 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java @@ -59,7 +59,6 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client .cluster() .prepareReloadSecureSettings() .setTimeout(request.param("timeout")) - .source(request.requiredContent(), request.getXContentType()) .setNodesIds(nodesIds); final NodesReloadSecureSettingsRequest nodesRequest = nodesRequestBuilder.request(); return channel -> nodesRequestBuilder @@ -68,12 +67,12 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client public RestResponse buildResponse(NodesReloadSecureSettingsResponse response, XContentBuilder builder) throws Exception { builder.startObject(); - RestActions.buildNodesHeader(builder, channel.request(), response); - builder.field("cluster_name", response.getClusterName().value()); - response.toXContent(builder, channel.request()); + { + RestActions.buildNodesHeader(builder, channel.request(), response); + builder.field("cluster_name", response.getClusterName().value()); + response.toXContent(builder, channel.request()); + } builder.endObject(); - // clear password for the original request - nodesRequest.secureSettingsPassword().close(); return new BytesRestResponse(RestStatus.OK, builder); } }); diff --git a/server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java b/server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java index 7952758240544..3f9e258ffec1c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java @@ -20,11 +20,9 @@ package org.elasticsearch.action.admin; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.admin.cluster.node.reload.NodesReloadSecureSettingsResponse; import org.elasticsearch.common.settings.KeyStoreWrapper; import org.elasticsearch.common.settings.SecureSettings; -import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.Plugin; @@ -44,11 +42,11 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.containsString; public class ReloadSecureSettingsIT extends ESIntegTestCase { @@ -62,7 +60,7 @@ public void testMissingKeystoreFile() throws Exception { Files.deleteIfExists(KeyStoreWrapper.keystorePath(environment.configFile())); final int initialReloadCount = mockReloadablePlugin.getReloadCount(); final CountDownLatch latch = new CountDownLatch(1); - client().admin().cluster().prepareReloadSecureSettings().setSecureStorePassword(new SecureString(new char[0])).execute( + client().admin().cluster().prepareReloadSecureSettings().execute( new ActionListener() { @Override public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) { @@ -96,44 +94,6 @@ public void onFailure(Exception e) { assertThat(mockReloadablePlugin.getReloadCount(), equalTo(initialReloadCount)); } - public void testNullKeystorePassword() throws Exception { - final PluginsService pluginsService = internalCluster().getInstance(PluginsService.class); - final MockReloadablePlugin mockReloadablePlugin = pluginsService.filterPlugins(MockReloadablePlugin.class) - .stream().findFirst().get(); - final AtomicReference reloadSettingsError = new AtomicReference<>(); - final int initialReloadCount = mockReloadablePlugin.getReloadCount(); - final CountDownLatch latch = new CountDownLatch(1); - client().admin().cluster().prepareReloadSecureSettings().execute( - new ActionListener() { - @Override - public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) { - try { - reloadSettingsError.set(new AssertionError("Null keystore password should fail")); - } finally { - latch.countDown(); - } - } - - @Override - public void onFailure(Exception e) { - try { - assertThat(e, instanceOf(ActionRequestValidationException.class)); - assertThat(e.getMessage(), containsString("secure settings password cannot be null")); - } catch (final AssertionError ae) { - reloadSettingsError.set(ae); - } finally { - latch.countDown(); - } - } - }); - latch.await(); - if (reloadSettingsError.get() != null) { - throw reloadSettingsError.get(); - } - // in the null password case no reload should be triggered - assertThat(mockReloadablePlugin.getReloadCount(), equalTo(initialReloadCount)); - } - public void testInvalidKeystoreFile() throws Exception { final PluginsService pluginsService = internalCluster().getInstance(PluginsService.class); final MockReloadablePlugin mockReloadablePlugin = pluginsService.filterPlugins(MockReloadablePlugin.class) @@ -149,7 +109,7 @@ public void testInvalidKeystoreFile() throws Exception { Files.copy(keystore, KeyStoreWrapper.keystorePath(environment.configFile()), StandardCopyOption.REPLACE_EXISTING); } final CountDownLatch latch = new CountDownLatch(1); - client().admin().cluster().prepareReloadSecureSettings().setSecureStorePassword(new SecureString(new char[0])).execute( + client().admin().cluster().prepareReloadSecureSettings().execute( new ActionListener() { @Override public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) { @@ -181,52 +141,6 @@ public void onFailure(Exception e) { assertThat(mockReloadablePlugin.getReloadCount(), equalTo(initialReloadCount)); } - public void testWrongKeystorePassword() throws Exception { - final PluginsService pluginsService = internalCluster().getInstance(PluginsService.class); - final MockReloadablePlugin mockReloadablePlugin = pluginsService.filterPlugins(MockReloadablePlugin.class) - .stream().findFirst().get(); - final Environment environment = internalCluster().getInstance(Environment.class); - final AtomicReference reloadSettingsError = new AtomicReference<>(); - final int initialReloadCount = mockReloadablePlugin.getReloadCount(); - // "some" keystore should be present in this case - writeEmptyKeystore(environment, new char[0]); - final CountDownLatch latch = new CountDownLatch(1); - client().admin() - .cluster() - .prepareReloadSecureSettings() - .setSecureStorePassword(new SecureString(new char[] { 'W', 'r', 'o', 'n', 'g' })) - .execute(new ActionListener() { - @Override - public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) { - try { - assertThat(nodesReloadResponse, notNullValue()); - final Map nodesMap = nodesReloadResponse.getNodesMap(); - assertThat(nodesMap.size(), equalTo(cluster().size())); - for (final NodesReloadSecureSettingsResponse.NodeResponse nodeResponse : nodesReloadResponse.getNodes()) { - assertThat(nodeResponse.reloadException(), notNullValue()); - assertThat(nodeResponse.reloadException(), instanceOf(SecurityException.class)); - } - } catch (final AssertionError e) { - reloadSettingsError.set(e); - } finally { - latch.countDown(); - } - } - - @Override - public void onFailure(Exception e) { - reloadSettingsError.set(new AssertionError("Nodes request failed", e)); - latch.countDown(); - } - }); - latch.await(); - if (reloadSettingsError.get() != null) { - throw reloadSettingsError.get(); - } - // in the wrong password case no reload should be triggered - assertThat(mockReloadablePlugin.getReloadCount(), equalTo(initialReloadCount)); - } - public void testMisbehavingPlugin() throws Exception { final Environment environment = internalCluster().getInstance(Environment.class); final PluginsService pluginsService = internalCluster().getInstance(PluginsService.class); @@ -247,7 +161,7 @@ public void testMisbehavingPlugin() throws Exception { .get(Settings.builder().put(environment.settings()).setSecureSettings(secureSettings).build()) .toString(); final CountDownLatch latch = new CountDownLatch(1); - client().admin().cluster().prepareReloadSecureSettings().setSecureStorePassword(new SecureString(new char[0])).execute( + client().admin().cluster().prepareReloadSecureSettings().execute( new ActionListener() { @Override public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) { @@ -314,7 +228,7 @@ protected Collection> nodePlugins() { private void successfulReloadCall() throws InterruptedException { final AtomicReference reloadSettingsError = new AtomicReference<>(); final CountDownLatch latch = new CountDownLatch(1); - client().admin().cluster().prepareReloadSecureSettings().setSecureStorePassword(new SecureString(new char[0])).execute( + client().admin().cluster().prepareReloadSecureSettings().execute( new ActionListener() { @Override public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) { From dbff0926d49becedd43f0615ded376b1bd73eedc Mon Sep 17 00:00:00 2001 From: Hazem Khaled Date: Thu, 16 Aug 2018 14:54:04 +0300 Subject: [PATCH 14/42] [DOCS] Update WordPress plugins links (#32194) --- docs/plugins/integrations.asciidoc | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/plugins/integrations.asciidoc b/docs/plugins/integrations.asciidoc index 90f2c685fdaeb..8bffe5193ed7b 100644 --- a/docs/plugins/integrations.asciidoc +++ b/docs/plugins/integrations.asciidoc @@ -17,14 +17,11 @@ Integrations are not plugins, but are external tools or modules that make it eas * https://drupal.org/project/elasticsearch_connector[Drupal]: Drupal Elasticsearch integration. -* https://wordpress.org/plugins/wpsolr-search-engine/[WPSOLR]: - Elasticsearch (and Apache Solr) WordPress Plugin - -* http://searchbox-io.github.com/wp-elasticsearch/[Wp-Elasticsearch]: +* https://wordpress.org/plugins/elasticpress/[ElasticPress]: Elasticsearch WordPress Plugin -* https://github.com/wallmanderco/elasticsearch-indexer[Elasticsearch Indexer]: - Elasticsearch WordPress Plugin +* https://wordpress.org/plugins/wpsolr-search-engine/[WPSOLR]: + Elasticsearch (and Apache Solr) WordPress Plugin * https://doc.tiki.org/Elasticsearch[Tiki Wiki CMS Groupware]: Tiki has native support for Elasticsearch. This provides faster & better From 6333e004c224fe2dd5799f040c3d5dbc76459be0 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Thu, 16 Aug 2018 07:18:43 -0500 Subject: [PATCH 15/42] HLRC: adding machine learning delete job (#32820) * HLRC: adding machine learning delete job * Fixing whitespace * Moving docs and tests around * Unifying ml asciidoc file naming convention --- .../client/MachineLearningClient.java | 40 ++++++++++ .../client/RequestConverters.java | 16 ++++ .../client/MachineLearningIT.java | 15 ++++ .../client/RequestConvertersTests.java | 15 ++++ .../MlClientDocumentationIT.java | 56 +++++++++++++- .../high-level/ml/delete-job.asciidoc | 49 ++++++++++++ .../high-level/supported-apis.asciidoc | 2 + .../protocol/xpack/ml/DeleteJobRequest.java | 75 +++++++++++++++++++ .../protocol/xpack/ml/DeleteJobResponse.java | 60 +++++++++++++++ .../xpack/ml/DeleteJobRequestTests.java | 45 +++++++++++ .../xpack/ml/DeleteJobResponseTests.java | 42 +++++++++++ 11 files changed, 412 insertions(+), 3 deletions(-) create mode 100644 docs/java-rest/high-level/ml/delete-job.asciidoc create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/DeleteJobRequest.java create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/DeleteJobResponse.java create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/DeleteJobRequestTests.java create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/DeleteJobResponseTests.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java index a3e5ba72b773f..2e7914e64abdb 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java @@ -19,6 +19,8 @@ package org.elasticsearch.client; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest; +import org.elasticsearch.protocol.xpack.ml.DeleteJobResponse; import org.elasticsearch.protocol.xpack.ml.OpenJobRequest; import org.elasticsearch.protocol.xpack.ml.OpenJobResponse; import org.elasticsearch.protocol.xpack.ml.PutJobRequest; @@ -80,6 +82,44 @@ public void putJobAsync(PutJobRequest request, RequestOptions options, ActionLis Collections.emptySet()); } + /** + * Deletes the given Machine Learning Job + *

    + * For additional info + * see ML Delete Job documentation + *

    + * @param request the request to delete the job + * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return action acknowledgement + * @throws IOException when there is a serialization issue sending the request or receiving the response + */ + public DeleteJobResponse deleteJob(DeleteJobRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(request, + RequestConverters::deleteMachineLearningJob, + options, + DeleteJobResponse::fromXContent, + Collections.emptySet()); + } + + /** + * Deletes the given Machine Learning Job asynchronously and notifies the listener on completion + *

    + * For additional info + * see ML Delete Job documentation + *

    + * @param request the request to delete the job + * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener Listener to be notified upon request completion + */ + public void deleteJobAsync(DeleteJobRequest request, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(request, + RequestConverters::deleteMachineLearningJob, + options, + DeleteJobResponse::fromXContent, + listener, + Collections.emptySet()); + } + /** * Opens a Machine Learning Job. * When you open a new job, it starts with an empty model. diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 596dd51f88af5..6ae673a36910f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -112,6 +112,7 @@ import org.elasticsearch.protocol.xpack.license.GetLicenseRequest; import org.elasticsearch.protocol.xpack.license.PutLicenseRequest; import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest; +import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest; import org.elasticsearch.protocol.xpack.ml.OpenJobRequest; import org.elasticsearch.protocol.xpack.ml.PutJobRequest; import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest; @@ -1222,6 +1223,21 @@ static Request putMachineLearningJob(PutJobRequest putJobRequest) throws IOExcep return request; } + static Request deleteMachineLearningJob(DeleteJobRequest deleteJobRequest) { + String endpoint = new EndpointBuilder() + .addPathPartAsIs("_xpack") + .addPathPartAsIs("ml") + .addPathPartAsIs("anomaly_detectors") + .addPathPart(deleteJobRequest.getJobId()) + .build(); + Request request = new Request(HttpDelete.METHOD_NAME, endpoint); + + Params params = new Params(request); + params.putParam("force", Boolean.toString(deleteJobRequest.isForce())); + + return request; + } + static Request machineLearningOpenJob(OpenJobRequest openJobRequest) throws IOException { String endpoint = new EndpointBuilder() .addPathPartAsIs("_xpack") diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java index 94e73a14c188c..0037460150f1a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java @@ -20,6 +20,8 @@ import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest; +import org.elasticsearch.protocol.xpack.ml.DeleteJobResponse; import org.elasticsearch.protocol.xpack.ml.OpenJobRequest; import org.elasticsearch.protocol.xpack.ml.OpenJobResponse; import org.elasticsearch.protocol.xpack.ml.PutJobRequest; @@ -48,6 +50,19 @@ public void testPutJob() throws Exception { assertThat(createdJob.getJobType(), is(Job.ANOMALY_DETECTOR_JOB_TYPE)); } + public void testDeleteJob() throws Exception { + String jobId = randomValidJobId(); + Job job = buildJob(jobId); + MachineLearningClient machineLearningClient = highLevelClient().machineLearning(); + machineLearningClient.putJob(new PutJobRequest(job), RequestOptions.DEFAULT); + + DeleteJobResponse response = execute(new DeleteJobRequest(jobId), + machineLearningClient::deleteJob, + machineLearningClient::deleteJobAsync); + + assertTrue(response.isAcknowledged()); + } + public void testOpenJob() throws Exception { String jobId = randomValidJobId(); Job job = buildJob(jobId); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index daf754d9daec4..b4fd6c914b39c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -128,6 +128,7 @@ import org.elasticsearch.index.rankeval.RestRankEvalAction; import org.elasticsearch.protocol.xpack.XPackInfoRequest; import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest; +import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest; import org.elasticsearch.protocol.xpack.ml.OpenJobRequest; import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest; import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest; @@ -2645,6 +2646,20 @@ public void testXPackDeleteWatch() { assertThat(request.getEntity(), nullValue()); } + public void testDeleteMachineLearningJob() { + String jobId = randomAlphaOfLength(10); + DeleteJobRequest deleteJobRequest = new DeleteJobRequest(jobId); + + Request request = RequestConverters.deleteMachineLearningJob(deleteJobRequest); + assertEquals(HttpDelete.METHOD_NAME, request.getMethod()); + assertEquals("/_xpack/ml/anomaly_detectors/" + jobId, request.getEndpoint()); + assertEquals(Boolean.toString(false), request.getParameters().get("force")); + + deleteJobRequest.setForce(true); + request = RequestConverters.deleteMachineLearningJob(deleteJobRequest); + assertEquals(Boolean.toString(true), request.getParameters().get("force")); + } + public void testPostMachineLearningOpenJob() throws Exception { String jobId = "some-job-id"; OpenJobRequest openJobRequest = new OpenJobRequest(jobId); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index 50cd244c0fa07..a77d8b43e5737 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -25,6 +25,8 @@ import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest; +import org.elasticsearch.protocol.xpack.ml.DeleteJobResponse; import org.elasticsearch.protocol.xpack.ml.OpenJobRequest; import org.elasticsearch.protocol.xpack.ml.OpenJobResponse; import org.elasticsearch.protocol.xpack.ml.PutJobRequest; @@ -122,6 +124,56 @@ public void onFailure(Exception e) { } } + public void testDeleteJob() throws Exception { + RestHighLevelClient client = highLevelClient(); + + String jobId = "my-first-machine-learning-job"; + + Job job = MachineLearningIT.buildJob(jobId); + client.machineLearning().putJob(new PutJobRequest(job), RequestOptions.DEFAULT); + + Job secondJob = MachineLearningIT.buildJob("my-second-machine-learning-job"); + client.machineLearning().putJob(new PutJobRequest(secondJob), RequestOptions.DEFAULT); + + { + //tag::x-pack-delete-ml-job-request + DeleteJobRequest deleteJobRequest = new DeleteJobRequest("my-first-machine-learning-job"); + deleteJobRequest.setForce(false); //<1> + DeleteJobResponse deleteJobResponse = client.machineLearning().deleteJob(deleteJobRequest, RequestOptions.DEFAULT); + //end::x-pack-delete-ml-job-request + + //tag::x-pack-delete-ml-job-response + boolean isAcknowledged = deleteJobResponse.isAcknowledged(); //<1> + //end::x-pack-delete-ml-job-response + } + { + //tag::x-pack-delete-ml-job-request-listener + ActionListener listener = new ActionListener() { + @Override + public void onResponse(DeleteJobResponse deleteJobResponse) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + //end::x-pack-delete-ml-job-request-listener + + // Replace the empty listener by a blocking listener in test + final CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + //tag::x-pack-delete-ml-job-request-async + DeleteJobRequest deleteJobRequest = new DeleteJobRequest("my-second-machine-learning-job"); + client.machineLearning().deleteJobAsync(deleteJobRequest, RequestOptions.DEFAULT, listener); // <1> + //end::x-pack-delete-ml-job-request-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + } + public void testOpenJob() throws Exception { RestHighLevelClient client = highLevelClient(); @@ -143,7 +195,6 @@ public void testOpenJob() throws Exception { //end::x-pack-ml-open-job-execute } - { //tag::x-pack-ml-open-job-listener ActionListener listener = new ActionListener() { @@ -154,7 +205,7 @@ public void onResponse(OpenJobResponse openJobResponse) { @Override public void onFailure(Exception e) { - //<2> + // <2> } }; //end::x-pack-ml-open-job-listener @@ -169,6 +220,5 @@ public void onFailure(Exception e) { assertTrue(latch.await(30L, TimeUnit.SECONDS)); } - } } diff --git a/docs/java-rest/high-level/ml/delete-job.asciidoc b/docs/java-rest/high-level/ml/delete-job.asciidoc new file mode 100644 index 0000000000000..44a6a47940955 --- /dev/null +++ b/docs/java-rest/high-level/ml/delete-job.asciidoc @@ -0,0 +1,49 @@ +[[java-rest-high-x-pack-ml-delete-job]] +=== Delete Job API + +[[java-rest-high-x-pack-machine-learning-delete-job-request]] +==== Delete Job Request + +A `DeleteJobRequest` object requires a non-null `jobId` and can optionally set `force`. +Can be executed as follows: + +["source","java",subs="attributes,callouts,macros"] +--------------------------------------------------- +include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-delete-ml-job-request] +--------------------------------------------------- +<1> Use to forcefully delete an opened job; +this method is quicker than closing and deleting the job. +Defaults to `false` + +[[java-rest-high-x-pack-machine-learning-delete-job-response]] +==== Delete Job Response + +The returned `DeleteJobResponse` object indicates the acknowledgement of the request: +["source","java",subs="attributes,callouts,macros"] +--------------------------------------------------- +include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-delete-ml-job-response] +--------------------------------------------------- +<1> `isAcknowledged` was the deletion request acknowledged or not + +[[java-rest-high-x-pack-machine-learning-delete-job-async]] +==== Delete Job Asynchronously + +This request can also be made asynchronously. +["source","java",subs="attributes,callouts,macros"] +--------------------------------------------------- +include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-delete-ml-job-request-async] +--------------------------------------------------- +<1> The `DeleteJobRequest` to execute and the `ActionListener` to alert on completion or error. + +The deletion request returns immediately. Once the request is completed, the `ActionListener` is +called back using the `onResponse` or `onFailure`. The latter indicates some failure occurred when +making the request. + +A typical listener for a `DeleteJobRequest` could be defined as follows: + +["source","java",subs="attributes,callouts,macros"] +--------------------------------------------------- +include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-delete-ml-job-request-listener] +--------------------------------------------------- +<1> The action to be taken when it is completed +<2> What to do when a failure occurs diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index a2db3436317c3..6bcb736243a7c 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -205,9 +205,11 @@ include::licensing/delete-license.asciidoc[] The Java High Level REST Client supports the following Machine Learning APIs: * <> +* <> * <> include::ml/put-job.asciidoc[] +include::ml/delete-job.asciidoc[] include::ml/open-job.asciidoc[] == Migration APIs diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/DeleteJobRequest.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/DeleteJobRequest.java new file mode 100644 index 0000000000000..1b7450de0929c --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/DeleteJobRequest.java @@ -0,0 +1,75 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; + +import java.util.Objects; + +public class DeleteJobRequest extends ActionRequest { + + private String jobId; + private boolean force; + + public DeleteJobRequest(String jobId) { + this.jobId = Objects.requireNonNull(jobId, "[job_id] must not be null"); + } + + public String getJobId() { + return jobId; + } + + public void setJobId(String jobId) { + this.jobId = Objects.requireNonNull(jobId, "[job_id] must not be null"); + } + + public boolean isForce() { + return force; + } + + public void setForce(boolean force) { + this.force = force; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, force); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || obj.getClass() != getClass()) { + return false; + } + + DeleteJobRequest other = (DeleteJobRequest) obj; + return Objects.equals(jobId, other.jobId) && Objects.equals(force, other.force); + } + +} diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/DeleteJobResponse.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/DeleteJobResponse.java new file mode 100644 index 0000000000000..0b4faa38f545f --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/DeleteJobResponse.java @@ -0,0 +1,60 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml; + +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +public class DeleteJobResponse extends AcknowledgedResponse { + + public DeleteJobResponse(boolean acknowledged) { + super(acknowledged); + } + + public DeleteJobResponse() { + } + + public static DeleteJobResponse fromXContent(XContentParser parser) throws IOException { + AcknowledgedResponse response = AcknowledgedResponse.fromXContent(parser); + return new DeleteJobResponse(response.isAcknowledged()); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + DeleteJobResponse that = (DeleteJobResponse) other; + return isAcknowledged() == that.isAcknowledged(); + } + + @Override + public int hashCode() { + return Objects.hash(isAcknowledged()); + } + +} diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/DeleteJobRequestTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/DeleteJobRequestTests.java new file mode 100644 index 0000000000000..fb8a38fa0c68e --- /dev/null +++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/DeleteJobRequestTests.java @@ -0,0 +1,45 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml; + +import org.elasticsearch.protocol.xpack.ml.job.config.JobTests; +import org.elasticsearch.test.ESTestCase; + +public class DeleteJobRequestTests extends ESTestCase { + + private DeleteJobRequest createTestInstance() { + return new DeleteJobRequest(JobTests.randomValidJobId()); + } + + public void test_WithNullJobId() { + NullPointerException ex = expectThrows(NullPointerException.class, () -> new DeleteJobRequest(null)); + assertEquals("[job_id] must not be null", ex.getMessage()); + + ex = expectThrows(NullPointerException.class, () -> createTestInstance().setJobId(null)); + assertEquals("[job_id] must not be null", ex.getMessage()); + } + + public void test_WithForce() { + DeleteJobRequest deleteJobRequest = createTestInstance(); + assertFalse(deleteJobRequest.isForce()); + + deleteJobRequest.setForce(true); + assertTrue(deleteJobRequest.isForce()); + } +} diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/DeleteJobResponseTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/DeleteJobResponseTests.java new file mode 100644 index 0000000000000..a73179a08983d --- /dev/null +++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/DeleteJobResponseTests.java @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; + +public class DeleteJobResponseTests extends AbstractXContentTestCase { + + @Override + protected DeleteJobResponse createTestInstance() { + return new DeleteJobResponse(); + } + + @Override + protected DeleteJobResponse doParseInstance(XContentParser parser) throws IOException { + return DeleteJobResponse.fromXContent(parser); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } +} From 087da3dc8a8f320c2e0a09aef6443aedda375c71 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Thu, 16 Aug 2018 15:33:17 +0200 Subject: [PATCH 16/42] [Test] Fix DuelScrollIT#testDuelIndexOrderQueryThenFetch This commit disables the automatic `refresh_interval` in order to ensure that index readers cannot differ between the normal and scroll search. This issue is related to the 7.5 Lucene upgrade which contains a change that makes single segment merge more likely to occur (max deletes percentage). Closes #32682 --- .../java/org/elasticsearch/search/scroll/DuelScrollIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java b/server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java index 1ddd11e5d0f7d..4005f1218a92f 100644 --- a/server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java +++ b/server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java @@ -199,6 +199,8 @@ private int createIndex(boolean singleShard) throws Exception { } // no replicas, as they might be ordered differently settings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0); + // we need to control refreshes as they might take different merges into account + settings.put("index.refresh_interval", -1); assertAcked(prepareCreate("test").setSettings(settings.build()).get()); final int numDocs = randomIntBetween(10, 200); @@ -257,7 +259,6 @@ private void testDuelIndexOrder(SearchType searchType, boolean trackScores, int } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32682") public void testDuelIndexOrderQueryThenFetch() throws Exception { final SearchType searchType = RandomPicks.randomFrom(random(), Arrays.asList(SearchType.QUERY_THEN_FETCH, SearchType.DFS_QUERY_THEN_FETCH)); From bc3efbf5cfee7ae5804d225d995f266871b983cc Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 16 Aug 2018 15:25:51 +0200 Subject: [PATCH 17/42] AwaitFix FullClusterRestartIT#testRollupIDSchemeAfterRestart. --- .../org/elasticsearch/xpack/restart/FullClusterRestartIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index 71026c6ae836f..5e74a9933bcc1 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -344,6 +344,7 @@ public void testRollupAfterRestart() throws Exception { } } + @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/32773") public void testRollupIDSchemeAfterRestart() throws Exception { assumeTrue("Rollup can be tested with 6.3.0 and onwards", oldClusterVersion.onOrAfter(Version.V_6_3_0)); assumeTrue("Rollup ID scheme changed in 6.4", oldClusterVersion.before(Version.V_6_4_0)); From 3bbffd22039143daee00fe940eb05d4fbc42a974 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Thu, 16 Aug 2018 11:32:35 -0400 Subject: [PATCH 18/42] Fix docs for fixed filename for heap dump path (#32882) The docs here incorrectly state that it is okay for a heap dump file to exist when heap dump path is configured to a fixed filename. This is incorrect, the JVM will fail to write the heap dump if a heap dump file already exists at the specified location (see the DumpWriter constructor DumpWriter::DumpWriter(const char* path) in the JVM source). --- .../setup/important-settings/heap-dump-path.asciidoc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/reference/setup/important-settings/heap-dump-path.asciidoc b/docs/reference/setup/important-settings/heap-dump-path.asciidoc index b0d301b21d0b8..fb8c7ff35f0d0 100644 --- a/docs/reference/setup/important-settings/heap-dump-path.asciidoc +++ b/docs/reference/setup/important-settings/heap-dump-path.asciidoc @@ -8,8 +8,8 @@ distributions, and the `data` directory under the root of the Elasticsearch installation for the <> archive distributions). If this path is not suitable for receiving heap dumps, you should modify the entry `-XX:HeapDumpPath=...` in -<>. If you specify a fixed filename instead -of a directory, the JVM will repeatedly use the same file; this is one -mechanism for preventing heap dumps from accumulating in the heap dump -path. Alternatively, you can configure a scheduled task via your OS to -remove heap dumps that are older than a configured age. +<>. If you specify a directory, the JVM +will generate a filename for the heap dump based on the PID of the running +instance. If you specify a fixed filename instead of a directory, the file must +not exist when the JVM needs to perform a heap dump on an out of memory +exception, otherwise the heap dump will fail. From 07e2e299e1a4f21871d9d1bc291e38444108f732 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Thu, 16 Aug 2018 08:03:21 -0700 Subject: [PATCH 19/42] Painless: Special Case def (#32871) This removes def from the classes map in PainlessLookup and instead always special cases it. This prevents potential calls against the def type that shouldn't be made and forces all cases of def throughout Painless code to be special cased. --- .../java/org/elasticsearch/painless/ScriptClassInfo.java | 3 ++- .../org/elasticsearch/painless/lookup/PainlessLookup.java | 3 ++- .../elasticsearch/painless/lookup/PainlessLookupBuilder.java | 5 +---- .../elasticsearch/painless/lookup/PainlessLookupUtility.java | 4 ++-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java index 345db46f8875f..7de8353194dda 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java @@ -21,6 +21,7 @@ import org.elasticsearch.painless.lookup.PainlessLookup; import org.elasticsearch.painless.lookup.PainlessLookupUtility; +import org.elasticsearch.painless.lookup.def; import java.lang.invoke.MethodType; import java.lang.reflect.Field; @@ -190,7 +191,7 @@ private static Class definitionTypeForClass(PainlessLookup painlessLookup, Cl componentType = componentType.getComponentType(); } - if (painlessLookup.lookupPainlessClass(componentType) == null) { + if (componentType != def.class && painlessLookup.lookupPainlessClass(componentType) == null) { throw new IllegalArgumentException(unknownErrorMessageSource.apply(componentType)); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java index 16b8ac14f14f2..55855a3cb1efb 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java @@ -26,6 +26,7 @@ import java.util.Set; import java.util.function.Function; +import static org.elasticsearch.painless.lookup.PainlessLookupUtility.DEF_CLASS_NAME; import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessConstructorKey; import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessFieldKey; import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessMethodKey; @@ -47,7 +48,7 @@ public final class PainlessLookup { public boolean isValidCanonicalClassName(String canonicalClassName) { Objects.requireNonNull(canonicalClassName); - return canonicalClassNamesToClasses.containsKey(canonicalClassName); + return DEF_CLASS_NAME.equals(canonicalClassName) || canonicalClassNamesToClasses.containsKey(canonicalClassName); } public Class canonicalTypeNameToType(String canonicalTypeName) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java index e644453a4c1ba..c8353b54c9f44 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java @@ -211,9 +211,6 @@ public static PainlessLookup buildFromWhitelists(List whitelists) { public PainlessLookupBuilder() { canonicalClassNamesToClasses = new HashMap<>(); classesToPainlessClassBuilders = new HashMap<>(); - - canonicalClassNamesToClasses.put(DEF_CLASS_NAME, def.class); - classesToPainlessClassBuilders.put(def.class, new PainlessClassBuilder()); } private Class canonicalTypeNameToType(String canonicalTypeName) { @@ -225,7 +222,7 @@ private boolean isValidType(Class type) { type = type.getComponentType(); } - return classesToPainlessClassBuilders.containsKey(type); + return type == def.class || classesToPainlessClassBuilders.containsKey(type); } public void addPainlessClass(ClassLoader classLoader, String javaClassName, boolean importClassName) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java index f2eb434516961..71cacab9eba9d 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java @@ -82,7 +82,7 @@ public static Class canonicalTypeNameToType(String canonicalTypeName, Map type = canonicalClassNamesToClasses.get(canonicalTypeName); + Class type = DEF_CLASS_NAME.equals(canonicalTypeName) ? def.class : canonicalClassNamesToClasses.get(canonicalTypeName); if (type != null) { return type; @@ -105,7 +105,7 @@ public static Class canonicalTypeNameToType(String canonicalTypeName, Map Date: Thu, 16 Aug 2018 12:15:19 -0700 Subject: [PATCH 20/42] [DOCS] Move breaking change info re rejecting regex search if regex string is too long (#32878) --- docs/reference/migration/migrate_6_0/search.asciidoc | 7 ------- docs/reference/migration/migrate_6_4.asciidoc | 10 ++++++++++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/reference/migration/migrate_6_0/search.asciidoc b/docs/reference/migration/migrate_6_0/search.asciidoc index 00c26813f1589..3416c940e225e 100644 --- a/docs/reference/migration/migrate_6_0/search.asciidoc +++ b/docs/reference/migration/migrate_6_0/search.asciidoc @@ -215,13 +215,6 @@ The ability to query and index context enabled suggestions without contexts has been deprecated. Context enabled suggestion queries without contexts have to visit every suggestion, which degrades the search performance considerably. -==== Limiting the length of regex that can be used in a Regexp Query request - -Regexp Query with long string made of many operators may run into a stack overflow. -To safeguard against this, the maximum length of regex that can be used in a -Regexp Query request has been limited to 1000. This default maximum can be changed -for a particular index with the index setting `index.max_regex_length`. - ==== Limiting the max number of expansion of span_multi queries `span_multi` queries will hit too many clauses failure if the number of terms that match the diff --git a/docs/reference/migration/migrate_6_4.asciidoc b/docs/reference/migration/migrate_6_4.asciidoc index 16a78fae80149..2ee1c4339eed8 100644 --- a/docs/reference/migration/migrate_6_4.asciidoc +++ b/docs/reference/migration/migrate_6_4.asciidoc @@ -36,6 +36,16 @@ to follow up the operation with a request to adjust to the desired settings on the target index, or send the desired value of these settings with the resize operation. +[[breaking_64_search_changes]] +=== Search and query DSL changes + +==== Limiting the length of regex that can be used in a Regexp Query request + +<> with long string made of many operators may run into a stack overflow. +To safeguard against this, the maximum length of regex that can be used in a +Regexp Query request has been limited to 1000. This default maximum can be changed +for a particular index with the index setting `index.max_regex_length`. + [[breaking_64_scripting_changes]] === Scripting From 3c27d59b0ad2608d0d2b284460723901e5350fc2 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Thu, 16 Aug 2018 16:25:34 -0400 Subject: [PATCH 21/42] Guard against null in email admin watches (#32923) The Kibana settings docs that these watches rely on can sometimes contain no xpack settings. When this is the case, we will end up with a null pointer exception in the script. We need to guard against in these scripts so this commit does that. --- .../monitoring/watches/elasticsearch_cluster_status.json | 2 +- .../main/resources/monitoring/watches/elasticsearch_nodes.json | 2 +- .../monitoring/watches/elasticsearch_version_mismatch.json | 2 +- .../resources/monitoring/watches/kibana_version_mismatch.json | 2 +- .../resources/monitoring/watches/logstash_version_mismatch.json | 2 +- .../resources/monitoring/watches/xpack_license_expiration.json | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_cluster_status.json b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_cluster_status.json index 28741890c7f9c..8fc820d9bb0ee 100644 --- a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_cluster_status.json +++ b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_cluster_status.json @@ -145,7 +145,7 @@ }, "transform": { "script": { - "source": "ctx.vars.email_recipient = (ctx.payload.kibana_settings.hits.total > 0) ? ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack.default_admin_email : null;ctx.vars.is_new = ctx.vars.fails_check && !ctx.vars.not_resolved;ctx.vars.is_resolved = !ctx.vars.fails_check && ctx.vars.not_resolved;def state = ctx.payload.check.hits.hits[0]._source.cluster_state.status;if (ctx.vars.not_resolved){ctx.payload = ctx.payload.alert.hits.hits[0]._source;if (ctx.vars.fails_check == false) {ctx.payload.resolved_timestamp = ctx.execution_time;}} else {ctx.payload = ['timestamp': ctx.execution_time, 'metadata': ctx.metadata.xpack];}if (ctx.vars.fails_check) {ctx.payload.prefix = 'Elasticsearch cluster status is ' + state + '.';if (state == 'red') {ctx.payload.message = 'Allocate missing primary shards and replica shards.';ctx.payload.metadata.severity = 2100;} else {ctx.payload.message = 'Allocate missing replica shards.';ctx.payload.metadata.severity = 1100;}}ctx.vars.state = state.toUpperCase();ctx.payload.update_timestamp = ctx.execution_time;return ctx.payload;" + "source": "ctx.vars.email_recipient = (ctx.payload.kibana_settings.hits.total > 0 && ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack != null) ? ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack.default_admin_email : null;ctx.vars.is_new = ctx.vars.fails_check && !ctx.vars.not_resolved;ctx.vars.is_resolved = !ctx.vars.fails_check && ctx.vars.not_resolved;def state = ctx.payload.check.hits.hits[0]._source.cluster_state.status;if (ctx.vars.not_resolved){ctx.payload = ctx.payload.alert.hits.hits[0]._source;if (ctx.vars.fails_check == false) {ctx.payload.resolved_timestamp = ctx.execution_time;}} else {ctx.payload = ['timestamp': ctx.execution_time, 'metadata': ctx.metadata.xpack];}if (ctx.vars.fails_check) {ctx.payload.prefix = 'Elasticsearch cluster status is ' + state + '.';if (state == 'red') {ctx.payload.message = 'Allocate missing primary shards and replica shards.';ctx.payload.metadata.severity = 2100;} else {ctx.payload.message = 'Allocate missing replica shards.';ctx.payload.metadata.severity = 1100;}}ctx.vars.state = state.toUpperCase();ctx.payload.update_timestamp = ctx.execution_time;return ctx.payload;" } }, "actions": { diff --git a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_nodes.json b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_nodes.json index cdf62055d0556..b595adab64b5e 100644 --- a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_nodes.json +++ b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_nodes.json @@ -151,7 +151,7 @@ }, "transform": { "script": { - "source": "void formatResults(StringBuilder message, String type, Map typeMap) {if (typeMap.empty == false) {message.append(' Node');if (typeMap.size() != 1) {message.append('s were');} else {message.append(' was');}message.append(' ').append(type).append(' [').append(typeMap.size()).append(']: ').append(typeMap.values().stream().collect(Collectors.joining(', ', '[', ']'))).append('.');}}ctx.vars.email_recipient = (ctx.payload.kibana_settings.hits.total > 0) ? ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack.default_admin_email : null;def clusterState = ctx.payload.check.hits.hits[0]._source.cluster_state;def persistentUuidToName = [:];def latestNodes = clusterState.nodes;def ephemeralUuidToPersistentUuid = [:];def payload = ['timestamp': ctx.execution_time,'updated_timestamp': ctx.execution_time,'resolved_timestamp': ctx.execution_time,'metadata': ctx.metadata.xpack,'prefix': 'Elasticsearch cluster nodes have changed!','nodes': ['hash': clusterState.nodes_hash,'added': persistentUuidToName,'removed': [:],'restarted': [:]]];for (def latestNode : latestNodes.entrySet()) {persistentUuidToName[latestNode.key] = latestNode.value.name;ephemeralUuidToPersistentUuid[latestNode.value.ephemeral_id] = latestNode.key;}def previousNodes = ctx.payload.check.hits.hits[1]._source.cluster_state.nodes;def previousPersistentUuidToName = [:];for (def previousNode : previousNodes.entrySet()){if (persistentUuidToName.containsKey(previousNode.key) == false){payload.nodes.removed[previousNode.key] = previousNode.value.name;}else{if (ephemeralUuidToPersistentUuid.containsKey(previousNode.value.ephemeral_id) == false) {payload.nodes.restarted[previousNode.key] = persistentUuidToName[previousNode.key];}persistentUuidToName.remove(previousNode.key);}}StringBuilder message = new StringBuilder();formatResults(message, 'removed', payload.nodes.removed);formatResults(message, 'added', payload.nodes.added);formatResults(message, 'restarted', payload.nodes.restarted);payload.message = message.toString().trim();return payload;" + "source": "void formatResults(StringBuilder message, String type, Map typeMap) {if (typeMap.empty == false) {message.append(' Node');if (typeMap.size() != 1) {message.append('s were');} else {message.append(' was');}message.append(' ').append(type).append(' [').append(typeMap.size()).append(']: ').append(typeMap.values().stream().collect(Collectors.joining(', ', '[', ']'))).append('.');}}ctx.vars.email_recipient = (ctx.payload.kibana_settings.hits.total > 0 && ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack != null) ? ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack.default_admin_email : null;def clusterState = ctx.payload.check.hits.hits[0]._source.cluster_state;def persistentUuidToName = [:];def latestNodes = clusterState.nodes;def ephemeralUuidToPersistentUuid = [:];def payload = ['timestamp': ctx.execution_time,'updated_timestamp': ctx.execution_time,'resolved_timestamp': ctx.execution_time,'metadata': ctx.metadata.xpack,'prefix': 'Elasticsearch cluster nodes have changed!','nodes': ['hash': clusterState.nodes_hash,'added': persistentUuidToName,'removed': [:],'restarted': [:]]];for (def latestNode : latestNodes.entrySet()) {persistentUuidToName[latestNode.key] = latestNode.value.name;ephemeralUuidToPersistentUuid[latestNode.value.ephemeral_id] = latestNode.key;}def previousNodes = ctx.payload.check.hits.hits[1]._source.cluster_state.nodes;def previousPersistentUuidToName = [:];for (def previousNode : previousNodes.entrySet()){if (persistentUuidToName.containsKey(previousNode.key) == false){payload.nodes.removed[previousNode.key] = previousNode.value.name;}else{if (ephemeralUuidToPersistentUuid.containsKey(previousNode.value.ephemeral_id) == false) {payload.nodes.restarted[previousNode.key] = persistentUuidToName[previousNode.key];}persistentUuidToName.remove(previousNode.key);}}StringBuilder message = new StringBuilder();formatResults(message, 'removed', payload.nodes.removed);formatResults(message, 'added', payload.nodes.added);formatResults(message, 'restarted', payload.nodes.restarted);payload.message = message.toString().trim();return payload;" } }, "actions": { diff --git a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_version_mismatch.json b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_version_mismatch.json index 1f3155107494b..6b8fa09408a2b 100644 --- a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_version_mismatch.json +++ b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_version_mismatch.json @@ -141,7 +141,7 @@ }, "transform": { "script": { - "source": "ctx.vars.email_recipient = (ctx.payload.kibana_settings.hits.total > 0) ? ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack.default_admin_email : null;ctx.vars.is_new = ctx.vars.fails_check && !ctx.vars.not_resolved;ctx.vars.is_resolved = !ctx.vars.fails_check && ctx.vars.not_resolved;def versionMessage = null;if (ctx.vars.fails_check) {def versions = new ArrayList(ctx.payload.check.hits.hits[0]._source.cluster_stats.nodes.versions);Collections.sort(versions);versionMessage = 'Versions: [' + String.join(', ', versions) + '].';}if (ctx.vars.not_resolved) {ctx.payload = ctx.payload.alert.hits.hits[0]._source;if (ctx.vars.fails_check) {ctx.payload.message = versionMessage;} else {ctx.payload.resolved_timestamp = ctx.execution_time;}} else {ctx.payload = [ 'timestamp': ctx.execution_time, 'prefix': 'This cluster is running with multiple versions of Elasticsearch.', 'message': versionMessage, 'metadata': ctx.metadata.xpack ];}ctx.payload.update_timestamp = ctx.execution_time;return ctx.payload;" + "source": "ctx.vars.email_recipient = (ctx.payload.kibana_settings.hits.total > 0 && ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack != null) ? ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack.default_admin_email : null;ctx.vars.is_new = ctx.vars.fails_check && !ctx.vars.not_resolved;ctx.vars.is_resolved = !ctx.vars.fails_check && ctx.vars.not_resolved;def versionMessage = null;if (ctx.vars.fails_check) {def versions = new ArrayList(ctx.payload.check.hits.hits[0]._source.cluster_stats.nodes.versions);Collections.sort(versions);versionMessage = 'Versions: [' + String.join(', ', versions) + '].';}if (ctx.vars.not_resolved) {ctx.payload = ctx.payload.alert.hits.hits[0]._source;if (ctx.vars.fails_check) {ctx.payload.message = versionMessage;} else {ctx.payload.resolved_timestamp = ctx.execution_time;}} else {ctx.payload = [ 'timestamp': ctx.execution_time, 'prefix': 'This cluster is running with multiple versions of Elasticsearch.', 'message': versionMessage, 'metadata': ctx.metadata.xpack ];}ctx.payload.update_timestamp = ctx.execution_time;return ctx.payload;" } }, "actions": { diff --git a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/kibana_version_mismatch.json b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/kibana_version_mismatch.json index d8fc4b64f5620..f3a0cae7efa5e 100644 --- a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/kibana_version_mismatch.json +++ b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/kibana_version_mismatch.json @@ -161,7 +161,7 @@ }, "transform": { "script": { - "source": "ctx.vars.email_recipient = (ctx.payload.kibana_settings.hits.total > 0) ? ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack.default_admin_email : null;ctx.vars.is_new = ctx.vars.fails_check && !ctx.vars.not_resolved;ctx.vars.is_resolved = !ctx.vars.fails_check && ctx.vars.not_resolved;def versionMessage = null;if (ctx.vars.fails_check) {versionMessage = 'Versions: [' + String.join(', ', ctx.vars.versions) + '].';}if (ctx.vars.not_resolved) {ctx.payload = ctx.payload.alert.hits.hits[0]._source;if (ctx.vars.fails_check) {ctx.payload.message = versionMessage;} else {ctx.payload.resolved_timestamp = ctx.execution_time;}} else {ctx.payload = [ 'timestamp': ctx.execution_time, 'prefix': 'This cluster is running with multiple versions of Kibana.', 'message': versionMessage, 'metadata': ctx.metadata.xpack ];}ctx.payload.update_timestamp = ctx.execution_time;return ctx.payload;" + "source": "ctx.vars.email_recipient = (ctx.payload.kibana_settings.hits.total > 0 && ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack != null) ? ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack.default_admin_email : null;ctx.vars.is_new = ctx.vars.fails_check && !ctx.vars.not_resolved;ctx.vars.is_resolved = !ctx.vars.fails_check && ctx.vars.not_resolved;def versionMessage = null;if (ctx.vars.fails_check) {versionMessage = 'Versions: [' + String.join(', ', ctx.vars.versions) + '].';}if (ctx.vars.not_resolved) {ctx.payload = ctx.payload.alert.hits.hits[0]._source;if (ctx.vars.fails_check) {ctx.payload.message = versionMessage;} else {ctx.payload.resolved_timestamp = ctx.execution_time;}} else {ctx.payload = [ 'timestamp': ctx.execution_time, 'prefix': 'This cluster is running with multiple versions of Kibana.', 'message': versionMessage, 'metadata': ctx.metadata.xpack ];}ctx.payload.update_timestamp = ctx.execution_time;return ctx.payload;" } }, "actions": { diff --git a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/logstash_version_mismatch.json b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/logstash_version_mismatch.json index 47de4a7ff5d49..c07b2a07fd3dd 100644 --- a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/logstash_version_mismatch.json +++ b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/logstash_version_mismatch.json @@ -161,7 +161,7 @@ }, "transform": { "script": { - "source": "ctx.vars.email_recipient = (ctx.payload.kibana_settings.hits.total > 0) ? ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack.default_admin_email : null;ctx.vars.is_new = ctx.vars.fails_check && !ctx.vars.not_resolved;ctx.vars.is_resolved = !ctx.vars.fails_check && ctx.vars.not_resolved;def versionMessage = null;if (ctx.vars.fails_check) {versionMessage = 'Versions: [' + String.join(', ', ctx.vars.versions) + '].';}if (ctx.vars.not_resolved) {ctx.payload = ctx.payload.alert.hits.hits[0]._source;if (ctx.vars.fails_check) {ctx.payload.message = versionMessage;} else {ctx.payload.resolved_timestamp = ctx.execution_time;}} else {ctx.payload = [ 'timestamp': ctx.execution_time, 'prefix': 'This cluster is running with multiple versions of Logstash.', 'message': versionMessage, 'metadata': ctx.metadata.xpack ];}ctx.payload.update_timestamp = ctx.execution_time;return ctx.payload;" + "source": "ctx.vars.email_recipient = (ctx.payload.kibana_settings.hits.total > 0 && ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack != null) ? ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack.default_admin_email : null;ctx.vars.is_new = ctx.vars.fails_check && !ctx.vars.not_resolved;ctx.vars.is_resolved = !ctx.vars.fails_check && ctx.vars.not_resolved;def versionMessage = null;if (ctx.vars.fails_check) {versionMessage = 'Versions: [' + String.join(', ', ctx.vars.versions) + '].';}if (ctx.vars.not_resolved) {ctx.payload = ctx.payload.alert.hits.hits[0]._source;if (ctx.vars.fails_check) {ctx.payload.message = versionMessage;} else {ctx.payload.resolved_timestamp = ctx.execution_time;}} else {ctx.payload = [ 'timestamp': ctx.execution_time, 'prefix': 'This cluster is running with multiple versions of Logstash.', 'message': versionMessage, 'metadata': ctx.metadata.xpack ];}ctx.payload.update_timestamp = ctx.execution_time;return ctx.payload;" } }, "actions": { diff --git a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/xpack_license_expiration.json b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/xpack_license_expiration.json index 1e7cd99cde554..38ce6ea86bf07 100644 --- a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/xpack_license_expiration.json +++ b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/xpack_license_expiration.json @@ -134,7 +134,7 @@ }, "transform": { "script": { - "source": "ctx.vars.email_recipient = (ctx.payload.kibana_settings.hits.total > 0) ? ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack.default_admin_email : null;ctx.vars.is_new = ctx.vars.fails_check && !ctx.vars.not_resolved;ctx.vars.is_resolved = !ctx.vars.fails_check && ctx.vars.not_resolved;def alertMessage = null;if (ctx.vars.fails_check) { alertMessage = 'Update your license.';} if (ctx.vars.not_resolved) { ctx.payload = ctx.payload.alert.hits.hits[0]._source;ctx.payload.metadata = ctx.metadata.xpack;if (ctx.vars.fails_check == false) { ctx.payload.resolved_timestamp = ctx.execution_time;} } else { ctx.payload = [ 'timestamp': ctx.execution_time, 'prefix': 'This cluster\\'s license is going to expire in {{#relativeTime}}metadata.time{{/relativeTime}} at {{#absoluteTime}}metadata.time{{/absoluteTime}}.', 'message': alertMessage, 'metadata': ctx.metadata.xpack ];} if (ctx.vars.fails_check) { ctx.payload.metadata.time = ctx.vars.expiry.toString();} ctx.payload.update_timestamp = ctx.execution_time;return ctx.payload;" + "source": "ctx.vars.email_recipient = (ctx.payload.kibana_settings.hits.total > 0 && ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack != null) ? ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack.default_admin_email : null;ctx.vars.is_new = ctx.vars.fails_check && !ctx.vars.not_resolved;ctx.vars.is_resolved = !ctx.vars.fails_check && ctx.vars.not_resolved;def alertMessage = null;if (ctx.vars.fails_check) { alertMessage = 'Update your license.';} if (ctx.vars.not_resolved) { ctx.payload = ctx.payload.alert.hits.hits[0]._source;ctx.payload.metadata = ctx.metadata.xpack;if (ctx.vars.fails_check == false) { ctx.payload.resolved_timestamp = ctx.execution_time;} } else { ctx.payload = [ 'timestamp': ctx.execution_time, 'prefix': 'This cluster\\'s license is going to expire in {{#relativeTime}}metadata.time{{/relativeTime}} at {{#absoluteTime}}metadata.time{{/absoluteTime}}.', 'message': alertMessage, 'metadata': ctx.metadata.xpack ];} if (ctx.vars.fails_check) { ctx.payload.metadata.time = ctx.vars.expiry.toString();} ctx.payload.update_timestamp = ctx.execution_time;return ctx.payload;" } }, "actions": { From 813b18989ef060967788ebfa1a336c0ee6915126 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Thu, 16 Aug 2018 17:36:58 -0700 Subject: [PATCH 22/42] For filters aggs, make sure that rewrites preserve other_bucket. (#32921) --- .../bucket/filter/FiltersAggregationBuilder.java | 5 ++++- .../search/aggregations/bucket/FiltersTests.java | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregationBuilder.java index e35bf376aae4d..810126e851251 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregationBuilder.java @@ -209,7 +209,10 @@ protected AggregationBuilder doRewrite(QueryRewriteContext queryShardContext) th } } if (changed) { - return new FiltersAggregationBuilder(getName(), rewrittenFilters, this.keyed); + FiltersAggregationBuilder rewritten = new FiltersAggregationBuilder(getName(), rewrittenFilters, this.keyed); + rewritten.otherBucket(otherBucket); + rewritten.otherBucketKey(otherBucketKey); + return rewritten; } else { return this; } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersTests.java index 4577986da270c..0e5656f020062 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersTests.java @@ -175,4 +175,18 @@ public void testRewrite() throws IOException { assertSame(rewritten, rewritten.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L))); } + + public void testRewritePreservesOtherBucket() throws IOException { + FiltersAggregationBuilder originalFilters = new FiltersAggregationBuilder("my-agg", new BoolQueryBuilder()); + originalFilters.otherBucket(randomBoolean()); + originalFilters.otherBucketKey(randomAlphaOfLength(10)); + + AggregationBuilder rewritten = originalFilters.rewrite(new QueryRewriteContext(xContentRegistry(), + null, null, () -> 0L)); + assertThat(rewritten, instanceOf(FiltersAggregationBuilder.class)); + + FiltersAggregationBuilder rewrittenFilters = (FiltersAggregationBuilder) rewritten; + assertEquals(originalFilters.otherBucket(), rewrittenFilters.otherBucket()); + assertEquals(originalFilters.otherBucketKey(), rewrittenFilters.otherBucketKey()); + } } From a13174fb3e711201afa9a0ff11419b3542b7343f Mon Sep 17 00:00:00 2001 From: Jay Modi Date: Thu, 16 Aug 2018 21:16:06 -0600 Subject: [PATCH 23/42] Security: remove put privilege API (#32879) This commit removes the put privilege API in favor of having a single API to create and update privileges. If we see the need to have an API like this in the future we can always add it back. --- .../PutPrivilegesRequestBuilder.java | 27 ---------- .../core/security/client/SecurityClient.java | 6 --- .../xpack/security/Security.java | 2 - .../privilege/RestPutPrivilegeAction.java | 49 ------------------- .../privilege/RestPutPrivilegesAction.java | 2 + .../PutPrivilegesRequestBuilderTests.java | 30 ------------ .../api/xpack.security.put_privilege.json | 33 ------------- .../api/xpack.security.put_privileges.json | 2 +- .../test/privileges/10_basic.yml | 42 ++++++++-------- .../authz/40_condtional_cluster_priv.yml | 40 +++++++++------ 10 files changed, 49 insertions(+), 184 deletions(-) delete mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegeAction.java delete mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_privilege.json diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilder.java index d52a4dd2bb6ed..fe75ad9d8e4f2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilder.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilder.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Objects; @@ -35,32 +34,6 @@ public PutPrivilegesRequestBuilder(ElasticsearchClient client, PutPrivilegesActi super(client, action, new PutPrivilegesRequest()); } - /** - * Populate the put privileges request using the given source, application name and privilege name - * The source must contain a single privilege object which matches the application and privilege names. - */ - public PutPrivilegesRequestBuilder source(String applicationName, String expectedName, - BytesReference source, XContentType xContentType) - throws IOException { - Objects.requireNonNull(xContentType); - // EMPTY is ok here because we never call namedObject - try (InputStream stream = source.streamInput(); - XContentParser parser = xContentType.xContent() - .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { - XContentParser.Token token = parser.currentToken(); - if (token == null) { - token = parser.nextToken(); - } - if (token == XContentParser.Token.START_OBJECT) { - final ApplicationPrivilegeDescriptor privilege = parsePrivilege(parser, applicationName, expectedName); - this.request.setPrivileges(Collections.singleton(privilege)); - } else { - throw new ElasticsearchParseException("expected an object but found {} instead", token); - } - } - return this; - } - ApplicationPrivilegeDescriptor parsePrivilege(XContentParser parser, String applicationName, String privilegeName) throws IOException { ApplicationPrivilegeDescriptor privilege = ApplicationPrivilegeDescriptor.parse(parser, applicationName, privilegeName, false); checkPrivilegeName(privilege, applicationName, privilegeName); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java index 3e4129b54e688..5edb17728d761 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java @@ -295,12 +295,6 @@ public GetPrivilegesRequestBuilder prepareGetPrivileges(String applicationName, return new GetPrivilegesRequestBuilder(client, GetPrivilegesAction.INSTANCE).application(applicationName).privileges(privileges); } - public PutPrivilegesRequestBuilder preparePutPrivilege(String applicationName, String privilegeName, - BytesReference bytesReference, XContentType xContentType) throws IOException { - return new PutPrivilegesRequestBuilder(client, PutPrivilegesAction.INSTANCE) - .source(applicationName, privilegeName, bytesReference, xContentType); - } - public PutPrivilegesRequestBuilder preparePutPrivileges(BytesReference bytesReference, XContentType xContentType) throws IOException { return new PutPrivilegesRequestBuilder(client, PutPrivilegesAction.INSTANCE).source(bytesReference, xContentType); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index acfe6437f4164..16d92a20e77f6 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -193,7 +193,6 @@ import org.elasticsearch.xpack.security.rest.action.oauth2.RestInvalidateTokenAction; import org.elasticsearch.xpack.security.rest.action.privilege.RestDeletePrivilegesAction; import org.elasticsearch.xpack.security.rest.action.privilege.RestGetPrivilegesAction; -import org.elasticsearch.xpack.security.rest.action.privilege.RestPutPrivilegeAction; import org.elasticsearch.xpack.security.rest.action.privilege.RestPutPrivilegesAction; import org.elasticsearch.xpack.security.rest.action.realm.RestClearRealmCacheAction; import org.elasticsearch.xpack.security.rest.action.role.RestClearRolesCacheAction; @@ -788,7 +787,6 @@ public List getRestHandlers(Settings settings, RestController restC new RestSamlInvalidateSessionAction(settings, restController, getLicenseState()), new RestGetPrivilegesAction(settings, restController, getLicenseState()), new RestPutPrivilegesAction(settings, restController, getLicenseState()), - new RestPutPrivilegeAction(settings, restController, getLicenseState()), new RestDeletePrivilegesAction(settings, restController, getLicenseState()) ); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegeAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegeAction.java deleted file mode 100644 index 6c3ef8e70fabf..0000000000000 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegeAction.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.security.rest.action.privilege; - -import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.rest.RestController; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesRequestBuilder; -import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege; -import org.elasticsearch.xpack.core.security.client.SecurityClient; -import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler; - -import java.io.IOException; - -import static org.elasticsearch.rest.RestRequest.Method.POST; -import static org.elasticsearch.rest.RestRequest.Method.PUT; - -/** - * Rest endpoint to add one or more {@link ApplicationPrivilege} objects to the security index - */ -public class RestPutPrivilegeAction extends SecurityBaseRestHandler { - - public RestPutPrivilegeAction(Settings settings, RestController controller, XPackLicenseState licenseState) { - super(settings, licenseState); - controller.registerHandler(PUT, "/_xpack/security/privilege/{application}/{privilege}", this); - controller.registerHandler(POST, "/_xpack/security/privilege/{application}/{privilege}", this); - } - - @Override - public String getName() { - return "xpack_security_put_privilege_action"; - } - - @Override - public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException { - final String application = request.param("application"); - final String privilege = request.param("privilege"); - PutPrivilegesRequestBuilder requestBuilder = new SecurityClient(client) - .preparePutPrivilege(application, privilege, request.requiredContent(), request.getXContentType()) - .setRefreshPolicy(request.param("refresh")); - - return RestPutPrivilegesAction.execute(requestBuilder); - } -} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegesAction.java index eb1104c9bc036..dc565e3f87339 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegesAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegesAction.java @@ -29,6 +29,7 @@ import java.util.Map; import static org.elasticsearch.rest.RestRequest.Method.POST; +import static org.elasticsearch.rest.RestRequest.Method.PUT; /** * Rest endpoint to add one or more {@link ApplicationPrivilege} objects to the security index @@ -37,6 +38,7 @@ public class RestPutPrivilegesAction extends SecurityBaseRestHandler { public RestPutPrivilegesAction(Settings settings, RestController controller, XPackLicenseState licenseState) { super(settings, licenseState); + controller.registerHandler(PUT, "/_xpack/security/privilege/", this); controller.registerHandler(POST, "/_xpack/security/privilege/", this); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilderTests.java index db0548c03ef30..2ece398d3d19f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilderTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilderTests.java @@ -52,36 +52,6 @@ private ApplicationPrivilegeDescriptor descriptor(String app, String name, Strin return new ApplicationPrivilegeDescriptor(app, name, Sets.newHashSet(actions), Collections.emptyMap()); } - public void testBuildRequestFromJsonObject() throws Exception { - final PutPrivilegesRequestBuilder builder = new PutPrivilegesRequestBuilder(null, PutPrivilegesAction.INSTANCE); - builder.source("foo", "read", new BytesArray( - "{ \"application\":\"foo\", \"name\":\"read\", \"actions\":[ \"data:/read/*\", \"admin:/read/*\" ] }" - ), XContentType.JSON); - final List privileges = builder.request().getPrivileges(); - assertThat(privileges, iterableWithSize(1)); - assertThat(privileges, contains(descriptor("foo", "read", "data:/read/*", "admin:/read/*"))); - } - - public void testPrivilegeNameValidationOfSingleElement() throws Exception { - final PutPrivilegesRequestBuilder builder = new PutPrivilegesRequestBuilder(null, PutPrivilegesAction.INSTANCE); - final IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> - builder.source("foo", "write", new BytesArray( - "{ \"application\":\"foo\", \"name\":\"read\", \"actions\":[ \"data:/read/*\", \"admin:/read/*\" ] }" - ), XContentType.JSON)); - assertThat(exception.getMessage(), containsString("write")); - assertThat(exception.getMessage(), containsString("read")); - } - - public void testApplicationNameValidationOfSingleElement() throws Exception { - final PutPrivilegesRequestBuilder builder = new PutPrivilegesRequestBuilder(null, PutPrivilegesAction.INSTANCE); - final IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> - builder.source("bar", "read", new BytesArray( - "{ \"application\":\"foo\", \"name\":\"read\", \"actions\":[ \"data:/read/*\", \"admin:/read/*\" ] }" - ), XContentType.JSON)); - assertThat(exception.getMessage(), containsString("foo")); - assertThat(exception.getMessage(), containsString("bar")); - } - public void testPrivilegeNameValidationOfMultipleElement() throws Exception { final PutPrivilegesRequestBuilder builder = new PutPrivilegesRequestBuilder(null, PutPrivilegesAction.INSTANCE); final IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_privilege.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_privilege.json deleted file mode 100644 index 3d453682c6431..0000000000000 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_privilege.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "xpack.security.put_privilege": { - "documentation": "TODO", - "methods": [ "POST", "PUT" ], - "url": { - "path": "/_xpack/security/privilege/{application}/{name}", - "paths": [ "/_xpack/security/privilege/{application}/{name}" ], - "parts": { - "application": { - "type" : "string", - "description" : "Application name", - "required" : true - }, - "name": { - "type" : "string", - "description" : "Privilege name", - "required" : true - } - }, - "params": { - "refresh": { - "type" : "enum", - "options": ["true", "false", "wait_for"], - "description" : "If `true` (the default) then refresh the affected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` then do nothing with refreshes." - } - } - }, - "body": { - "description" : "The privilege to add", - "required" : true - } - } -} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_privileges.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_privileges.json index 07eb541715810..312db3c9a1821 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_privileges.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_privileges.json @@ -1,7 +1,7 @@ { "xpack.security.put_privileges": { "documentation": "TODO", - "methods": [ "POST" ], + "methods": [ "PUT", "POST" ], "url": { "path": "/_xpack/security/privilege/", "paths": [ diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/10_basic.yml index e8dddf2153576..30fa3a8d07840 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/10_basic.yml @@ -30,24 +30,26 @@ teardown: ignore: 404 --- "Test put and get privileges": - # Single privilege, with names in URL + # Single privilege - do: - xpack.security.put_privilege: - application: app - name: p1 + xpack.security.put_privileges: body: > { - "application": "app", - "name": "p1", - "actions": [ "data:read/*" , "action:login" ], - "metadata": { - "key1" : "val1a", - "key2" : "val2a" + "app": { + "p1": { + "application": "app", + "name": "p1", + "actions": [ "data:read/*" , "action:login" ], + "metadata": { + "key1" : "val1a", + "key2" : "val2a" + } + } } } - match: { "app.p1" : { created: true } } - # Multiple privileges, no names in URL + # Multiple privileges - do: xpack.security.put_privileges: body: > @@ -84,18 +86,18 @@ teardown: - match: { "app.p3" : { created: true } } - match: { "app2.p1" : { created: true } } - # Update existing privilege, with names in URL + # Update existing privilege - do: - xpack.security.put_privilege: - application: app - name: p1 + xpack.security.put_privileges: body: > { - "application": "app", - "name": "p1", - "actions": [ "data:read/*" , "action:login" ], - "metadata": { - "key3" : "val3" + "app": { + "p1": { + "actions": [ "data:read/*" , "action:login" ], + "metadata": { + "key3" : "val3" + } + } } } - match: { "app.p1" : { created: false } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/40_condtional_cluster_priv.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/40_condtional_cluster_priv.yml index b3a1e22069083..a7d3fabd2a282 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/40_condtional_cluster_priv.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/40_condtional_cluster_priv.yml @@ -31,21 +31,25 @@ setup: } - do: - xpack.security.put_privilege: - application: app-allow - name: read + xpack.security.put_privileges: body: > { - "actions": [ "data:read/*" ] + "app-allow": { + "read": { + "actions": [ "data:read/*" ] + } + } } - do: - xpack.security.put_privilege: - application: app_deny - name: read + xpack.security.put_privileges: body: > { - "actions": [ "data:read/*" ] + "app-deny": { + "read": { + "actions": [ "data:read/*" ] + } + } } --- @@ -82,12 +86,14 @@ teardown: - do: headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user - xpack.security.put_privilege: - application: app - name: read + xpack.security.put_privileges: body: > { - "actions": [ "data:read/*" ] + "app": { + "read": { + "actions": [ "data:read/*" ] + } + } } - match: { "app.read" : { created: true } } @@ -112,12 +118,14 @@ teardown: "Test put application privileges when not allowed": - do: headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user - xpack.security.put_privilege: - application: app_deny - name: write + xpack.security.put_privileges: body: > { - "actions": [ "data:write/*" ] + "app_deny": { + "write": { + "actions": [ "data:write/*" ] + } + } } catch: forbidden From eb41dc46ff9592d8e3a7cef47783c03c289c41dc Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Fri, 17 Aug 2018 09:41:39 +0300 Subject: [PATCH 24/42] RFC: Test that example plugins build stand-alone (#32235) Add tests for build-tools to make sure example plugins build stand-alone using it. This will catch issues such as referencing files from the buildSrc directly, breaking external uses of build-tools. --- build.gradle | 8 + buildSrc/build.gradle | 13 ++ .../elasticsearch/gradle/BuildPlugin.groovy | 3 +- .../gradle/plugin/PluginBuildPlugin.groovy | 2 - .../plugin/PluginPropertiesExtension.groovy | 30 +++- .../gradle/plugin/PluginPropertiesTask.groovy | 1 - .../gradle/test/RestIntegTestTask.groovy | 11 +- .../gradle/BuildExamplePluginsIT.java | 164 ++++++++++++++++++ plugins/examples/custom-settings/build.gradle | 3 +- .../examples/painless-whitelist/build.gradle | 5 +- plugins/examples/rescore/build.gradle | 4 +- plugins/examples/rest-handler/build.gradle | 5 +- .../script-expert-scoring/build.gradle | 4 +- 13 files changed, 233 insertions(+), 20 deletions(-) create mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java diff --git a/build.gradle b/build.gradle index 805bb346adc84..a292d251ff6d2 100644 --- a/build.gradle +++ b/build.gradle @@ -77,8 +77,15 @@ subprojects { } } } + repositories { + maven { + name = 'localTest' + url = "${rootProject.buildDir}/local-test-repo" + } + } } } + plugins.withType(BuildPlugin).whenPluginAdded { project.licenseFile = project.rootProject.file('licenses/APACHE-LICENSE-2.0.txt') project.noticeFile = project.rootProject.file('NOTICE.txt') @@ -222,6 +229,7 @@ subprojects { "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level', "org.elasticsearch.client:test:${version}": ':client:test', "org.elasticsearch.client:transport:${version}": ':client:transport', + "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi', "org.elasticsearch.test:framework:${version}": ':test:framework', "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip', "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:archives:zip', diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 5775b2b6323f1..967c2e27ee8df 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -162,11 +162,24 @@ if (project != rootProject) { // it's fine as we run them as part of :buildSrc test.enabled = false task integTest(type: Test) { + // integration test requires the local testing repo for example plugin builds + dependsOn project.rootProject.allprojects.collect { + it.tasks.matching { it.name == 'publishNebulaPublicationToLocalTestRepository'} + } exclude "**/*Tests.class" include "**/*IT.class" testClassesDirs = sourceSets.test.output.classesDirs classpath = sourceSets.test.runtimeClasspath inputs.dir(file("src/testKit")) + // tell BuildExamplePluginsIT where to find the example plugins + systemProperty ( + 'test.build-tools.plugin.examples', + files( + project(':example-plugins').subprojects.collect { it.projectDir } + ).asPath, + ) + systemProperty 'test.local-test-repo-path', "${rootProject.buildDir}/local-test-repo" + systemProperty 'test.lucene-snapshot-revision', (versions.lucene =~ /\w+-snapshot-([a-z0-9]+)/)[0][1] } check.dependsOn(integTest) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 089b30ddf8b31..7e9ccef2d2120 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -554,7 +554,7 @@ class BuildPlugin implements Plugin { project.publishing { publications { nebula(MavenPublication) { - artifact project.tasks.shadowJar + artifacts = [ project.tasks.shadowJar ] artifactId = project.archivesBaseName /* * Configure the pom to include the "shadow" as compile dependencies @@ -584,7 +584,6 @@ class BuildPlugin implements Plugin { } } } - } /** Adds compiler settings to the project */ diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy index 9c1f7a15754b0..b44f24064921a 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -25,7 +25,6 @@ import org.elasticsearch.gradle.NoticeTask import org.elasticsearch.gradle.test.RestIntegTestTask import org.elasticsearch.gradle.test.RunTask import org.gradle.api.InvalidUserDataException -import org.gradle.api.JavaVersion import org.gradle.api.Project import org.gradle.api.Task import org.gradle.api.XmlProvider @@ -39,7 +38,6 @@ import java.nio.file.Path import java.nio.file.StandardCopyOption import java.util.regex.Matcher import java.util.regex.Pattern - /** * Encapsulates build configuration for an Elasticsearch plugin. */ diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy index 6cfe44c806833..c250d7695a832 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy @@ -20,6 +20,7 @@ package org.elasticsearch.gradle.plugin import org.gradle.api.Project import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFile /** * A container for plugin properties that will be written to the plugin descriptor, for easy @@ -55,18 +56,39 @@ class PluginPropertiesExtension { boolean requiresKeystore = false /** A license file that should be included in the built plugin zip. */ - @Input - File licenseFile = null + private File licenseFile = null /** * A notice file that should be included in the built plugin zip. This will be * extended with notices from the {@code licenses/} directory. */ - @Input - File noticeFile = null + private File noticeFile = null + + Project project = null PluginPropertiesExtension(Project project) { name = project.name version = project.version + this.project = project + } + + @InputFile + File getLicenseFile() { + return licenseFile + } + + void setLicenseFile(File licenseFile) { + project.ext.licenseFile = licenseFile + this.licenseFile = licenseFile + } + + @InputFile + File getNoticeFile() { + return noticeFile + } + + void setNoticeFile(File noticeFile) { + project.ext.noticeFile = noticeFile + this.noticeFile = noticeFile } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy index 8e913153f05ad..9588f77a71db7 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy @@ -23,7 +23,6 @@ import org.gradle.api.InvalidUserDataException import org.gradle.api.Task import org.gradle.api.tasks.Copy import org.gradle.api.tasks.OutputFile - /** * Creates a plugin descriptor. */ diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy index d2101c48aabdc..2838849981a1b 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy @@ -31,6 +31,7 @@ import org.gradle.api.provider.Provider import org.gradle.api.tasks.Copy import org.gradle.api.tasks.Input import org.gradle.api.tasks.TaskState +import org.gradle.plugins.ide.idea.IdeaPlugin import java.nio.charset.StandardCharsets import java.nio.file.Files @@ -243,10 +244,12 @@ public class RestIntegTestTask extends DefaultTask { } } } - project.idea { - module { - if (scopes.TEST != null) { - scopes.TEST.plus.add(project.configurations.restSpec) + if (project.plugins.hasPlugin(IdeaPlugin)) { + project.idea { + module { + if (scopes.TEST != null) { + scopes.TEST.plus.add(project.configurations.restSpec) + } } } } diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java new file mode 100644 index 0000000000000..9b63d6f45e06b --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java @@ -0,0 +1,164 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.apache.commons.io.FileUtils; +import org.elasticsearch.gradle.test.GradleIntegrationTestCase; +import org.gradle.testkit.runner.GradleRunner; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +public class BuildExamplePluginsIT extends GradleIntegrationTestCase { + + private static List EXAMPLE_PLUGINS = Collections.unmodifiableList( + Arrays.stream( + Objects.requireNonNull(System.getProperty("test.build-tools.plugin.examples")) + .split(File.pathSeparator) + ).map(File::new).collect(Collectors.toList()) + ); + + @Rule + public TemporaryFolder tmpDir = new TemporaryFolder(); + + public final File examplePlugin; + + public BuildExamplePluginsIT(File examplePlugin) { + this.examplePlugin = examplePlugin; + } + + @BeforeClass + public static void assertProjectsExist() { + assertEquals( + EXAMPLE_PLUGINS, + EXAMPLE_PLUGINS.stream().filter(File::exists).collect(Collectors.toList()) + ); + } + + @ParametersFactory + public static Iterable parameters() { + return EXAMPLE_PLUGINS + .stream() + .map(each -> new Object[] {each}) + .collect(Collectors.toList()); + } + + public void testCurrentExamplePlugin() throws IOException { + FileUtils.copyDirectory(examplePlugin, tmpDir.getRoot()); + // just get rid of deprecation warnings + Files.write( + getTempPath("settings.gradle"), + "enableFeaturePreview('STABLE_PUBLISHING')\n".getBytes(StandardCharsets.UTF_8) + ); + + adaptBuildScriptForTest(); + + Files.write( + tmpDir.newFile("NOTICE.txt").toPath(), + "dummy test notice".getBytes(StandardCharsets.UTF_8) + ); + + GradleRunner.create() + .withProjectDir(tmpDir.getRoot()) + .withArguments("clean", "check", "-s", "-i", "--warning-mode=all", "--scan") + .withPluginClasspath() + .build(); + } + + private void adaptBuildScriptForTest() throws IOException { + // Add the local repo as a build script URL so we can pull in build-tools and apply the plugin under test + // + is ok because we have no other repo and just want to pick up latest + writeBuildScript( + "buildscript {\n" + + " repositories {\n" + + " maven {\n" + + " url = '" + getLocalTestRepoPath() + "'\n" + + " }\n" + + " }\n" + + " dependencies {\n" + + " classpath \"org.elasticsearch.gradle:build-tools:+\"\n" + + " }\n" + + "}\n" + ); + // get the original file + Files.readAllLines(getTempPath("build.gradle"), StandardCharsets.UTF_8) + .stream() + .map(line -> line + "\n") + .forEach(this::writeBuildScript); + // Add a repositories section to be able to resolve dependencies + String luceneSnapshotRepo = ""; + String luceneSnapshotRevision = System.getProperty("test.lucene-snapshot-revision"); + if (luceneSnapshotRepo != null) { + luceneSnapshotRepo = " maven {\n" + + " url \"http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + luceneSnapshotRevision + "\"\n" + + " }\n"; + } + writeBuildScript("\n" + + "repositories {\n" + + " maven {\n" + + " url \"" + getLocalTestRepoPath() + "\"\n" + + " }\n" + + luceneSnapshotRepo + + "}\n" + ); + Files.delete(getTempPath("build.gradle")); + Files.move(getTempPath("build.gradle.new"), getTempPath("build.gradle")); + System.err.print("Generated build script is:"); + Files.readAllLines(getTempPath("build.gradle")).forEach(System.err::println); + } + + private Path getTempPath(String fileName) { + return new File(tmpDir.getRoot(), fileName).toPath(); + } + + private Path writeBuildScript(String script) { + try { + Path path = getTempPath("build.gradle.new"); + return Files.write( + path, + script.getBytes(StandardCharsets.UTF_8), + Files.exists(path) ? StandardOpenOption.APPEND : StandardOpenOption.CREATE_NEW + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private String getLocalTestRepoPath() { + String property = System.getProperty("test.local-test-repo-path"); + Objects.requireNonNull(property, "test.local-test-repo-path not passed to tests"); + File file = new File(property); + assertTrue("Expected " + property + " to exist, but it did not!", file.exists()); + return file.getAbsolutePath(); + } + +} diff --git a/plugins/examples/custom-settings/build.gradle b/plugins/examples/custom-settings/build.gradle index e0e728cec2427..3caf29c8513b5 100644 --- a/plugins/examples/custom-settings/build.gradle +++ b/plugins/examples/custom-settings/build.gradle @@ -16,13 +16,14 @@ * specific language governing permissions and limitations * under the License. */ - apply plugin: 'elasticsearch.esplugin' esplugin { name 'custom-settings' description 'An example plugin showing how to register custom settings' classname 'org.elasticsearch.example.customsettings.ExampleCustomSettingsPlugin' + licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt') + noticeFile rootProject.file('NOTICE.txt') } integTestCluster { diff --git a/plugins/examples/painless-whitelist/build.gradle b/plugins/examples/painless-whitelist/build.gradle index 12bbff8b0419e..71c0871fa83d6 100644 --- a/plugins/examples/painless-whitelist/build.gradle +++ b/plugins/examples/painless-whitelist/build.gradle @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - apply plugin: 'elasticsearch.esplugin' esplugin { @@ -24,10 +23,12 @@ esplugin { description 'An example whitelisting additional classes and methods in painless' classname 'org.elasticsearch.example.painlesswhitelist.MyWhitelistPlugin' extendedPlugins = ['lang-painless'] + licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt') + noticeFile rootProject.file('NOTICE.txt') } dependencies { - compileOnly project(':modules:lang-painless') + compileOnly "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${versions.elasticsearch}" } integTestCluster { diff --git a/plugins/examples/rescore/build.gradle b/plugins/examples/rescore/build.gradle index 4adeb0c721baf..cdecd760c81e8 100644 --- a/plugins/examples/rescore/build.gradle +++ b/plugins/examples/rescore/build.gradle @@ -16,11 +16,13 @@ * specific language governing permissions and limitations * under the License. */ - apply plugin: 'elasticsearch.esplugin' esplugin { name 'example-rescore' description 'An example plugin implementing rescore and verifying that plugins *can* implement rescore' classname 'org.elasticsearch.example.rescore.ExampleRescorePlugin' + licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt') + noticeFile rootProject.file('NOTICE.txt') } + diff --git a/plugins/examples/rest-handler/build.gradle b/plugins/examples/rest-handler/build.gradle index cfe84e6a45a93..eff2fd1b6c6e4 100644 --- a/plugins/examples/rest-handler/build.gradle +++ b/plugins/examples/rest-handler/build.gradle @@ -16,13 +16,14 @@ * specific language governing permissions and limitations * under the License. */ - apply plugin: 'elasticsearch.esplugin' esplugin { name 'rest-handler' description 'An example plugin showing how to register a REST handler' classname 'org.elasticsearch.example.resthandler.ExampleRestHandlerPlugin' + licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt') + noticeFile rootProject.file('NOTICE.txt') } // No unit tests in this example @@ -40,4 +41,4 @@ integTestCluster { } integTestRunner { systemProperty 'external.address', "${ -> exampleFixture.addressAndPort }" -} +} \ No newline at end of file diff --git a/plugins/examples/script-expert-scoring/build.gradle b/plugins/examples/script-expert-scoring/build.gradle index 7c602d9bc027d..e9da62acdcff4 100644 --- a/plugins/examples/script-expert-scoring/build.gradle +++ b/plugins/examples/script-expert-scoring/build.gradle @@ -16,13 +16,15 @@ * specific language governing permissions and limitations * under the License. */ - apply plugin: 'elasticsearch.esplugin' esplugin { name 'script-expert-scoring' description 'An example script engine to use low level Lucene internals for expert scoring' classname 'org.elasticsearch.example.expertscript.ExpertScriptPlugin' + licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt') + noticeFile rootProject.file('NOTICE.txt') } test.enabled = false + From 911aeedfafa87e79e7cfdfbfad90bcf95dd56208 Mon Sep 17 00:00:00 2001 From: JeffSaxeVA <42005772+JeffSaxeVA@users.noreply.github.com> Date: Fri, 17 Aug 2018 04:56:06 -0400 Subject: [PATCH 25/42] [DOCS] Add "remove a tag" script logic as an example (#32556) It took me quite a while of online searching and experimenting to realize the function-call asymmetry in the Add versus Remove from a list, like the "tags" list! I realize we cannot give examples for every single thing the user wants to do in Painless, but this is such a common use case (removing a tag from a single doc, or from a set of docs with Update-By-Query) that I believe it ought to be demonstrated immediately after the "add a tag" example. We have an example of removing an entire document field, but not removing one element of a list (a multi-valued field). Also, a minor grammar fix: I have added an apostrophe to the word "its" in the accompanying text of the example just above. --- docs/reference/docs/update.asciidoc | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc index 7ba7e2da63369..1cfc122bee402 100644 --- a/docs/reference/docs/update.asciidoc +++ b/docs/reference/docs/update.asciidoc @@ -47,7 +47,7 @@ POST test/_doc/1/_update // TEST[continued] We can add a tag to the list of tags (note, if the tag exists, it -will still add it, since its a list): +will still add it, since it's a list): [source,js] -------------------------------------------------- @@ -65,6 +65,28 @@ POST test/_doc/1/_update // CONSOLE // TEST[continued] +We can remove a tag from the list of tags. Note that the Painless function to +`remove` a tag takes as its parameter the array index of the element you wish +to remove, so you need a bit more logic to locate it while avoiding a runtime +error. Note that if the tag was present more than once in the list, this will +remove only one occurrence of it: + +[source,js] +-------------------------------------------------- +POST test/_doc/1/_update +{ + "script" : { + "source": "if (ctx._source.tags.contains(params.tag)) { ctx._source.tags.remove(ctx._source.tags.indexOf(params.tag)) }", + "lang": "painless", + "params" : { + "tag" : "blue" + } + } +} +-------------------------------------------------- +// CONSOLE +// TEST[continued] + In addition to `_source`, the following variables are available through the `ctx` map: `_index`, `_type`, `_id`, `_version`, `_routing` and `_now` (the current timestamp). @@ -172,7 +194,7 @@ the request was ignored. "_index": "test", "_type": "_doc", "_id": "1", - "_version": 6, + "_version": 7, "result": "noop" } -------------------------------------------------- From 40cfcf296d60f55f6ea3c781c4afcc9e67d43a5d Mon Sep 17 00:00:00 2001 From: JB Nizet Date: Fri, 17 Aug 2018 10:59:26 +0200 Subject: [PATCH 26/42] Fix allowed value for HighlighterBuilder encoder in javadocs (#32780) Relates to #32745 --- .../search/fetch/subphase/highlight/HighlightBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java index 049de439ac750..9483e76d072a8 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java @@ -220,7 +220,7 @@ public HighlightBuilder tagsSchema(String schemaName) { /** * Set encoder for the highlighting - * are {@code styled} and {@code default}. + * are {@code html} and {@code default}. * * @param encoder name */ From 518cd52709b01a800bdc9c79ba11e028266fb8ff Mon Sep 17 00:00:00 2001 From: markwalkom Date: Fri, 17 Aug 2018 19:03:09 +1000 Subject: [PATCH 27/42] [DOCS] Update getting-started.asciidoc (#29518) Highlighted that you can change shard counts using `_shrink` and `_split`. --- docs/reference/getting-started.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc index 9a0719873cec8..9b722515304bf 100755 --- a/docs/reference/getting-started.asciidoc +++ b/docs/reference/getting-started.asciidoc @@ -93,7 +93,8 @@ Replication is important for two primary reasons: To summarize, each index can be split into multiple shards. An index can also be replicated zero (meaning no replicas) or more times. Once replicated, each index will have primary shards (the original shards that were replicated from) and replica shards (the copies of the primary shards). -The number of shards and replicas can be defined per index at the time the index is created. After the index is created, you may change the number of replicas dynamically anytime but you cannot change the number of shards after-the-fact. + +The number of shards and replicas can be defined per index at the time the index is created. After the index is created, you may also change the number of replicas dynamically anytime. You can change the number of shards for an existing index using the {ref}/indices-shrink-index.html[`_shrink`] and {ref}/indices-split-index.html[`_split`] APIs, however this is not a trivial task and pre-planning for the correct number of shards is the optimal approach. By default, each index in Elasticsearch is allocated 5 primary shards and 1 replica which means that if you have at least two nodes in your cluster, your index will have 5 primary shards and another 5 replica shards (1 complete replica) for a total of 10 shards per index. From 9c0540a6386572256782d8a88c5738b1d9ca39a6 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Fri, 17 Aug 2018 10:57:00 +0100 Subject: [PATCH 28/42] HLRC: Move ML request converters into their own class (#32906) --- .../client/MLRequestConverters.java | 78 ++++++++++++++++ .../client/MachineLearningClient.java | 12 +-- .../client/RequestConverters.java | 45 +--------- .../client/MLRequestConvertersTests.java | 90 +++++++++++++++++++ .../client/RequestConvertersTests.java | 29 ------ .../xpack/ml/job/config/AnalysisConfig.java | 4 + .../xpack/ml/job/config/Detector.java | 4 + .../protocol/xpack/ml/job/config/Job.java | 6 +- 8 files changed, 188 insertions(+), 80 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java new file mode 100644 index 0000000000000..e26a4c629a0b0 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java @@ -0,0 +1,78 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.elasticsearch.client.RequestConverters.EndpointBuilder; +import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest; +import org.elasticsearch.protocol.xpack.ml.OpenJobRequest; +import org.elasticsearch.protocol.xpack.ml.PutJobRequest; + +import java.io.IOException; + +import static org.elasticsearch.client.RequestConverters.REQUEST_BODY_CONTENT_TYPE; +import static org.elasticsearch.client.RequestConverters.createEntity; + +final class MLRequestConverters { + + private MLRequestConverters() {} + + static Request putJob(PutJobRequest putJobRequest) throws IOException { + String endpoint = new EndpointBuilder() + .addPathPartAsIs("_xpack") + .addPathPartAsIs("ml") + .addPathPartAsIs("anomaly_detectors") + .addPathPart(putJobRequest.getJob().getId()) + .build(); + Request request = new Request(HttpPut.METHOD_NAME, endpoint); + request.setEntity(createEntity(putJobRequest, REQUEST_BODY_CONTENT_TYPE)); + return request; + } + + static Request openJob(OpenJobRequest openJobRequest) throws IOException { + String endpoint = new EndpointBuilder() + .addPathPartAsIs("_xpack") + .addPathPartAsIs("ml") + .addPathPartAsIs("anomaly_detectors") + .addPathPart(openJobRequest.getJobId()) + .addPathPartAsIs("_open") + .build(); + Request request = new Request(HttpPost.METHOD_NAME, endpoint); + request.setJsonEntity(openJobRequest.toString()); + return request; + } + + static Request deleteJob(DeleteJobRequest deleteJobRequest) { + String endpoint = new EndpointBuilder() + .addPathPartAsIs("_xpack") + .addPathPartAsIs("ml") + .addPathPartAsIs("anomaly_detectors") + .addPathPart(deleteJobRequest.getJobId()) + .build(); + Request request = new Request(HttpDelete.METHOD_NAME, endpoint); + + RequestConverters.Params params = new RequestConverters.Params(request); + params.putParam("force", Boolean.toString(deleteJobRequest.isForce())); + + return request; + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java index 2e7914e64abdb..32b6cd6cf2c67 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java @@ -57,7 +57,7 @@ public final class MachineLearningClient { */ public PutJobResponse putJob(PutJobRequest request, RequestOptions options) throws IOException { return restHighLevelClient.performRequestAndParseEntity(request, - RequestConverters::putMachineLearningJob, + MLRequestConverters::putJob, options, PutJobResponse::fromXContent, Collections.emptySet()); @@ -75,7 +75,7 @@ public PutJobResponse putJob(PutJobRequest request, RequestOptions options) thro */ public void putJobAsync(PutJobRequest request, RequestOptions options, ActionListener listener) { restHighLevelClient.performRequestAsyncAndParseEntity(request, - RequestConverters::putMachineLearningJob, + MLRequestConverters::putJob, options, PutJobResponse::fromXContent, listener, @@ -95,7 +95,7 @@ public void putJobAsync(PutJobRequest request, RequestOptions options, ActionLis */ public DeleteJobResponse deleteJob(DeleteJobRequest request, RequestOptions options) throws IOException { return restHighLevelClient.performRequestAndParseEntity(request, - RequestConverters::deleteMachineLearningJob, + MLRequestConverters::deleteJob, options, DeleteJobResponse::fromXContent, Collections.emptySet()); @@ -113,7 +113,7 @@ public DeleteJobResponse deleteJob(DeleteJobRequest request, RequestOptions opti */ public void deleteJobAsync(DeleteJobRequest request, RequestOptions options, ActionListener listener) { restHighLevelClient.performRequestAsyncAndParseEntity(request, - RequestConverters::deleteMachineLearningJob, + MLRequestConverters::deleteJob, options, DeleteJobResponse::fromXContent, listener, @@ -138,7 +138,7 @@ public void deleteJobAsync(DeleteJobRequest request, RequestOptions options, Act */ public OpenJobResponse openJob(OpenJobRequest request, RequestOptions options) throws IOException { return restHighLevelClient.performRequestAndParseEntity(request, - RequestConverters::machineLearningOpenJob, + MLRequestConverters::openJob, options, OpenJobResponse::fromXContent, Collections.emptySet()); @@ -160,7 +160,7 @@ public OpenJobResponse openJob(OpenJobRequest request, RequestOptions options) t */ public void openJobAsync(OpenJobRequest request, RequestOptions options, ActionListener listener) { restHighLevelClient.performRequestAsyncAndParseEntity(request, - RequestConverters::machineLearningOpenJob, + MLRequestConverters::openJob, options, OpenJobResponse::fromXContent, listener, diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 6ae673a36910f..989ad30ec72eb 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -112,9 +112,6 @@ import org.elasticsearch.protocol.xpack.license.GetLicenseRequest; import org.elasticsearch.protocol.xpack.license.PutLicenseRequest; import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest; -import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest; -import org.elasticsearch.protocol.xpack.ml.OpenJobRequest; -import org.elasticsearch.protocol.xpack.ml.PutJobRequest; import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest; import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest; import org.elasticsearch.rest.action.search.RestSearchAction; @@ -1211,46 +1208,6 @@ static Request deleteLicense(DeleteLicenseRequest deleteLicenseRequest) { return request; } - static Request putMachineLearningJob(PutJobRequest putJobRequest) throws IOException { - String endpoint = new EndpointBuilder() - .addPathPartAsIs("_xpack") - .addPathPartAsIs("ml") - .addPathPartAsIs("anomaly_detectors") - .addPathPart(putJobRequest.getJob().getId()) - .build(); - Request request = new Request(HttpPut.METHOD_NAME, endpoint); - request.setEntity(createEntity(putJobRequest, REQUEST_BODY_CONTENT_TYPE)); - return request; - } - - static Request deleteMachineLearningJob(DeleteJobRequest deleteJobRequest) { - String endpoint = new EndpointBuilder() - .addPathPartAsIs("_xpack") - .addPathPartAsIs("ml") - .addPathPartAsIs("anomaly_detectors") - .addPathPart(deleteJobRequest.getJobId()) - .build(); - Request request = new Request(HttpDelete.METHOD_NAME, endpoint); - - Params params = new Params(request); - params.putParam("force", Boolean.toString(deleteJobRequest.isForce())); - - return request; - } - - static Request machineLearningOpenJob(OpenJobRequest openJobRequest) throws IOException { - String endpoint = new EndpointBuilder() - .addPathPartAsIs("_xpack") - .addPathPartAsIs("ml") - .addPathPartAsIs("anomaly_detectors") - .addPathPart(openJobRequest.getJobId()) - .addPathPartAsIs("_open") - .build(); - Request request = new Request(HttpPost.METHOD_NAME, endpoint); - request.setJsonEntity(openJobRequest.toString()); - return request; - } - static Request getMigrationAssistance(IndexUpgradeInfoRequest indexUpgradeInfoRequest) { EndpointBuilder endpointBuilder = new EndpointBuilder() .addPathPartAsIs("_xpack/migration/assistance") @@ -1262,7 +1219,7 @@ static Request getMigrationAssistance(IndexUpgradeInfoRequest indexUpgradeInfoRe return request; } - private static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException { + static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException { BytesRef source = XContentHelper.toXContent(toXContent, xContentType, false).toBytesRef(); return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType)); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java new file mode 100644 index 0000000000000..43a41960e003c --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java @@ -0,0 +1,90 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpPost; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest; +import org.elasticsearch.protocol.xpack.ml.OpenJobRequest; +import org.elasticsearch.protocol.xpack.ml.PutJobRequest; +import org.elasticsearch.protocol.xpack.ml.job.config.AnalysisConfig; +import org.elasticsearch.protocol.xpack.ml.job.config.Detector; +import org.elasticsearch.protocol.xpack.ml.job.config.Job; +import org.elasticsearch.test.ESTestCase; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Collections; + +import static org.hamcrest.Matchers.equalTo; + +public class MLRequestConvertersTests extends ESTestCase { + + public void testPutJob() throws IOException { + Job job = createValidJob("foo"); + PutJobRequest putJobRequest = new PutJobRequest(job); + + Request request = MLRequestConverters.putJob(putJobRequest); + + assertThat(request.getEndpoint(), equalTo("/_xpack/ml/anomaly_detectors/foo")); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, request.getEntity().getContent())) { + Job parsedJob = Job.PARSER.apply(parser, null).build(); + assertThat(parsedJob, equalTo(job)); + } + } + + public void testOpenJob() throws Exception { + String jobId = "some-job-id"; + OpenJobRequest openJobRequest = new OpenJobRequest(jobId); + openJobRequest.setTimeout(TimeValue.timeValueMinutes(10)); + + Request request = MLRequestConverters.openJob(openJobRequest); + assertEquals(HttpPost.METHOD_NAME, request.getMethod()); + assertEquals("/_xpack/ml/anomaly_detectors/" + jobId + "/_open", request.getEndpoint()); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + request.getEntity().writeTo(bos); + assertEquals(bos.toString("UTF-8"), "{\"job_id\":\""+ jobId +"\",\"timeout\":\"10m\"}"); + } + + public void testDeleteJob() { + String jobId = randomAlphaOfLength(10); + DeleteJobRequest deleteJobRequest = new DeleteJobRequest(jobId); + + Request request = MLRequestConverters.deleteJob(deleteJobRequest); + assertEquals(HttpDelete.METHOD_NAME, request.getMethod()); + assertEquals("/_xpack/ml/anomaly_detectors/" + jobId, request.getEndpoint()); + assertEquals(Boolean.toString(false), request.getParameters().get("force")); + + deleteJobRequest.setForce(true); + request = MLRequestConverters.deleteJob(deleteJobRequest); + assertEquals(Boolean.toString(true), request.getParameters().get("force")); + } + + private static Job createValidJob(String jobId) { + AnalysisConfig.Builder analysisConfig = AnalysisConfig.builder(Collections.singletonList( + Detector.builder().setFunction("count").build())); + Job.Builder jobBuilder = Job.builder(jobId); + jobBuilder.setAnalysisConfig(analysisConfig); + return jobBuilder.build(); + } +} \ No newline at end of file diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index b4fd6c914b39c..aad238e9e3655 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -128,8 +128,6 @@ import org.elasticsearch.index.rankeval.RestRankEvalAction; import org.elasticsearch.protocol.xpack.XPackInfoRequest; import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest; -import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest; -import org.elasticsearch.protocol.xpack.ml.OpenJobRequest; import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest; import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest; import org.elasticsearch.repositories.fs.FsRepository; @@ -2646,33 +2644,6 @@ public void testXPackDeleteWatch() { assertThat(request.getEntity(), nullValue()); } - public void testDeleteMachineLearningJob() { - String jobId = randomAlphaOfLength(10); - DeleteJobRequest deleteJobRequest = new DeleteJobRequest(jobId); - - Request request = RequestConverters.deleteMachineLearningJob(deleteJobRequest); - assertEquals(HttpDelete.METHOD_NAME, request.getMethod()); - assertEquals("/_xpack/ml/anomaly_detectors/" + jobId, request.getEndpoint()); - assertEquals(Boolean.toString(false), request.getParameters().get("force")); - - deleteJobRequest.setForce(true); - request = RequestConverters.deleteMachineLearningJob(deleteJobRequest); - assertEquals(Boolean.toString(true), request.getParameters().get("force")); - } - - public void testPostMachineLearningOpenJob() throws Exception { - String jobId = "some-job-id"; - OpenJobRequest openJobRequest = new OpenJobRequest(jobId); - openJobRequest.setTimeout(TimeValue.timeValueMinutes(10)); - - Request request = RequestConverters.machineLearningOpenJob(openJobRequest); - assertEquals(HttpPost.METHOD_NAME, request.getMethod()); - assertEquals("/_xpack/ml/anomaly_detectors/" + jobId + "/_open", request.getEndpoint()); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - request.getEntity().writeTo(bos); - assertEquals(bos.toString("UTF-8"), "{\"job_id\":\""+ jobId +"\",\"timeout\":\"10m\"}"); - } - /** * Randomize the {@link FetchSourceContext} request parameters. */ diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/AnalysisConfig.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/AnalysisConfig.java index 00fa1bdd47fed..7baaae52a8bfa 100644 --- a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/AnalysisConfig.java +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/AnalysisConfig.java @@ -300,6 +300,10 @@ public int hashCode() { multivariateByFields); } + public static Builder builder(List detectors) { + return new Builder(detectors); + } + public static class Builder { private List detectors; diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Detector.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Detector.java index 3274b03877f14..042d48b700688 100644 --- a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Detector.java +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Detector.java @@ -265,6 +265,10 @@ public int hashCode() { excludeFrequent, rules, detectorIndex); } + public static Builder builder() { + return new Builder(); + } + public static class Builder { private String detectorDescription; diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Job.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Job.java index 6bc1be3b56384..59840cfec2ae0 100644 --- a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Job.java +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Job.java @@ -412,6 +412,10 @@ public final String toString() { return Strings.toString(this); } + public static Builder builder(String id) { + return new Builder(id); + } + public static class Builder { private String id; @@ -435,7 +439,7 @@ public static class Builder { private String resultsIndexName; private boolean deleted; - public Builder() { + private Builder() { } public Builder(String id) { From 7a2d70072be9aff4b29733c01a37d83c4f4f2059 Mon Sep 17 00:00:00 2001 From: Ioannis Kakavas Date: Fri, 17 Aug 2018 14:06:24 +0300 Subject: [PATCH 29/42] Enable FIPS140LicenseBootstrapCheck (#32903) This commit ensures that xpack.security.fips_mode.enabled: true cannot be set in a node that doesn't have the appropriate license. --- .../main/java/org/elasticsearch/xpack/security/Security.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 16d92a20e77f6..f99d7b8e729a4 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -310,7 +310,8 @@ public Security(Settings settings, final Path configPath) { new TLSLicenseBootstrapCheck(), new FIPS140SecureSettingsBootstrapCheck(settings, env), new FIPS140JKSKeystoreBootstrapCheck(settings), - new FIPS140PasswordHashingAlgorithmBootstrapCheck(settings))); + new FIPS140PasswordHashingAlgorithmBootstrapCheck(settings), + new FIPS140LicenseBootstrapCheck(XPackSettings.FIPS_MODE_ENABLED.get(settings)))); checks.addAll(InternalRealms.getBootstrapChecks(settings, env)); this.bootstrapChecks = Collections.unmodifiableList(checks); Automatons.updateMaxDeterminizedStates(settings); From ba38b2efe91a348d29d8c8f55195a5a0187d117c Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Fri, 17 Aug 2018 07:21:17 -0500 Subject: [PATCH 30/42] [ML] fix updating opened jobs scheduled events (#31651) (#32881) * ML: fix updating opened jobs scheduled events (#31651) * Adding UpdateParamsTests license header * Adding integration test and addressing PR comments * addressing test and job names --- .../xpack/core/ml/job/config/JobUpdate.java | 2 +- .../core/ml/job/config/JobUpdateTests.java | 2 + .../job/process/autodetect/UpdateParams.java | 1 + .../process/autodetect/UpdateParamsTests.java | 45 ++++++++++ .../ml/integration/ScheduledEventsIT.java | 83 ++++++++++++++++++- 5 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/UpdateParamsTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java index 89bec3d113867..e77bb0b94919f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java @@ -237,7 +237,7 @@ public Version getJobVersion() { } public boolean isAutodetectProcessUpdate() { - return modelPlotConfig != null || detectorUpdates != null; + return modelPlotConfig != null || detectorUpdates != null || groups != null; } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java index 75b47a0c43615..37bec1196e756 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java @@ -271,6 +271,8 @@ public void testIsAutodetectProcessUpdate() { assertTrue(update.isAutodetectProcessUpdate()); update = new JobUpdate.Builder("foo").setDetectorUpdates(Collections.singletonList(mock(JobUpdate.DetectorUpdate.class))).build(); assertTrue(update.isAutodetectProcessUpdate()); + update = new JobUpdate.Builder("foo").setGroups(Arrays.asList("bar")).build(); + assertTrue(update.isAutodetectProcessUpdate()); } public void testUpdateAnalysisLimitWithValueGreaterThanMax() { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/UpdateParams.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/UpdateParams.java index 127fb18e5fff4..2d338890f9fa4 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/UpdateParams.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/UpdateParams.java @@ -66,6 +66,7 @@ public static UpdateParams fromJobUpdate(JobUpdate jobUpdate) { return new Builder(jobUpdate.getJobId()) .modelPlotConfig(jobUpdate.getModelPlotConfig()) .detectorUpdates(jobUpdate.getDetectorUpdates()) + .updateScheduledEvents(jobUpdate.getGroups() != null) .build(); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/UpdateParamsTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/UpdateParamsTests.java new file mode 100644 index 0000000000000..2683c1131f5bf --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/UpdateParamsTests.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; +import org.elasticsearch.xpack.core.ml.job.config.JobUpdate; +import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfig; +import org.elasticsearch.xpack.core.ml.job.config.Operator; +import org.elasticsearch.xpack.core.ml.job.config.RuleCondition; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + + +public class UpdateParamsTests extends ESTestCase { + + public void testFromJobUpdate() { + String jobId = "foo"; + DetectionRule rule = new DetectionRule.Builder(Arrays.asList( + new RuleCondition(RuleCondition.AppliesTo.ACTUAL, + Operator.GT, 1.0))).build(); + List rules = Arrays.asList(rule); + List detectorUpdates = Collections.singletonList( + new JobUpdate.DetectorUpdate(2, null, rules)); + JobUpdate.Builder updateBuilder = new JobUpdate.Builder(jobId) + .setModelPlotConfig(new ModelPlotConfig()) + .setDetectorUpdates(detectorUpdates); + + UpdateParams params = UpdateParams.fromJobUpdate(updateBuilder.build()); + + assertFalse(params.isUpdateScheduledEvents()); + assertEquals(params.getDetectorUpdates(), updateBuilder.build().getDetectorUpdates()); + assertEquals(params.getModelPlotConfig(), updateBuilder.build().getModelPlotConfig()); + + params = UpdateParams.fromJobUpdate(updateBuilder.setGroups(Arrays.asList("bar")).build()); + + assertTrue(params.isUpdateScheduledEvents()); + } + +} diff --git a/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ScheduledEventsIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ScheduledEventsIT.java index 6703e4ef2365b..fb261908e2c10 100644 --- a/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ScheduledEventsIT.java +++ b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ScheduledEventsIT.java @@ -12,11 +12,13 @@ import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.xpack.core.ml.action.GetBucketsAction; import org.elasticsearch.xpack.core.ml.action.GetRecordsAction; +import org.elasticsearch.xpack.core.ml.action.UpdateJobAction; import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.config.JobUpdate; import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord; import org.elasticsearch.xpack.core.ml.job.results.Bucket; import org.junit.After; @@ -193,9 +195,9 @@ public void testScheduledEventWithInterimResults() throws IOException { /** * Test an open job picks up changes to scheduled events/calendars */ - public void testOnlineUpdate() throws Exception { + public void testAddEventsToOpenJob() throws Exception { TimeValue bucketSpan = TimeValue.timeValueMinutes(30); - Job.Builder job = createJob("scheduled-events-online-update", bucketSpan); + Job.Builder job = createJob("scheduled-events-add-events-to-open-job", bucketSpan); long startTime = 1514764800000L; final int bucketCount = 5; @@ -209,7 +211,7 @@ public void testOnlineUpdate() throws Exception { // Now create a calendar and events for the job while it is open String calendarId = "test-calendar-online-update"; - putCalendar(calendarId, Collections.singletonList(job.getId()), "testOnlineUpdate calendar"); + putCalendar(calendarId, Collections.singletonList(job.getId()), "testAddEventsToOpenJob calendar"); List events = new ArrayList<>(); long eventStartTime = startTime + (bucketCount + 1) * bucketSpan.millis(); @@ -257,6 +259,81 @@ public void testOnlineUpdate() throws Exception { assertEquals(0, buckets.get(8).getScheduledEvents().size()); } + /** + * An open job that later gets added to a calendar, should take the scheduled events into account + */ + public void testAddOpenedJobToGroupWithCalendar() throws Exception { + TimeValue bucketSpan = TimeValue.timeValueMinutes(30); + String groupName = "opened-calendar-job-group"; + Job.Builder job = createJob("scheduled-events-add-opened-job-to-group-with-calendar", bucketSpan); + + long startTime = 1514764800000L; + final int bucketCount = 5; + + // Open the job + openJob(job.getId()); + + // write some buckets of data + postData(job.getId(), generateData(startTime, bucketSpan, bucketCount, bucketIndex -> randomIntBetween(100, 200)) + .stream().collect(Collectors.joining())); + + String calendarId = "test-calendar-open-job-update"; + + // Create a new calendar referencing groupName + putCalendar(calendarId, Collections.singletonList(groupName), "testAddOpenedJobToGroupWithCalendar calendar"); + + // Put events in the calendar + List events = new ArrayList<>(); + long eventStartTime = startTime + (bucketCount + 1) * bucketSpan.millis(); + long eventEndTime = eventStartTime + (long)(1.5 * bucketSpan.millis()); + events.add(new ScheduledEvent.Builder().description("Some Event") + .startTime(ZonedDateTime.ofInstant(Instant.ofEpochMilli(eventStartTime), ZoneOffset.UTC)) + .endTime(ZonedDateTime.ofInstant(Instant.ofEpochMilli(eventEndTime), ZoneOffset.UTC)) + .calendarId(calendarId).build()); + + postScheduledEvents(calendarId, events); + + // Update the job to be a member of the group + UpdateJobAction.Request jobUpdateRequest = new UpdateJobAction.Request(job.getId(), + new JobUpdate.Builder(job.getId()).setGroups(Collections.singletonList(groupName)).build()); + client().execute(UpdateJobAction.INSTANCE, jobUpdateRequest).actionGet(); + + // Wait until the notification that the job was updated is indexed + assertBusy(() -> { + SearchResponse searchResponse = client().prepareSearch(".ml-notifications") + .setSize(1) + .addSort("timestamp", SortOrder.DESC) + .setQuery(QueryBuilders.boolQuery() + .filter(QueryBuilders.termQuery("job_id", job.getId())) + .filter(QueryBuilders.termQuery("level", "info")) + ).get(); + SearchHit[] hits = searchResponse.getHits().getHits(); + assertThat(hits.length, equalTo(1)); + assertThat(hits[0].getSourceAsMap().get("message"), equalTo("Job updated: [groups]")); + }); + + // write some more buckets of data that cover the scheduled event period + postData(job.getId(), generateData(startTime + bucketCount * bucketSpan.millis(), bucketSpan, 5, + bucketIndex -> randomIntBetween(100, 200)) + .stream().collect(Collectors.joining())); + // and close + closeJob(job.getId()); + + GetBucketsAction.Request getBucketsRequest = new GetBucketsAction.Request(job.getId()); + List buckets = getBuckets(getBucketsRequest); + + // the first 6 buckets have no events + for (int i=0; i<=bucketCount; i++) { + assertEquals(0, buckets.get(i).getScheduledEvents().size()); + } + // 7th and 8th buckets have the event but the last one does not + assertEquals(1, buckets.get(6).getScheduledEvents().size()); + assertEquals("Some Event", buckets.get(6).getScheduledEvents().get(0)); + assertEquals(1, buckets.get(7).getScheduledEvents().size()); + assertEquals("Some Event", buckets.get(7).getScheduledEvents().get(0)); + assertEquals(0, buckets.get(8).getScheduledEvents().size()); + } + private Job.Builder createJob(String jobId, TimeValue bucketSpan) { Detector.Builder detector = new Detector.Builder("count", null); AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build())); From b3a60e3a23663ab309de2647b1762dfe14720560 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 17 Aug 2018 12:36:45 +0200 Subject: [PATCH 31/42] Remove assertion in testDocStats on deletedDocs counter (#32914) testDocStats test is flaky and sometimes it's failing on jenkins and failure is not reproducible locally. The reason for this failure is in timing. If the number of deleted documents is greater than 33% of inserted documents, Lucene will schedule segments to merge if TieredMergePolicy is used (it's not the case for LogMergePolicy, but ES is only using TieredMergePolicy). If this merge is performed before stats are retrieved - we will get 0 for "deleted" counter. So basically this counter could be either 0 or numOfDeletedDocs at this point, but this is the too loose assertion and we decided to remove it at all. Closes #32766 --- .../java/org/elasticsearch/index/shard/IndexShardTests.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 79c6353cfbf8a..1cb9cb7c3d47f 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -2389,8 +2389,7 @@ public void testRecoverFromLocalShard() throws IOException { closeShards(sourceShard, targetShard); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32766") - public void testDocStats() throws IOException { + public void testDocStats() throws IOException, InterruptedException { IndexShard indexShard = null; try { indexShard = newStartedShard(); @@ -2439,8 +2438,6 @@ public void testDocStats() throws IOException { assertTrue(searcher.reader().numDocs() <= docStats.getCount()); } assertThat(docStats.getCount(), equalTo(numDocs)); - // Lucene will delete a segment if all docs are deleted from it; this means that we lose the deletes when deleting all docs - assertThat(docStats.getDeleted(), equalTo(numDocsToDelete == numDocs ? 0 : numDocsToDelete)); } // merge them away From cbbb2b78d37bbcc5276c093383bb477cbf991340 Mon Sep 17 00:00:00 2001 From: Ioannis Kakavas Date: Fri, 17 Aug 2018 18:23:13 +0300 Subject: [PATCH 32/42] [TEST] Run pre 6.4 nodes in non-FIPS JVMs (#32901) Elasticsearch versions earlier than 6.4.0 cannot properly run in a FIPS 140 JVM. This commit ensures that we use a non-FIPS JVM for nodes that we spin up in BWC tests even when we're testing FIPS. --- .../groovy/org/elasticsearch/gradle/test/NodeInfo.groovy | 6 ++++++ .../qa/full-cluster-restart/with-system-key/build.gradle | 8 -------- x-pack/qa/rolling-upgrade/with-system-key/build.gradle | 8 -------- 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy index 5ccc87dc7fadc..41c8f3820c04e 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy @@ -177,6 +177,12 @@ class NodeInfo { javaVersion = 8 } else if (nodeVersion.onOrAfter("6.2.0") && nodeVersion.before("6.3.0")) { javaVersion = 9 + } else if (project.inFipsJvm && nodeVersion.onOrAfter("6.3.0") && nodeVersion.before("6.4.0")) { + /* + * Elasticsearch versions before 6.4.0 cannot be run in a FIPS-140 JVM. If we're running + * bwc tests in a FIPS-140 JVM, ensure that the pre v6.4.0 nodes use a Java 10 JVM instead. + */ + javaVersion = 10 } args.addAll("-E", "node.portsfile=true") diff --git a/x-pack/qa/full-cluster-restart/with-system-key/build.gradle b/x-pack/qa/full-cluster-restart/with-system-key/build.gradle index 928280b6584bd..e69de29bb2d1d 100644 --- a/x-pack/qa/full-cluster-restart/with-system-key/build.gradle +++ b/x-pack/qa/full-cluster-restart/with-system-key/build.gradle @@ -1,8 +0,0 @@ -import org.elasticsearch.gradle.test.RestIntegTestTask - -// Skip test on FIPS FIXME https://github.com/elastic/elasticsearch/issues/32737 -if (project.inFipsJvm) { - tasks.withType(RestIntegTestTask) { - enabled = false - } -} diff --git a/x-pack/qa/rolling-upgrade/with-system-key/build.gradle b/x-pack/qa/rolling-upgrade/with-system-key/build.gradle index 928280b6584bd..e69de29bb2d1d 100644 --- a/x-pack/qa/rolling-upgrade/with-system-key/build.gradle +++ b/x-pack/qa/rolling-upgrade/with-system-key/build.gradle @@ -1,8 +0,0 @@ -import org.elasticsearch.gradle.test.RestIntegTestTask - -// Skip test on FIPS FIXME https://github.com/elastic/elasticsearch/issues/32737 -if (project.inFipsJvm) { - tasks.withType(RestIntegTestTask) { - enabled = false - } -} From 2f20ae259738a0557c722bb430ab1113c7e94a12 Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Fri, 17 Aug 2018 09:18:08 -0700 Subject: [PATCH 33/42] [DOCS] Splits the roles API documentation into multiple pages (#32794) --- x-pack/docs/build.gradle | 14 ++ x-pack/docs/en/rest-api/security.asciidoc | 10 +- .../security/clear-roles-cache.asciidoc | 39 ++++ .../rest-api/security/create-roles.asciidoc | 103 +++++++++ .../rest-api/security/delete-roles.asciidoc | 52 +++++ .../en/rest-api/security/get-roles.asciidoc | 86 ++++++++ .../docs/en/rest-api/security/roles.asciidoc | 206 +----------------- .../xpack.security.clear_cached_roles.json | 2 +- .../api/xpack.security.delete_role.json | 2 +- .../api/xpack.security.get_role.json | 2 +- .../api/xpack.security.put_role.json | 2 +- 11 files changed, 312 insertions(+), 206 deletions(-) create mode 100644 x-pack/docs/en/rest-api/security/clear-roles-cache.asciidoc create mode 100644 x-pack/docs/en/rest-api/security/create-roles.asciidoc create mode 100644 x-pack/docs/en/rest-api/security/delete-roles.asciidoc create mode 100644 x-pack/docs/en/rest-api/security/get-roles.asciidoc diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle index 4e989b4b67dd3..ea60ec2ca49a8 100644 --- a/x-pack/docs/build.gradle +++ b/x-pack/docs/build.gradle @@ -723,3 +723,17 @@ setups['sensor_prefab_data'] = ''' {"node.terms.value":"c","temperature.sum.value":202.0,"temperature.max.value":202.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":202.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":4.0,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516294800000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} ''' +setups['admin_role'] = ''' + - do: + xpack.security.put_role: + name: "my_admin_role" + body: > + { + "cluster": ["all"], + "indices": [ + {"names": ["index1", "index2" ], "privileges": ["all"], "field_security" : {"grant" : [ "title", "body" ]}} + ], + "run_as": [ "other_user" ], + "metadata" : {"version": 1} + } +''' diff --git a/x-pack/docs/en/rest-api/security.asciidoc b/x-pack/docs/en/rest-api/security.asciidoc index 227e343192a50..4d59cb22daa06 100644 --- a/x-pack/docs/en/rest-api/security.asciidoc +++ b/x-pack/docs/en/rest-api/security.asciidoc @@ -2,20 +2,26 @@ [[security-api]] == Security APIs +You can use the following APIs to perform {security} activities. + * <> * <> * <> -* <> * <> * <> * <> * <> +include::security/roles.asciidoc[] + include::security/authenticate.asciidoc[] include::security/change-password.asciidoc[] include::security/clear-cache.asciidoc[] +include::security/clear-roles-cache.asciidoc[] +include::security/create-roles.asciidoc[] +include::security/delete-roles.asciidoc[] +include::security/get-roles.asciidoc[] include::security/privileges.asciidoc[] -include::security/roles.asciidoc[] include::security/role-mapping.asciidoc[] include::security/ssl.asciidoc[] include::security/tokens.asciidoc[] diff --git a/x-pack/docs/en/rest-api/security/clear-roles-cache.asciidoc b/x-pack/docs/en/rest-api/security/clear-roles-cache.asciidoc new file mode 100644 index 0000000000000..591d7eb2d11e4 --- /dev/null +++ b/x-pack/docs/en/rest-api/security/clear-roles-cache.asciidoc @@ -0,0 +1,39 @@ +[role="xpack"] +[[security-api-clear-role-cache]] +=== Clear roles cache API + +Evicts roles from the native role cache. + +==== Request + +`POST /_xpack/security/role//_clear_cache` + +==== Description + +For more information about the native realm, see +{stack-ov}/realms.html[Realms] and <>. + +==== Path Parameters + +`name`:: + (string) The name of the role. + + +//==== Request Body + +==== Authorization + +To use this API, you must have at least the `manage_security` cluster +privilege. + + +==== Examples + +The clear roles cache API evicts roles from the native role cache. For example, +to clear the cache for `my_admin_role`: + +[source,js] +-------------------------------------------------- +POST /_xpack/security/role/my_admin_role/_clear_cache +-------------------------------------------------- +// CONSOLE diff --git a/x-pack/docs/en/rest-api/security/create-roles.asciidoc b/x-pack/docs/en/rest-api/security/create-roles.asciidoc new file mode 100644 index 0000000000000..4b4088ab67f35 --- /dev/null +++ b/x-pack/docs/en/rest-api/security/create-roles.asciidoc @@ -0,0 +1,103 @@ +[role="xpack"] +[[security-api-put-role]] +=== Create roles API + +Adds roles in the native realm. + +==== Request + +`POST /_xpack/security/role/` + + +`PUT /_xpack/security/role/` + + +==== Description + +The role API is generally the preferred way to manage roles, rather than using +file-based role management. For more information about the native realm, see +{stack-ov}/realms.html[Realms] and <>. + + +==== Path Parameters + +`name`:: + (string) The name of the role. + + +==== Request Body + +The following parameters can be specified in the body of a PUT or POST request +and pertain to adding a role: + +`cluster`:: (list) A list of cluster privileges. These privileges define the +cluster level actions that users with this role are able to execute. + +`indices`:: (list) A list of indices permissions entries. +`field_security`::: (list) The document fields that the owners of the role have +read access to. For more information, see +{stack-ov}/field-and-document-access-control.html[Setting up field and document level security]. +`names` (required)::: (list) A list of indices (or index name patterns) to which the +permissions in this entry apply. +`privileges`(required)::: (list) The index level privileges that the owners of the role +have on the specified indices. +`query`::: A search query that defines the documents the owners of the role have +read access to. A document within the specified indices must match this query in +order for it to be accessible by the owners of the role. + +`metadata`:: (object) Optional meta-data. Within the `metadata` object, keys +that begin with `_` are reserved for system usage. + +`run_as`:: (list) A list of users that the owners of this role can impersonate. +For more information, see +{stack-ov}/run-as-privilege.html[Submitting requests on behalf of other users]. + +For more information, see {stack-ov}/defining-roles.html[Defining roles]. + + +==== Authorization + +To use this API, you must have at least the `manage_security` cluster +privilege. + + +==== Examples + +The following example adds a role called `my_admin_role`: + +[source,js] +-------------------------------------------------- +POST /_xpack/security/role/my_admin_role +{ + "cluster": ["all"], + "indices": [ + { + "names": [ "index1", "index2" ], + "privileges": ["all"], + "field_security" : { // optional + "grant" : [ "title", "body" ] + }, + "query": "{\"match\": {\"title\": \"foo\"}}" // optional + } + ], + "run_as": [ "other_user" ], // optional + "metadata" : { // optional + "version" : 1 + } +} +-------------------------------------------------- +// CONSOLE + +A successful call returns a JSON structure that shows whether the role has been +created or updated. + +[source,js] +-------------------------------------------------- +{ + "role": { + "created": true <1> + } +} +-------------------------------------------------- +// TESTRESPONSE +<1> When an existing role is updated, `created` is set to false. + diff --git a/x-pack/docs/en/rest-api/security/delete-roles.asciidoc b/x-pack/docs/en/rest-api/security/delete-roles.asciidoc new file mode 100644 index 0000000000000..8b5ee338fa8fd --- /dev/null +++ b/x-pack/docs/en/rest-api/security/delete-roles.asciidoc @@ -0,0 +1,52 @@ +[role="xpack"] +[[security-api-delete-role]] +=== Delete roles API + +Removes roles in the native realm. + +==== Request + +`DELETE /_xpack/security/role/` + + +==== Description + +The Roles API is generally the preferred way to manage roles, rather than using +file-based role management. For more information about the native realm, see +{stack-ov}/realms.html[Realms] and <>. + + +==== Path Parameters + +`name`:: + (string) The name of the role. + +//==== Request Body + +==== Authorization + +To use this API, you must have at least the `manage_security` cluster +privilege. + + +==== Examples + +The following example deletes a `my_admin_role` role: + +[source,js] +-------------------------------------------------- +DELETE /_xpack/security/role/my_admin_role +-------------------------------------------------- +// CONSOLE +// TEST[setup:admin_role] + +If the role is successfully deleted, the request returns `{"found": true}`. +Otherwise, `found` is set to false. + +[source,js] +-------------------------------------------------- +{ + "found" : true +} +-------------------------------------------------- +// TESTRESPONSE diff --git a/x-pack/docs/en/rest-api/security/get-roles.asciidoc b/x-pack/docs/en/rest-api/security/get-roles.asciidoc new file mode 100644 index 0000000000000..9490f03803751 --- /dev/null +++ b/x-pack/docs/en/rest-api/security/get-roles.asciidoc @@ -0,0 +1,86 @@ +[role="xpack"] +[[security-api-get-role]] +=== Get roles API + +Retrieves roles in the native realm. + +==== Request + +`GET /_xpack/security/role` + + +`GET /_xpack/security/role/` + + +==== Description + +For more information about the native realm, see +{stack-ov}/realms.html[Realms] and <>. + +==== Path Parameters + +`name`:: + (string) The name of the role. You can specify multiple roles as a + comma-separated list. If you do not specify this parameter, the API + returns information about all roles. + +//==== Request Body + +==== Authorization + +To use this API, you must have at least the `manage_security` cluster +privilege. + + +==== Examples + +The following example retrieves information about the `my_admin_role` role in +the native realm: + +[source,js] +-------------------------------------------------- +GET /_xpack/security/role/my_admin_role +-------------------------------------------------- +// CONSOLE +// TEST[setup:admin_role] + +A successful call returns an array of roles with the JSON representation of the +role. If the role is not defined in the native realm, the request returns 404. + +[source,js] +-------------------------------------------------- +{ + "my_admin_role": { + "cluster" : [ "all" ], + "indices" : [ + { + "names" : [ "index1", "index2" ], + "privileges" : [ "all" ], + "field_security" : { + "grant" : [ "title", "body" ]} + } + ], + "applications" : [ ], + "run_as" : [ "other_user" ], + "metadata" : { + "version" : 1 + }, + "transient_metadata": { + "enabled": true + } + } +} +-------------------------------------------------- +// TESTRESPONSE + +To retrieve all roles, omit the role name: + +[source,js] +-------------------------------------------------- +GET /_xpack/security/role +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +NOTE: If single role is requested, that role is returned as the response. When +requesting multiple roles, an object is returned holding the found roles, each +keyed by the relevant role name. + diff --git a/x-pack/docs/en/rest-api/security/roles.asciidoc b/x-pack/docs/en/rest-api/security/roles.asciidoc index 38ff774099ea5..a0d8aa7fea183 100644 --- a/x-pack/docs/en/rest-api/security/roles.asciidoc +++ b/x-pack/docs/en/rest-api/security/roles.asciidoc @@ -1,203 +1,9 @@ -[role="xpack"] +[float] [[security-api-roles]] -=== Role Management APIs +=== Roles -The Roles API enables you to add, remove, and retrieve roles in the `native` -realm. +You can use the following APIs to add, remove, and retrieve roles in the native realm: -==== Request - -`GET /_xpack/security/role` + - -`GET /_xpack/security/role/` + - -`POST /_xpack/security/role//_clear_cache` + - -`POST /_xpack/security/role/` + - -`PUT /_xpack/security/role/` - - -==== Description - -The Roles API is generally the preferred way to manage roles, rather than using -file-based role management. For more information, see -{xpack-ref}/authorization.html[Configuring Role-based Access Control]. - - -==== Path Parameters - -`name`:: - (string) The name of the role. If you do not specify this parameter, the - Get Roles API returns information about all roles. - - -==== Request Body - -The following parameters can be specified in the body of a PUT or POST request -and pertain to adding a role: - -`cluster`:: (list) A list of cluster privileges. These privileges define the -cluster level actions that users with this role are able to execute. - -`indices`:: (list) A list of indices permissions entries. -`field_security`::: (list) The document fields that the owners of the role have -read access to. For more information, see -{xpack-ref}/field-and-document-access-control.html[Setting Up Field and Document Level Security]. -`names` (required)::: (list) A list of indices (or index name patterns) to which the -permissions in this entry apply. -`privileges`(required)::: (list) The index level privileges that the owners of the role -have on the specified indices. -`query`::: A search query that defines the documents the owners of the role have -read access to. A document within the specified indices must match this query in -order for it to be accessible by the owners of the role. - -`metadata`:: (object) Optional meta-data. Within the `metadata` object, keys -that begin with `_` are reserved for system usage. - -`run_as`:: (list) A list of users that the owners of this role can impersonate. -For more information, see -{xpack-ref}/run-as-privilege.html[Submitting Requests on Behalf of Other Users]. - -For more information, see {xpack-ref}/defining-roles.html[Defining Roles]. - - -==== Authorization - -To use this API, you must have at least the `manage_security` cluster -privilege. - - -==== Examples - -[[security-api-put-role]] -To add a role, submit a PUT or POST request to the `/_xpack/security/role/` -endpoint: - -[source,js] --------------------------------------------------- -POST /_xpack/security/role/my_admin_role -{ - "cluster": ["all"], - "indices": [ - { - "names": [ "index1", "index2" ], - "privileges": ["all"], - "field_security" : { // optional - "grant" : [ "title", "body" ] - }, - "query": "{\"match\": {\"title\": \"foo\"}}" // optional - } - ], - "run_as": [ "other_user" ], // optional - "metadata" : { // optional - "version" : 1 - } -} --------------------------------------------------- -// CONSOLE - -A successful call returns a JSON structure that shows whether the role has been -created or updated. - -[source,js] --------------------------------------------------- -{ - "role": { - "created": true <1> - } -} --------------------------------------------------- -// TESTRESPONSE -<1> When an existing role is updated, `created` is set to false. - -[[security-api-get-role]] -To retrieve a role from the `native` Security realm, issue a GET request to the -`/_xpack/security/role/` endpoint: - -[source,js] --------------------------------------------------- -GET /_xpack/security/role/my_admin_role --------------------------------------------------- -// CONSOLE -// TEST[continued] - -A successful call returns an array of roles with the JSON representation of the -role. If the role is not defined in the `native` realm, the request 404s. - -[source,js] --------------------------------------------------- -{ - "my_admin_role": { - "cluster" : [ "all" ], - "indices" : [ { - "names" : [ "index1", "index2" ], - "privileges" : [ "all" ], - "field_security" : { - "grant" : [ "title", "body" ] - }, - "query" : "{\"match\": {\"title\": \"foo\"}}" - } ], - "applications" : [ ], - "run_as" : [ "other_user" ], - "metadata" : { - "version" : 1 - }, - "transient_metadata": { - "enabled": true - } - } -} --------------------------------------------------- -// TESTRESPONSE - -You can specify multiple roles as a comma-separated list. To retrieve all roles, -omit the role name. - -[source,js] --------------------------------------------------- -# Retrieve roles "r1", "r2", and "my_admin_role" -GET /_xpack/security/role/r1,r2,my_admin_role - -# Retrieve all roles -GET /_xpack/security/role --------------------------------------------------- -// CONSOLE -// TEST[continued] - -NOTE: If single role is requested, that role is returned as the response. When -requesting multiple roles, an object is returned holding the found roles, each -keyed by the relevant role name. - -[[security-api-delete-role]] -To delete a role, submit a DELETE request to the `/_xpack/security/role/` -endpoint: - -[source,js] --------------------------------------------------- -DELETE /_xpack/security/role/my_admin_role --------------------------------------------------- -// CONSOLE -// TEST[continued] - -If the role is successfully deleted, the request returns `{"found": true}`. -Otherwise, `found` is set to false. - -[source,js] --------------------------------------------------- -{ - "found" : true -} --------------------------------------------------- -// TESTRESPONSE - -[[security-api-clear-role-cache]] -The Clear Roles Cache API evicts roles from the native role cache. To clear the -cache for a role, submit a POST request `/_xpack/security/role//_clear_cache` -endpoint: - -[source,js] --------------------------------------------------- -POST /_xpack/security/role/my_admin_role/_clear_cache --------------------------------------------------- -// CONSOLE +* <>, <> +* <> +* <> diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.clear_cached_roles.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.clear_cached_roles.json index c94333325b127..d945ebe3247e0 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.clear_cached_roles.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.clear_cached_roles.json @@ -1,6 +1,6 @@ { "xpack.security.clear_cached_roles": { - "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-roles.html#security-api-clear-role-cache", + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-clear-role-cache.html", "methods": [ "POST" ], "url": { "path": "/_xpack/security/role/{name}/_clear_cache", diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.delete_role.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.delete_role.json index 4351b1bc847a1..881105d60b8b3 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.delete_role.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.delete_role.json @@ -1,6 +1,6 @@ { "xpack.security.delete_role": { - "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-roles.html#security-api-delete-role", + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-delete-role.html", "methods": [ "DELETE" ], "url": { "path": "/_xpack/security/role/{name}", diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.get_role.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.get_role.json index 3479c911ccdce..67bdbb8a911a2 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.get_role.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.get_role.json @@ -1,6 +1,6 @@ { "xpack.security.get_role": { - "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-roles.html#security-api-get-role", + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-role.html", "methods": [ "GET" ], "url": { "path": "/_xpack/security/role/{name}", diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_role.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_role.json index 4152975189e24..63ef5ee37867c 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_role.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_role.json @@ -1,6 +1,6 @@ { "xpack.security.put_role": { - "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-roles.html#security-api-put-role", + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-put-role.html", "methods": [ "PUT", "POST" ], "url": { "path": "/_xpack/security/role/{name}", From 66b767724bd66c3e264a077efd7933eddb37ef51 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Fri, 17 Aug 2018 17:52:29 +0100 Subject: [PATCH 34/42] [ML][TEST] Fix BasicRenormalizationIT after adding multibucket feature As the multibucket feature was merged in, this test hit a side effect which means buckets trailing an anomaly could become anomalous. This commit fixes the problem by filtering low score records when we request them. --- .../xpack/ml/integration/BasicRenormalizationIT.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/BasicRenormalizationIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/BasicRenormalizationIT.java index 80afdeff82ad8..cc5a9f4f1b469 100644 --- a/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/BasicRenormalizationIT.java +++ b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/BasicRenormalizationIT.java @@ -7,6 +7,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction; +import org.elasticsearch.xpack.core.ml.action.GetRecordsAction; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; import org.elasticsearch.xpack.core.ml.job.config.Detector; @@ -36,7 +37,11 @@ public void testDefaultRenormalization() throws Exception { String jobId = "basic-renormalization-it-test-default-renormalization-job"; createAndRunJob(jobId, null); - List records = getRecords(jobId); + GetRecordsAction.Request getRecordsRequest = new GetRecordsAction.Request(jobId); + // Setting the record score to 10.0, to avoid the low score records due to multibucket trailing effect + getRecordsRequest.setRecordScore(10.0); + + List records = getRecords(getRecordsRequest); assertThat(records.size(), equalTo(2)); AnomalyRecord laterRecord = records.get(0); assertThat(laterRecord.getActual().get(0), equalTo(100.0)); From abdccab2d5ae5244259a2d16cb8fb0b504c77006 Mon Sep 17 00:00:00 2001 From: lcawl Date: Fri, 17 Aug 2018 10:20:28 -0700 Subject: [PATCH 35/42] [DOCS] Fixes links to role management APIs --- x-pack/docs/en/security/authorization/managing-roles.asciidoc | 2 +- x-pack/docs/en/security/authorization/mapping-roles.asciidoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/docs/en/security/authorization/managing-roles.asciidoc b/x-pack/docs/en/security/authorization/managing-roles.asciidoc index b8a01aa4519c6..aa872a88ce0a4 100644 --- a/x-pack/docs/en/security/authorization/managing-roles.asciidoc +++ b/x-pack/docs/en/security/authorization/managing-roles.asciidoc @@ -130,7 +130,7 @@ manage roles, log in to {kib} and go to *Management / Elasticsearch / Roles*. The _Role Management APIs_ enable you to add, update, remove and retrieve roles dynamically. When you use the APIs to manage roles in the `native` realm, the roles are stored in an internal {es} index. For more information and examples, -see {ref}/security-api-roles.html[Role Management APIs]. +see {ref}/security-api.html#security-api-roles.html[role management APIs]. [float] [[roles-management-file]] diff --git a/x-pack/docs/en/security/authorization/mapping-roles.asciidoc b/x-pack/docs/en/security/authorization/mapping-roles.asciidoc index cf8373a65f335..9626571dbd5b4 100644 --- a/x-pack/docs/en/security/authorization/mapping-roles.asciidoc +++ b/x-pack/docs/en/security/authorization/mapping-roles.asciidoc @@ -18,7 +18,7 @@ the API, and other roles that are mapped through files. When you use role-mappings, you assign existing roles to users. The available roles should either be added using the -{ref}/security-api-roles.html[Role Management APIs] or defined in the +{ref}/security-api.html#security-api-roles.html[role management APIs] or defined in the <>. Either role-mapping method can use either role management method. For example, when you use the role mapping API, you are able to map users to both API-managed roles and file-managed roles From 3a55f563dc1281bf5bdfcf13625bdaf3839f6ce7 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Fri, 17 Aug 2018 13:33:12 -0400 Subject: [PATCH 36/42] [Docs] Tweaks and fixes to rollup docs - Missing links to new IndexCaps API - Incorrect security permissions on IndexCaps API - GetJobs API must supply a job (or `_all`), omitting throws error - Link to search/agg limitations from RollupSearch API - Tweak URLs in quick reference - Formatting of overview page --- x-pack/docs/en/rest-api/rollup-api.asciidoc | 2 ++ .../docs/en/rest-api/rollup/rollup-caps.asciidoc | 4 ++-- .../en/rest-api/rollup/rollup-index-caps.asciidoc | 4 +--- .../en/rest-api/rollup/rollup-job-config.asciidoc | 6 ++++++ .../en/rest-api/rollup/rollup-search.asciidoc | 2 +- x-pack/docs/en/rollup/api-quickref.asciidoc | 15 ++++++++------- x-pack/docs/en/rollup/overview.asciidoc | 4 ++++ 7 files changed, 24 insertions(+), 13 deletions(-) diff --git a/x-pack/docs/en/rest-api/rollup-api.asciidoc b/x-pack/docs/en/rest-api/rollup-api.asciidoc index f1cd7c285a733..9a8ec00d77a0c 100644 --- a/x-pack/docs/en/rest-api/rollup-api.asciidoc +++ b/x-pack/docs/en/rest-api/rollup-api.asciidoc @@ -16,6 +16,7 @@ === Data * <> +* <> [float] [[rollup-search-endpoint]] @@ -31,5 +32,6 @@ include::rollup/put-job.asciidoc[] include::rollup/start-job.asciidoc[] include::rollup/stop-job.asciidoc[] include::rollup/rollup-caps.asciidoc[] +include::rollup/rollup-index-caps.asciidoc[] include::rollup/rollup-search.asciidoc[] include::rollup/rollup-job-config.asciidoc[] \ No newline at end of file diff --git a/x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc b/x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc index f770adf1f0d1c..1f233f195a09e 100644 --- a/x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc +++ b/x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc @@ -27,8 +27,8 @@ live? ==== Path Parameters `index`:: - (string) Index, indices or index-pattern to return rollup capabilities for. If omitted (or `_all` is used) all available - rollup job capabilities will be returned + (string) Index, indices or index-pattern to return rollup capabilities for. `_all` may be used to fetch + rollup capabilities from all jobs ==== Request Body diff --git a/x-pack/docs/en/rest-api/rollup/rollup-index-caps.asciidoc b/x-pack/docs/en/rest-api/rollup/rollup-index-caps.asciidoc index 4636d9775e9d3..e5ca70cd59cda 100644 --- a/x-pack/docs/en/rest-api/rollup/rollup-index-caps.asciidoc +++ b/x-pack/docs/en/rest-api/rollup/rollup-index-caps.asciidoc @@ -26,15 +26,13 @@ This API will allow you to determine: `index`:: (string) Index or index-pattern of concrete rollup indices to check for capabilities. - - ==== Request Body There is no request body for the Get Jobs API. ==== Authorization -You must have `monitor`, `monitor_rollup`, `manage` or `manage_rollup` cluster privileges to use this API. +You must have the `read` index privilege on the index that stores the rollup results. For more information, see {xpack-ref}/security-privileges.html[Security Privileges]. diff --git a/x-pack/docs/en/rest-api/rollup/rollup-job-config.asciidoc b/x-pack/docs/en/rest-api/rollup/rollup-job-config.asciidoc index ef0ea6f00f7ce..2ba92b6b59ea6 100644 --- a/x-pack/docs/en/rest-api/rollup/rollup-job-config.asciidoc +++ b/x-pack/docs/en/rest-api/rollup/rollup-job-config.asciidoc @@ -82,6 +82,12 @@ In the above example, there are several pieces of logistical configuration for t will tend to execute faster, but will require more memory during processing. This has no effect on how the data is rolled up, it is merely used for tweaking the speed/memory cost of the indexer. +[NOTE] +The `index_pattern` cannot be a pattern that would also match the destination `rollup_index`. E.g. the pattern +`"foo-*"` would match the rollup index `"foo-rollup"`. This causes problems because the rollup job would attempt +to rollup it's own data at runtime. If you attempt to configure a pattern that matches the `rollup_index`, an exception +will be thrown to prevent this behavior. + [[rollup-groups-config]] ==== Grouping Config diff --git a/x-pack/docs/en/rest-api/rollup/rollup-search.asciidoc b/x-pack/docs/en/rest-api/rollup/rollup-search.asciidoc index 470cbc4eaf57d..f595d52ec10a1 100644 --- a/x-pack/docs/en/rest-api/rollup/rollup-search.asciidoc +++ b/x-pack/docs/en/rest-api/rollup/rollup-search.asciidoc @@ -34,7 +34,7 @@ or using `_all`, is not permitted The request body supports a subset of features from the regular Search API. It supports: -- `query` param for specifying an DSL query, subject to some limitations +- `query` param for specifying an DSL query, subject to some limitations (see <> and <> - `aggregations` param for specifying aggregations Functionality that is not available: diff --git a/x-pack/docs/en/rollup/api-quickref.asciidoc b/x-pack/docs/en/rollup/api-quickref.asciidoc index 937c6a84e5e14..5e99f1c69841c 100644 --- a/x-pack/docs/en/rollup/api-quickref.asciidoc +++ b/x-pack/docs/en/rollup/api-quickref.asciidoc @@ -15,18 +15,19 @@ Most {rollup} endpoints have the following base: [[rollup-api-jobs]] === /job/ -* {ref}/rollup-put-job.html[PUT /job/+++]: Create a job -* {ref}/rollup-get-job.html[GET /job]: List jobs -* {ref}/rollup-get-job.html[GET /job/+++]: Get job details -* {ref}/rollup-start-job.html[POST /job//_start]: Start a job -* {ref}/rollup-stop-job.html[POST /job//_stop]: Stop a job -* {ref}/rollup-delete-job.html[DELETE /job/+++]: Delete a job +* {ref}/rollup-put-job.html[PUT /_xpack/rollup/job/+++]: Create a job +* {ref}/rollup-get-job.html[GET /_xpack/rollup/job]: List jobs +* {ref}/rollup-get-job.html[GET /_xpack/rollup/job/+++]: Get job details +* {ref}/rollup-start-job.html[POST /_xpack/rollup/job//_start]: Start a job +* {ref}/rollup-stop-job.html[POST /_xpack/rollup/job//_stop]: Stop a job +* {ref}/rollup-delete-job.html[DELETE /_xpack/rollup/job/+++]: Delete a job [float] [[rollup-api-data]] === /data/ -* {ref}/rollup-get-rollup-caps.html[GET /data//_rollup_caps+++]: Get Rollup Capabilities +* {ref}/rollup-get-rollup-caps.html[GET /_xpack/rollup/data//_rollup_caps+++]: Get Rollup Capabilities +* {ref}/rollup-get-rollup-index-caps.html[GET //_rollup/data/+++]: Get Rollup Index Capabilities [float] [[rollup-api-index]] diff --git a/x-pack/docs/en/rollup/overview.asciidoc b/x-pack/docs/en/rollup/overview.asciidoc index a3f29f23bd107..a9a983fbecc1d 100644 --- a/x-pack/docs/en/rollup/overview.asciidoc +++ b/x-pack/docs/en/rollup/overview.asciidoc @@ -20,6 +20,7 @@ So while the cost of storing a millisecond of sensor data from ten years ago is reading often diminishes with time. It's not useless -- it could easily contribute to a useful analysis -- but it's reduced value often leads to deletion rather than paying the fixed storage cost. +[float] === Rollup store historical data at reduced granularity That's where Rollup comes into play. The Rollup functionality summarizes old, high-granularity data into a reduced @@ -35,6 +36,7 @@ automates this process of summarizing historical data. Details about setting up and configuring Rollup are covered in <> +[float] === Rollup uses standard query DSL The Rollup feature exposes a new search endpoint (`/_rollup_search` vs the standard `/_search`) which knows how to search @@ -48,6 +50,7 @@ are covered more in <>. But if your queries, aggregations and dashboards only use the available functionality, redirecting them to historical data is trivial. +[float] === Rollup merges "live" and "rolled" data A useful feature of Rollup is the ability to query both "live", realtime data in addition to historical "rolled" data @@ -61,6 +64,7 @@ would only see data older than a month. The RollupSearch endpoint, however, sup It will take the results from both data sources and merge them together. If there is overlap between the "live" and "rolled" data, live data is preferred to increase accuracy. +[float] === Rollup is multi-interval aware Finally, Rollup is capable of intelligently utilizing the best interval available. If you've worked with summarizing From 3c9e11089c815a53cc579cd5c47014f4009cb83d Mon Sep 17 00:00:00 2001 From: lcawl Date: Fri, 17 Aug 2018 10:41:06 -0700 Subject: [PATCH 37/42] [DOCS] Fixes more broken links to role management APIs --- x-pack/docs/en/security/authorization/managing-roles.asciidoc | 2 +- x-pack/docs/en/security/authorization/mapping-roles.asciidoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/docs/en/security/authorization/managing-roles.asciidoc b/x-pack/docs/en/security/authorization/managing-roles.asciidoc index aa872a88ce0a4..a7887ee664874 100644 --- a/x-pack/docs/en/security/authorization/managing-roles.asciidoc +++ b/x-pack/docs/en/security/authorization/managing-roles.asciidoc @@ -130,7 +130,7 @@ manage roles, log in to {kib} and go to *Management / Elasticsearch / Roles*. The _Role Management APIs_ enable you to add, update, remove and retrieve roles dynamically. When you use the APIs to manage roles in the `native` realm, the roles are stored in an internal {es} index. For more information and examples, -see {ref}/security-api.html#security-api-roles.html[role management APIs]. +see {ref}/security-api.html#security-api-roles[role management APIs]. [float] [[roles-management-file]] diff --git a/x-pack/docs/en/security/authorization/mapping-roles.asciidoc b/x-pack/docs/en/security/authorization/mapping-roles.asciidoc index 9626571dbd5b4..1c6482510f0f3 100644 --- a/x-pack/docs/en/security/authorization/mapping-roles.asciidoc +++ b/x-pack/docs/en/security/authorization/mapping-roles.asciidoc @@ -18,7 +18,7 @@ the API, and other roles that are mapped through files. When you use role-mappings, you assign existing roles to users. The available roles should either be added using the -{ref}/security-api.html#security-api-roles.html[role management APIs] or defined in the +{ref}/security-api.html#security-api-roles[role management APIs] or defined in the <>. Either role-mapping method can use either role management method. For example, when you use the role mapping API, you are able to map users to both API-managed roles and file-managed roles From 91a01fc3532cae413a174e505cf16c60035c6aa1 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Fri, 17 Aug 2018 13:55:42 -0400 Subject: [PATCH 38/42] TEST: Mute testRetentionPolicyChangeDuringRecovery Tracked at #32089 --- .../java/org/elasticsearch/indices/recovery/RecoveryTests.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java index fde7ed874290a..609365e175b85 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java @@ -43,7 +43,6 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.translog.SnapshotMatchers; import org.elasticsearch.index.translog.Translog; -import org.elasticsearch.test.junit.annotations.TestLogging; import java.util.HashMap; import java.util.List; @@ -74,7 +73,7 @@ public void testTranslogHistoryTransferred() throws Exception { } } - @TestLogging("_root:TRACE") + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32089") public void testRetentionPolicyChangeDuringRecovery() throws Exception { try (ReplicationGroup shards = createGroup(0)) { shards.startPrimary(); From f5593016f95d32987a1e42f7ea92a26beffb93a3 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Fri, 17 Aug 2018 15:30:31 -0500 Subject: [PATCH 39/42] Bypassing failing test PainlessDomainSplitIT#testHRDSplit (#32966) --- .../elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/qa/ml-single-node-tests/src/test/java/org/elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java b/x-pack/qa/ml-single-node-tests/src/test/java/org/elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java index 79e9a81831fc8..0751d7307ae9a 100644 --- a/x-pack/qa/ml-single-node-tests/src/test/java/org/elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java +++ b/x-pack/qa/ml-single-node-tests/src/test/java/org/elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java @@ -240,6 +240,7 @@ public void testIsolated() throws Exception { } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32966") public void testHRDSplit() throws Exception { // Create job From bf08acc8231cb14ab35e1922393b0625a810a5ac Mon Sep 17 00:00:00 2001 From: lcawl Date: Fri, 17 Aug 2018 21:39:21 -0700 Subject: [PATCH 40/42] [DOCS] Creates redirects for role management APIs page --- docs/reference/redirects.asciidoc | 9 +++++++++ x-pack/docs/en/rest-api/security.asciidoc | 10 +++++++++- x-pack/docs/en/rest-api/security/role-mapping.asciidoc | 2 +- x-pack/docs/en/rest-api/security/roles.asciidoc | 9 --------- .../en/security/authorization/managing-roles.asciidoc | 2 +- .../en/security/authorization/mapping-roles.asciidoc | 2 +- 6 files changed, 21 insertions(+), 13 deletions(-) delete mode 100644 x-pack/docs/en/rest-api/security/roles.asciidoc diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index 56a6560080795..6e53567ccf24b 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -498,3 +498,12 @@ guide to the {painless}/index.html[Painless Scripting Language]. See the {painless}/painless-api-reference.html[Painless API Reference] in the guide to the {painless}/index.html[Painless Scripting Language]. + +[role="exclude", id="security-api-roles"] +=== Role management APIs + +You can use the following APIs to add, remove, and retrieve roles in the native realm: + +* <>, <> +* <> +* <> \ No newline at end of file diff --git a/x-pack/docs/en/rest-api/security.asciidoc b/x-pack/docs/en/rest-api/security.asciidoc index 4d59cb22daa06..476c9b95bfda3 100644 --- a/x-pack/docs/en/rest-api/security.asciidoc +++ b/x-pack/docs/en/rest-api/security.asciidoc @@ -12,7 +12,15 @@ You can use the following APIs to perform {security} activities. * <> * <> -include::security/roles.asciidoc[] +[float] +[[security-role-apis]] +=== Roles + +You can use the following APIs to add, remove, and retrieve roles in the native realm: + +* <>, <> +* <> +* <> include::security/authenticate.asciidoc[] include::security/change-password.asciidoc[] diff --git a/x-pack/docs/en/rest-api/security/role-mapping.asciidoc b/x-pack/docs/en/rest-api/security/role-mapping.asciidoc index 3844e30c62dc0..c8006346d4e8f 100644 --- a/x-pack/docs/en/rest-api/security/role-mapping.asciidoc +++ b/x-pack/docs/en/rest-api/security/role-mapping.asciidoc @@ -22,7 +22,7 @@ Role mappings have _rules_ that identify users and a list of _roles_ that are granted to those users. NOTE: This API does not create roles. Rather, it maps users to existing roles. -Roles can be created by using <> or +Roles can be created by using <> or {xpack-ref}/defining-roles.html#roles-management-file[roles files]. The role mapping rule is a logical condition that is expressed using a JSON DSL. diff --git a/x-pack/docs/en/rest-api/security/roles.asciidoc b/x-pack/docs/en/rest-api/security/roles.asciidoc deleted file mode 100644 index a0d8aa7fea183..0000000000000 --- a/x-pack/docs/en/rest-api/security/roles.asciidoc +++ /dev/null @@ -1,9 +0,0 @@ -[float] -[[security-api-roles]] -=== Roles - -You can use the following APIs to add, remove, and retrieve roles in the native realm: - -* <>, <> -* <> -* <> diff --git a/x-pack/docs/en/security/authorization/managing-roles.asciidoc b/x-pack/docs/en/security/authorization/managing-roles.asciidoc index a7887ee664874..f550c900edce0 100644 --- a/x-pack/docs/en/security/authorization/managing-roles.asciidoc +++ b/x-pack/docs/en/security/authorization/managing-roles.asciidoc @@ -130,7 +130,7 @@ manage roles, log in to {kib} and go to *Management / Elasticsearch / Roles*. The _Role Management APIs_ enable you to add, update, remove and retrieve roles dynamically. When you use the APIs to manage roles in the `native` realm, the roles are stored in an internal {es} index. For more information and examples, -see {ref}/security-api.html#security-api-roles[role management APIs]. +see {ref}/security-api.html#security-role-apis[role management APIs]. [float] [[roles-management-file]] diff --git a/x-pack/docs/en/security/authorization/mapping-roles.asciidoc b/x-pack/docs/en/security/authorization/mapping-roles.asciidoc index 1c6482510f0f3..36f3a1f27f346 100644 --- a/x-pack/docs/en/security/authorization/mapping-roles.asciidoc +++ b/x-pack/docs/en/security/authorization/mapping-roles.asciidoc @@ -18,7 +18,7 @@ the API, and other roles that are mapped through files. When you use role-mappings, you assign existing roles to users. The available roles should either be added using the -{ref}/security-api.html#security-api-roles[role management APIs] or defined in the +{ref}/security-api.html#security-role-apis[role management APIs] or defined in the <>. Either role-mapping method can use either role management method. For example, when you use the role mapping API, you are able to map users to both API-managed roles and file-managed roles From a68a9f05004c52efa2ac0e31b8110b1dac7f5c1a Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Fri, 17 Aug 2018 22:22:09 -0700 Subject: [PATCH 41/42] [DOCS] Splits the token APIs into separate pages (#32865) --- docs/reference/redirects.asciidoc | 10 ++- x-pack/docs/en/rest-api/security.asciidoc | 13 ++- .../rest-api/security/delete-tokens.asciidoc | 54 ++++++++++++ .../{tokens.asciidoc => get-tokens.asciidoc} | 86 ++++++------------- .../api/xpack.security.get_token.json | 2 +- .../api/xpack.security.invalidate_token.json | 2 +- 6 files changed, 104 insertions(+), 63 deletions(-) create mode 100644 x-pack/docs/en/rest-api/security/delete-tokens.asciidoc rename x-pack/docs/en/rest-api/security/{tokens.asciidoc => get-tokens.asciidoc} (62%) diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index 6e53567ccf24b..7b7915c3781df 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -506,4 +506,12 @@ You can use the following APIs to add, remove, and retrieve roles in the native * <>, <> * <> -* <> \ No newline at end of file +* <> + +[role="exclude",id="security-api-tokens"] +=== Token management APIs + +You can use the following APIs to create and invalidate bearer tokens for access +without requiring basic authentication: + +* <>, <> diff --git a/x-pack/docs/en/rest-api/security.asciidoc b/x-pack/docs/en/rest-api/security.asciidoc index 476c9b95bfda3..27e54df38b31c 100644 --- a/x-pack/docs/en/rest-api/security.asciidoc +++ b/x-pack/docs/en/rest-api/security.asciidoc @@ -9,7 +9,6 @@ You can use the following APIs to perform {security} activities. * <> * <> * <> -* <> * <> [float] @@ -22,15 +21,25 @@ You can use the following APIs to add, remove, and retrieve roles in the native * <> * <> +[float] +[[security-token-apis]] +=== Tokens + +You can use the following APIs to create and invalidate bearer tokens for access +without requiring basic authentication: + +* <>, <> + include::security/authenticate.asciidoc[] include::security/change-password.asciidoc[] include::security/clear-cache.asciidoc[] include::security/clear-roles-cache.asciidoc[] include::security/create-roles.asciidoc[] include::security/delete-roles.asciidoc[] +include::security/delete-tokens.asciidoc[] include::security/get-roles.asciidoc[] +include::security/get-tokens.asciidoc[] include::security/privileges.asciidoc[] include::security/role-mapping.asciidoc[] include::security/ssl.asciidoc[] -include::security/tokens.asciidoc[] include::security/users.asciidoc[] diff --git a/x-pack/docs/en/rest-api/security/delete-tokens.asciidoc b/x-pack/docs/en/rest-api/security/delete-tokens.asciidoc new file mode 100644 index 0000000000000..7d6bae2a4c40f --- /dev/null +++ b/x-pack/docs/en/rest-api/security/delete-tokens.asciidoc @@ -0,0 +1,54 @@ +[role="xpack"] +[[security-api-invalidate-token]] +=== Delete token API + +Invalidates a bearer token for access without requiring basic authentication. + +==== Request + +`DELETE /_xpack/security/oauth2/token` + +==== Description + +The tokens returned by the <> have a +finite period of time for which they are valid and after that time period, they +can no longer be used. That time period is defined by the +`xpack.security.authc.token.timeout` setting. For more information, see +<>. + +If you want to invalidate a token immediately, use this delete token API. + + +==== Request Body + +The following parameters can be specified in the body of a DELETE request and +pertain to deleting a token: + +`token` (required):: +(string) An access token. + +==== Examples + +The following example invalidates the specified token immediately: + +[source,js] +-------------------------------------------------- +DELETE /_xpack/security/oauth2/token +{ + "token" : "dGhpcyBpcyBub3QgYSByZWFsIHRva2VuIGJ1dCBpdCBpcyBvbmx5IHRlc3QgZGF0YS4gZG8gbm90IHRyeSB0byByZWFkIHRva2VuIQ==" +} +-------------------------------------------------- +// NOTCONSOLE + +A successful call returns a JSON structure that indicates whether the token +has already been invalidated. + +[source,js] +-------------------------------------------------- +{ + "created" : true <1> +} +-------------------------------------------------- +// NOTCONSOLE + +<1> When a token has already been invalidated, `created` is set to false. diff --git a/x-pack/docs/en/rest-api/security/tokens.asciidoc b/x-pack/docs/en/rest-api/security/get-tokens.asciidoc similarity index 62% rename from x-pack/docs/en/rest-api/security/tokens.asciidoc rename to x-pack/docs/en/rest-api/security/get-tokens.asciidoc index f991a5c0cb836..a2c4e6d7a37ec 100644 --- a/x-pack/docs/en/rest-api/security/tokens.asciidoc +++ b/x-pack/docs/en/rest-api/security/get-tokens.asciidoc @@ -1,15 +1,12 @@ [role="xpack"] -[[security-api-tokens]] -=== Token Management APIs +[[security-api-get-token]] +=== Get token API -The `token` API enables you to create and invalidate bearer tokens for access -without requiring basic authentication. +Creates a bearer token for access without requiring basic authentication. ==== Request -`POST /_xpack/security/oauth2/token` + - -`DELETE /_xpack/security/oauth2/token` +`POST /_xpack/security/oauth2/token` ==== Description @@ -19,20 +16,20 @@ you can explicitly enable the `xpack.security.authc.token.enabled` setting. When you are running in production mode, a bootstrap check prevents you from enabling the token service unless you also enable TLS on the HTTP interface. -The Get Token API takes the same parameters as a typical OAuth 2.0 token API +The get token API takes the same parameters as a typical OAuth 2.0 token API except for the use of a JSON request body. -A successful Get Token API call returns a JSON structure that contains the access +A successful get token API call returns a JSON structure that contains the access token, the amount of time (seconds) that the token expires in, the type, and the scope if available. -The tokens returned by the Get Token API have a finite period of time for which +The tokens returned by the get token API have a finite period of time for which they are valid and after that time period, they can no longer be used. That time period is defined by the `xpack.security.authc.token.timeout` setting. For more information, see <>. -If you want to invalidate a token immediately, you can do so by using the Delete -Token API. +If you want to invalidate a token immediately, you can do so by using the +<>. ==== Request Body @@ -41,28 +38,28 @@ The following parameters can be specified in the body of a POST request and pertain to creating a token: `grant_type`:: -(string) The type of grant. Currently only the `password` grant type is supported. +(string) The type of grant. Valid grant types are: `password` and `refresh_token`. -`password` (required):: -(string) The user's password. +`password`:: +(string) The user's password. If you specify the `password` grant type, this +parameter is required. + +`refresh_token`:: +(string) If you specify the `refresh_token` grant type, this parameter is +required. It contains the string that was returned when you created the token +and enables you to extend its life. `scope`:: (string) The scope of the token. Currently tokens are only issued for a scope of `FULL` regardless of the value sent with the request. -`username` (required):: -(string) The username that identifies the user. - -The following parameters can be specified in the body of a DELETE request and -pertain to deleting a token: - -`token`:: -(string) An access token. +`username`:: +(string) The username that identifies the user. If you specify the `password` +grant type, this parameter is required. ==== Examples -[[security-api-get-token]] -To obtain a token, submit a POST request to the `/_xpack/security/oauth2/token` -endpoint. + +The following example obtains a token for the `test_admin` user: [source,js] -------------------------------------------------- @@ -101,8 +98,8 @@ curl -H "Authorization: Bearer dGhpcyBpcyBub3QgYSByZWFsIHRva2VuIGJ1dCBpdCBpcyBvb // NOTCONSOLE [[security-api-refresh-token]] -To extend the life of an existing token, the token api may be called again with the refresh -token within 24 hours of the token's creation. +To extend the life of an existing token, you can call the API again with the +refresh token within 24 hours of the token's creation. For example: [source,js] -------------------------------------------------- @@ -116,7 +113,8 @@ POST /_xpack/security/oauth2/token // TEST[s/vLBPvmAB6KvwvJZr27cS/$body.refresh_token/] // TEST[continued] -The API will return a new token and refresh token. Each refresh token may only be used one time. +The API will return a new token and refresh token. Each refresh token may only +be used one time. [source,js] -------------------------------------------------- @@ -128,32 +126,4 @@ The API will return a new token and refresh token. Each refresh token may only b } -------------------------------------------------- // TESTRESPONSE[s/dGhpcyBpcyBub3QgYSByZWFsIHRva2VuIGJ1dCBpdCBpcyBvbmx5IHRlc3QgZGF0YS4gZG8gbm90IHRyeSB0byByZWFkIHRva2VuIQ==/$body.access_token/] -// TESTRESPONSE[s/vLBPvmAB6KvwvJZr27cS/$body.refresh_token/] - -[[security-api-invalidate-token]] -If a token must be invalidated immediately, you can do so by submitting a DELETE -request to `/_xpack/security/oauth2/token`. For example: - -[source,js] --------------------------------------------------- -DELETE /_xpack/security/oauth2/token -{ - "token" : "dGhpcyBpcyBub3QgYSByZWFsIHRva2VuIGJ1dCBpdCBpcyBvbmx5IHRlc3QgZGF0YS4gZG8gbm90IHRyeSB0byByZWFkIHRva2VuIQ==" -} --------------------------------------------------- -// CONSOLE -// TEST[s/dGhpcyBpcyBub3QgYSByZWFsIHRva2VuIGJ1dCBpdCBpcyBvbmx5IHRlc3QgZGF0YS4gZG8gbm90IHRyeSB0byByZWFkIHRva2VuIQ==/$body.access_token/] -// TEST[continued] - -A successful call returns a JSON structure that indicates whether the token -has already been invalidated. - -[source,js] --------------------------------------------------- -{ - "created" : true <1> -} --------------------------------------------------- -// TESTRESPONSE - -<1> When a token has already been invalidated, `created` is set to false. +// TESTRESPONSE[s/vLBPvmAB6KvwvJZr27cS/$body.refresh_token/] \ No newline at end of file diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.get_token.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.get_token.json index 8020d1ecd6d97..0b6f141d10e6a 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.get_token.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.get_token.json @@ -1,6 +1,6 @@ { "xpack.security.get_token": { - "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-tokens.html#security-api-get-token", + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-token.html", "methods": [ "POST" ], "url": { "path": "/_xpack/security/oauth2/token", diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.invalidate_token.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.invalidate_token.json index be032c2ffd020..27dd103091422 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.invalidate_token.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.invalidate_token.json @@ -1,6 +1,6 @@ { "xpack.security.invalidate_token": { - "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-tokens.html#security-api-invalidate-token", + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-invalidate-token.html", "methods": [ "DELETE" ], "url": { "path": "/_xpack/security/oauth2/token", From e5e1821a75c6216b8bebeaa2e00b9946dbd644f4 Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Fri, 17 Aug 2018 23:17:33 -0700 Subject: [PATCH 42/42] [DOCS] Splits the users API documentation into multiple pages (#32825) --- docs/reference/redirects.asciidoc | 11 + x-pack/docs/build.gradle | 13 + x-pack/docs/en/rest-api/security.asciidoc | 19 +- .../security/change-password.asciidoc | 21 +- .../rest-api/security/create-users.asciidoc | 107 +++++++++ .../rest-api/security/delete-users.asciidoc | 48 ++++ .../rest-api/security/disable-users.asciidoc | 43 ++++ .../rest-api/security/enable-users.asciidoc | 42 ++++ .../en/rest-api/security/get-users.asciidoc | 74 ++++++ .../docs/en/rest-api/security/users.asciidoc | 226 ------------------ .../api/xpack.security.delete_user.json | 2 +- .../api/xpack.security.disable_user.json | 2 +- .../api/xpack.security.enable_user.json | 2 +- .../api/xpack.security.get_user.json | 2 +- .../api/xpack.security.put_user.json | 2 +- 15 files changed, 375 insertions(+), 239 deletions(-) create mode 100644 x-pack/docs/en/rest-api/security/create-users.asciidoc create mode 100644 x-pack/docs/en/rest-api/security/delete-users.asciidoc create mode 100644 x-pack/docs/en/rest-api/security/disable-users.asciidoc create mode 100644 x-pack/docs/en/rest-api/security/enable-users.asciidoc create mode 100644 x-pack/docs/en/rest-api/security/get-users.asciidoc delete mode 100644 x-pack/docs/en/rest-api/security/users.asciidoc diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index 7b7915c3781df..e5a9acab9bfa8 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -515,3 +515,14 @@ You can use the following APIs to create and invalidate bearer tokens for access without requiring basic authentication: * <>, <> + +[role="exclude",id="security-api-users"] +=== User Management APIs + +You can use the following APIs to create, read, update, and delete users from the +native realm: + +* <>, <> +* <>, <> +* <> +* <> diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle index ea60ec2ca49a8..7654eef123f8d 100644 --- a/x-pack/docs/build.gradle +++ b/x-pack/docs/build.gradle @@ -737,3 +737,16 @@ setups['admin_role'] = ''' "metadata" : {"version": 1} } ''' +setups['jacknich_user'] = ''' + - do: + xpack.security.put_user: + username: "jacknich" + body: > + { + "password" : "test-password", + "roles" : [ "admin", "other_role1" ], + "full_name" : "Jack Nicholson", + "email" : "jacknich@example.com", + "metadata" : { "intelligence" : 7 } + } +''' diff --git a/x-pack/docs/en/rest-api/security.asciidoc b/x-pack/docs/en/rest-api/security.asciidoc index 27e54df38b31c..f5b0c8eef667d 100644 --- a/x-pack/docs/en/rest-api/security.asciidoc +++ b/x-pack/docs/en/rest-api/security.asciidoc @@ -9,7 +9,6 @@ You can use the following APIs to perform {security} activities. * <> * <> * <> -* <> [float] [[security-role-apis]] @@ -30,16 +29,32 @@ without requiring basic authentication: * <>, <> +[float] +[[security-user-apis]] +=== Users + +You can use the following APIs to create, read, update, and delete users from the +native realm: + +* <>, <> +* <>, <> +* <> +* <> + include::security/authenticate.asciidoc[] include::security/change-password.asciidoc[] include::security/clear-cache.asciidoc[] include::security/clear-roles-cache.asciidoc[] include::security/create-roles.asciidoc[] +include::security/create-users.asciidoc[] include::security/delete-roles.asciidoc[] include::security/delete-tokens.asciidoc[] +include::security/delete-users.asciidoc[] +include::security/disable-users.asciidoc[] +include::security/enable-users.asciidoc[] include::security/get-roles.asciidoc[] include::security/get-tokens.asciidoc[] +include::security/get-users.asciidoc[] include::security/privileges.asciidoc[] include::security/role-mapping.asciidoc[] include::security/ssl.asciidoc[] -include::security/users.asciidoc[] diff --git a/x-pack/docs/en/rest-api/security/change-password.asciidoc b/x-pack/docs/en/rest-api/security/change-password.asciidoc index 7dee98480e72c..6e6e8cf7375e4 100644 --- a/x-pack/docs/en/rest-api/security/change-password.asciidoc +++ b/x-pack/docs/en/rest-api/security/change-password.asciidoc @@ -1,9 +1,8 @@ [role="xpack"] [[security-api-change-password]] -=== Change Password API +=== Change passwords API -The Change Password API enables you to submit a request to change the password -of a user. +Changes the passwords of users in the native realm. ==== Request @@ -12,6 +11,15 @@ of a user. `POST _xpack/security/user//_password` +==== Description + +You can use the <> to update everything +but a user's `username` and `password`. This API changes a user's password. + +For more information about the native realm, see +{stack-ov}/realms.html[Realms] and <>. + + ==== Path Parameters `username`:: @@ -33,16 +41,17 @@ privilege can change passwords of other users. ==== Examples -The following example updates the password for the `elastic` user: +The following example updates the password for the `jacknich` user: [source,js] -------------------------------------------------- -POST _xpack/security/user/elastic/_password +POST /_xpack/security/user/jacknich/_password { - "password": "x-pack-test-password" + "password" : "s3cr3t" } -------------------------------------------------- // CONSOLE +// TEST[setup:jacknich_user] A successful call returns an empty JSON structure. diff --git a/x-pack/docs/en/rest-api/security/create-users.asciidoc b/x-pack/docs/en/rest-api/security/create-users.asciidoc new file mode 100644 index 0000000000000..5015d0401c223 --- /dev/null +++ b/x-pack/docs/en/rest-api/security/create-users.asciidoc @@ -0,0 +1,107 @@ +[role="xpack"] +[[security-api-put-user]] +=== Create users API + +Creates and updates users in the native realm. These users are commonly referred +to as _native users_. + + +==== Request + +`POST /_xpack/security/user/` + + +`PUT /_xpack/security/user/` + + +==== Description + +When updating a user, you can update everything but its `username` and `password`. +To change a user's password, use the +<>. + +For more information about the native realm, see +{stack-ov}/realms.html[Realms] and <>. + +==== Path Parameters + +`username` (required):: + (string) An identifier for the user. ++ +-- +[[username-validation]] +NOTE: Usernames must be at least 1 and no more than 1024 characters. They can +contain alphanumeric characters (`a-z`, `A-Z`, `0-9`), spaces, punctuation, and +printable symbols in the https://en.wikipedia.org/wiki/Basic_Latin_(Unicode_block)[Basic Latin (ASCII) block]. Leading or trailing whitespace is not allowed. + +-- + + +==== Request Body + +The following parameters can be specified in the body of a POST or PUT request: + +`enabled`:: +(boolean) Specifies whether the user is enabled. The default value is `true`. + +`email`:: +(string) The email of the user. + +`full_name`:: +(string) The full name of the user. + +`metadata`:: +(object) Arbitrary metadata that you want to associate with the user. + +`password` (required):: +(string) The user's password. Passwords must be at least 6 characters long. + +`roles` (required):: +(list) A set of roles the user has. The roles determine the user's access +permissions. To create a user without any roles, specify an empty list: `[]`. + + +==== Authorization + +To use this API, you must have at least the `manage_security` cluster privilege. + + +==== Examples + +The following example creates a user `jacknich`: + +[source,js] +-------------------------------------------------- +POST /_xpack/security/user/jacknich +{ + "password" : "j@rV1s", + "roles" : [ "admin", "other_role1" ], + "full_name" : "Jack Nicholson", + "email" : "jacknich@example.com", + "metadata" : { + "intelligence" : 7 + } +} +-------------------------------------------------- +// CONSOLE + +A successful call returns a JSON structure that shows whether the user has been +created or updated. + +[source,js] +-------------------------------------------------- +{ + "user": { + "created" : true <1> + } +} +-------------------------------------------------- +// TESTRESPONSE +<1> When an existing user is updated, `created` is set to false. + +After you add a user, requests from that user can be authenticated. For example: + +[source,shell] +-------------------------------------------------- +curl -u jacknich:j@rV1s http://localhost:9200/_cluster/health +-------------------------------------------------- +// NOTCONSOLE diff --git a/x-pack/docs/en/rest-api/security/delete-users.asciidoc b/x-pack/docs/en/rest-api/security/delete-users.asciidoc new file mode 100644 index 0000000000000..63a66795617bd --- /dev/null +++ b/x-pack/docs/en/rest-api/security/delete-users.asciidoc @@ -0,0 +1,48 @@ +[role="xpack"] +[[security-api-delete-user]] +=== Delete users API + +Deletes users from the native realm. + +==== Request + +`DELETE /_xpack/security/user/` + +==== Description + +For more information about the native realm, see +{stack-ov}/realms.html[Realms] and <>. + +==== Path Parameters + +`username` (required):: + (string) An identifier for the user. + +//==== Request Body + +==== Authorization + +To use this API, you must have at least the `manage_security` cluster privilege. + + +==== Examples + +The following example deletes the user `jacknich`: + +[source,js] +-------------------------------------------------- +DELETE /_xpack/security/user/jacknich +-------------------------------------------------- +// CONSOLE +// TEST[setup:jacknich_user] + +If the user is successfully deleted, the request returns `{"found": true}`. +Otherwise, `found` is set to false. + +[source,js] +-------------------------------------------------- +{ + "found" : true +} +-------------------------------------------------- +// TESTRESPONSE diff --git a/x-pack/docs/en/rest-api/security/disable-users.asciidoc b/x-pack/docs/en/rest-api/security/disable-users.asciidoc new file mode 100644 index 0000000000000..f5a6bc7e9a136 --- /dev/null +++ b/x-pack/docs/en/rest-api/security/disable-users.asciidoc @@ -0,0 +1,43 @@ +[role="xpack"] +[[security-api-disable-user]] +=== Disable users API + +Disables users in the native realm. + + +==== Request + +`PUT /_xpack/security/user//_disable` + + +==== Description + +By default, when you create users, they are enabled. You can use this API to +revoke a user's access to {es}. To re-enable a user, there is an +<>. + +For more information about the native realm, see +{stack-ov}/realms.html[Realms] and <>. + +==== Path Parameters + +`username` (required):: + (string) An identifier for the user. + +//==== Request Body + +==== Authorization + +To use this API, you must have at least the `manage_security` cluster privilege. + + +==== Examples + +The following example disables the user `jacknich`: + +[source,js] +-------------------------------------------------- +PUT /_xpack/security/user/jacknich/_disable +-------------------------------------------------- +// CONSOLE +// TEST[setup:jacknich_user] diff --git a/x-pack/docs/en/rest-api/security/enable-users.asciidoc b/x-pack/docs/en/rest-api/security/enable-users.asciidoc new file mode 100644 index 0000000000000..cebaaffa7b28d --- /dev/null +++ b/x-pack/docs/en/rest-api/security/enable-users.asciidoc @@ -0,0 +1,42 @@ +[role="xpack"] +[[security-api-enable-user]] +=== Enable users API + +Enables users in the native realm. + + +==== Request + +`PUT /_xpack/security/user//_enable` + + +==== Description + +By default, when you create users, they are enabled. You can use this enable +users API and the <> to change that attribute. + +For more information about the native realm, see +{stack-ov}/realms.html[Realms] and <>. + +==== Path Parameters + +`username` (required):: + (string) An identifier for the user. + +//==== Request Body + +==== Authorization + +To use this API, you must have at least the `manage_security` cluster privilege. + + +==== Examples + +The following example enables the user `jacknich`: + +[source,js] +-------------------------------------------------- +PUT /_xpack/security/user/jacknich/_enable +-------------------------------------------------- +// CONSOLE +// TEST[setup:jacknich_user] diff --git a/x-pack/docs/en/rest-api/security/get-users.asciidoc b/x-pack/docs/en/rest-api/security/get-users.asciidoc new file mode 100644 index 0000000000000..2a20baacb0f52 --- /dev/null +++ b/x-pack/docs/en/rest-api/security/get-users.asciidoc @@ -0,0 +1,74 @@ +[role="xpack"] +[[security-api-get-user]] +=== Get users API + +Retrieves information about users in the native realm. + + +==== Request + +`GET /_xpack/security/user` + + +`GET /_xpack/security/user/` + +==== Description + +For more information about the native realm, see +{stack-ov}/realms.html[Realms] and <>. + +==== Path Parameters + +`username`:: + (string) An identifier for the user. You can specify multiple usernames as a comma-separated list. If you omit this parameter, the API retrieves + information about all users. + +//==== Request Body + +==== Authorization + +To use this API, you must have at least the `manage_security` cluster privilege. + + +==== Examples + +To retrieve a native user, submit a GET request to the `/_xpack/security/user/` +endpoint: + +[source,js] +-------------------------------------------------- +GET /_xpack/security/user/jacknich +-------------------------------------------------- +// CONSOLE +// TEST[setup:jacknich_user] + +A successful call returns an array of users with the JSON representation of the +user. Note that user passwords are not included. + +[source,js] +-------------------------------------------------- +{ + "jacknich": { + "username": "jacknich", + "roles": [ + "admin", "other_role1" + ], + "full_name": "Jack Nicholson", + "email": "jacknich@example.com", + "metadata": { "intelligence" : 7 }, + "enabled": true + } +} +-------------------------------------------------- +// CONSOLE +// TESTRESPONSE + +If the user is not defined in the `native` realm, the request 404s. + +Omit the username to retrieve all users: + +[source,js] +-------------------------------------------------- +GET /_xpack/security/user +-------------------------------------------------- +// CONSOLE +// TEST[continued] diff --git a/x-pack/docs/en/rest-api/security/users.asciidoc b/x-pack/docs/en/rest-api/security/users.asciidoc deleted file mode 100644 index c84da5c7d75ff..0000000000000 --- a/x-pack/docs/en/rest-api/security/users.asciidoc +++ /dev/null @@ -1,226 +0,0 @@ -[role="xpack"] -[[security-api-users]] -=== User Management APIs - -The `user` API enables you to create, read, update, and delete users from the -`native` realm. These users are commonly referred to as *native users*. - - -==== Request - -`GET /_xpack/security/user` + - -`GET /_xpack/security/user/` + - -`DELETE /_xpack/security/user/` + - -`POST /_xpack/security/user/` + - -`PUT /_xpack/security/user/` + - -`PUT /_xpack/security/user//_disable` + - -`PUT /_xpack/security/user//_enable` + - -`PUT /_xpack/security/user//_password` - - -==== Description - -You can use the PUT user API to create or update users. When updating a user, -you can update everything but its `username` and `password`. To change a user's -password, use the <>. - -[[username-validation]] -NOTE: Usernames must be at least 1 and no more than 1024 characters. They can -contain alphanumeric characters (`a-z`, `A-Z`, `0-9`), spaces, punctuation, and -printable symbols in the https://en.wikipedia.org/wiki/Basic_Latin_(Unicode_block)[Basic Latin (ASCII) block]. -Leading or trailing whitespace is not allowed. - -==== Path Parameters - -`username`:: - (string) An identifier for the user. If you omit this parameter from a Get - User API request, it retrieves information about all users. - - -==== Request Body - -The following parameters can be specified in the body of a POST or PUT request -and pertain to creating a user: - -`enabled`:: -(boolean) Specifies whether the user is enabled. The default value is `true`. - -`email`:: -(string) The email of the user. - -`full_name`:: -(string) The full name of the user. - -`metadata`:: -(object) Arbitrary metadata that you want to associate with the user. - -`password` (required):: -(string) The user's password. Passwords must be at least 6 characters long. - -`roles` (required):: -(list) A set of roles the user has. The roles determine the user's access -permissions. To create a user without any roles, specify an empty list: `[]`. - -==== Authorization - -To use this API, you must have at least the `manage_security` cluster privilege. - - -==== Examples - -[[security-api-put-user]] -To add a user, submit a PUT or POST request to the `/_xpack/security/user/` -endpoint. - -[source,js] --------------------------------------------------- -POST /_xpack/security/user/jacknich -{ - "password" : "j@rV1s", - "roles" : [ "admin", "other_role1" ], - "full_name" : "Jack Nicholson", - "email" : "jacknich@example.com", - "metadata" : { - "intelligence" : 7 - } -} --------------------------------------------------- -// CONSOLE - -A successful call returns a JSON structure that shows whether the user has been -created or updated. - -[source,js] --------------------------------------------------- -{ - "user": { - "created" : true <1> - } -} --------------------------------------------------- -// TESTRESPONSE -<1> When an existing user is updated, `created` is set to false. - -After you add a user through the Users API, requests from that user can be -authenticated. For example: - -[source,shell] --------------------------------------------------- -curl -u jacknich:j@rV1s http://localhost:9200/_cluster/health --------------------------------------------------- -// NOTCONSOLE - -[[security-api-get-user]] -To retrieve a native user, submit a GET request to the `/_xpack/security/user/` -endpoint: - -[source,js] --------------------------------------------------- -GET /_xpack/security/user/jacknich --------------------------------------------------- -// CONSOLE -// TEST[continued] - -A successful call returns an array of users with the JSON representation of the -user. Note that user passwords are not included. - -[source,js] --------------------------------------------------- -{ - "jacknich": { <1> - "username" : "jacknich", - "roles" : [ "admin", "other_role1" ], - "full_name" : "Jack Nicholson", - "email" : "jacknich@example.com", - "enabled": true, - "metadata" : { - "intelligence" : 7 - } - } -} --------------------------------------------------- -// TESTRESPONSE -<1> If the user is not defined in the `native` realm, the request 404s. - -You can specify multiple usernames as a comma-separated list: - -[source,js] --------------------------------------------------- -GET /_xpack/security/user/jacknich,rdinero --------------------------------------------------- -// CONSOLE -// TEST[continued] - -Omit the username to retrieve all users: - -[source,js] --------------------------------------------------- -GET /_xpack/security/user --------------------------------------------------- -// CONSOLE -// TEST[continued] - -[[security-api-reset-user-password]] -To reset the password for a user, submit a PUT request to the -`/_xpack/security/user//_password` endpoint: - -[source,js] --------------------------------------------------- -PUT /_xpack/security/user/jacknich/_password -{ - "password" : "s3cr3t" -} --------------------------------------------------- -// CONSOLE -// TEST[continued] - -[[security-api-disable-user]] -To disable a user, submit a PUT request to the -`/_xpack/security/user//_disable` endpoint: - -[source,js] --------------------------------------------------- -PUT /_xpack/security/user/jacknich/_disable --------------------------------------------------- -// CONSOLE -// TEST[continued] - -[[security-api-enable-user]] -To enable a user, submit a PUT request to the -`/_xpack/security/user//_enable` endpoint: - -[source,js] --------------------------------------------------- -PUT /_xpack/security/user/jacknich/_enable --------------------------------------------------- -// CONSOLE -// TEST[continued] - -[[security-api-delete-user]] -To delete a user, submit a DELETE request to the `/_xpack/security/user/` -endpoint: - -[source,js] --------------------------------------------------- -DELETE /_xpack/security/user/jacknich --------------------------------------------------- -// CONSOLE -// TEST[continued] - -If the user is successfully deleted, the request returns `{"found": true}`. -Otherwise, `found` is set to false. - -[source,js] --------------------------------------------------- -{ - "found" : true -} --------------------------------------------------- -// TESTRESPONSE diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.delete_user.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.delete_user.json index d72c854a69dcb..fa1deb3e1ec13 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.delete_user.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.delete_user.json @@ -1,6 +1,6 @@ { "xpack.security.delete_user": { - "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-users.html#security-api-delete-user", + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-delete-user.html", "methods": [ "DELETE" ], "url": { "path": "/_xpack/security/user/{username}", diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.disable_user.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.disable_user.json index 3a72b3141911f..0e55e82ead628 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.disable_user.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.disable_user.json @@ -1,6 +1,6 @@ { "xpack.security.disable_user": { - "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-users.html#security-api-disable-user", + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-disable-user.html", "methods": [ "PUT", "POST" ], "url": { "path": "/_xpack/security/user/{username}/_disable", diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.enable_user.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.enable_user.json index c68144957f07d..da2f67adbea37 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.enable_user.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.enable_user.json @@ -1,6 +1,6 @@ { "xpack.security.enable_user": { - "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-users.html#security-api-enable-user", + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-enable-user.html", "methods": [ "PUT", "POST" ], "url": { "path": "/_xpack/security/user/{username}/_enable", diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.get_user.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.get_user.json index 910fb7d064582..94dcbca81e18e 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.get_user.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.get_user.json @@ -1,6 +1,6 @@ { "xpack.security.get_user": { - "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-users.html#security-api-get-user", + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-user.html", "methods": [ "GET" ], "url": { "path": "/_xpack/security/user/{username}", diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_user.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_user.json index de07498a40954..1b51783a05ef5 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_user.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_user.json @@ -1,6 +1,6 @@ { "xpack.security.put_user": { - "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-users.html#security-api-put-user", + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-put-user.html", "methods": [ "PUT", "POST" ], "url": { "path": "/_xpack/security/user/{username}",