From 7420fd0be357f61a0c4fd5c664010a2e7b1c015e Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Fri, 4 Nov 2016 09:46:28 -0600 Subject: [PATCH] Add "all fields" execution mode to simple_query_string query This commit introduces a new execution mode for the `simple_query_string` query, which is intended down the road to be a replacement for the current _all field. It now does auto-field-expansion and auto-leniency when the following criteria are ALL met: The _all field is disabled No default_field has been set in the index settings No fields are specified in the request Additionally, a user can force the "all-like" execution by setting the all_fields parameter to true. When executing in all field mode, the `simple_query_string` query will look at all the fields in the mapping that are not metafields and can be searched, and automatically expand the list of fields that are going to be queried. Relates to #20925, which is the `query_string` version of this work. This is basically the same behavior, but for the `simple_query_string` query. Relates to #19784 --- .../index/query/QueryStringQueryBuilder.java | 12 +- .../index/query/SimpleQueryParser.java | 6 + .../index/query/SimpleQueryStringBuilder.java | 73 ++++-- .../query/SimpleQueryStringBuilderTests.java | 25 +- .../search/query/QueryStringIT.java | 47 ++-- .../search/query/SimpleQueryStringIT.java | 220 ++++++++++++++++++ .../query/all-query-index-with-all.json | 35 +++ .../simple-query-string-query.asciidoc | 10 +- 8 files changed, 389 insertions(+), 39 deletions(-) create mode 100644 core/src/test/resources/org/elasticsearch/search/query/all-query-index-with-all.json diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java index eb508b9d4a4b2..08318874df210 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java @@ -119,7 +119,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder ALLOWED_QUERY_MAPPER_TYPES; + public static final Set ALLOWED_QUERY_MAPPER_TYPES; static { ALLOWED_QUERY_MAPPER_TYPES = new HashSet<>(); @@ -908,7 +908,11 @@ protected int doHashCode() { timeZone == null ? 0 : timeZone.getID(), escape, maxDeterminizedStates, splitOnWhitespace, useAllFields); } - private Map allQueryableDefaultFields(QueryShardContext context) { + /** + * Given a shard context, return a map of all fields in the mappings that + * can be queried. The map will be field name to a float of 1.0f. + */ + public static Map allQueryableDefaultFields(QueryShardContext context) { Collection allFields = context.simpleMatchToIndexNames("*"); Map fields = new HashMap<>(); for (String fieldName : allFields) { @@ -943,6 +947,10 @@ protected Query doToQuery(QueryShardContext context) throws IOException { Map resolvedFields = new TreeMap<>(); + if ((useAllFields != null && useAllFields) && (fieldsAndWeights.size() != 0 || this.defaultField != null)) { + throw addValidationError("cannot use [all_fields] parameter in conjunction with [default_field] or [fields]", null); + } + // If explicitly required to use all fields, use all fields, OR: // Automatically determine the fields (to replace the _all field) if all of the following are true: // - The _all field is disabled, diff --git a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java index 67d0d445113c6..4a49405ec2f33 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java @@ -270,6 +270,12 @@ static class Settings { public Settings() { } + public Settings(Settings other) { + this.lenient = other.lenient; + this.analyzeWildcard = other.analyzeWildcard; + this.quoteFieldSuffix = other.quoteFieldSuffix; + } + /** Specifies whether to use lenient parsing, defaults to false. */ public void lenient(boolean lenient) { this.lenient = lenient; diff --git a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java index fd29707506744..5bc04d13f8bbe 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java @@ -106,6 +106,7 @@ public class SimpleQueryStringBuilder extends AbstractQueryBuilder resolvedFieldsAndWeights = new TreeMap<>(); - // Use the default field if no fields specified - if (fieldsAndWeights.isEmpty()) { - resolvedFieldsAndWeights.put(resolveIndexName(context.defaultField(), context), AbstractQueryBuilder.DEFAULT_BOOST); + + if ((useAllFields != null && useAllFields) && (fieldsAndWeights.size() != 0)) { + throw addValidationError("cannot use [all_fields] parameter in conjunction with [fields]", null); + } + + // If explicitly required to use all fields, use all fields, OR: + // Automatically determine the fields (to replace the _all field) if all of the following are true: + // - The _all field is disabled, + // - and the default_field has not been changed in the settings + // - and no fields are specified in the request + Settings newSettings = new Settings(settings); + if ((this.useAllFields != null && this.useAllFields) || + (context.getMapperService().allEnabled() == false && + "_all".equals(context.defaultField()) && + this.fieldsAndWeights.isEmpty())) { + resolvedFieldsAndWeights = QueryStringQueryBuilder.allQueryableDefaultFields(context); + // Need to use lenient mode when using "all-mode" so exceptions aren't thrown due to mismatched types + newSettings.lenient(true); } else { - for (Map.Entry fieldEntry : fieldsAndWeights.entrySet()) { - if (Regex.isSimpleMatchPattern(fieldEntry.getKey())) { - for (String fieldName : context.getMapperService().simpleMatchToIndexNames(fieldEntry.getKey())) { - resolvedFieldsAndWeights.put(fieldName, fieldEntry.getValue()); + // Use the default field if no fields specified + if (fieldsAndWeights.isEmpty()) { + resolvedFieldsAndWeights.put(resolveIndexName(context.defaultField(), context), AbstractQueryBuilder.DEFAULT_BOOST); + } else { + for (Map.Entry fieldEntry : fieldsAndWeights.entrySet()) { + if (Regex.isSimpleMatchPattern(fieldEntry.getKey())) { + for (String fieldName : context.getMapperService().simpleMatchToIndexNames(fieldEntry.getKey())) { + resolvedFieldsAndWeights.put(fieldName, fieldEntry.getValue()); + } + } else { + resolvedFieldsAndWeights.put(resolveIndexName(fieldEntry.getKey(), context), fieldEntry.getValue()); } - } else { - resolvedFieldsAndWeights.put(resolveIndexName(fieldEntry.getKey(), context), fieldEntry.getValue()); } } } @@ -369,7 +403,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException { } - SimpleQueryParser sqp = new SimpleQueryParser(luceneAnalyzer, resolvedFieldsAndWeights, flags, settings, context); + SimpleQueryParser sqp = new SimpleQueryParser(luceneAnalyzer, resolvedFieldsAndWeights, flags, newSettings, context); sqp.setDefaultOperator(defaultOperator.toBooleanClauseOccur()); Query query = sqp.parse(queryText); @@ -419,6 +453,9 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep if (minimumShouldMatch != null) { builder.field(MINIMUM_SHOULD_MATCH_FIELD.getPreferredName(), minimumShouldMatch); } + if (useAllFields != null) { + builder.field(ALL_FIELDS_FIELD.getPreferredName(), useAllFields); + } printBoostAndQueryName(builder); builder.endObject(); @@ -439,6 +476,7 @@ public static Optional fromXContent(QueryParseContext boolean lenient = SimpleQueryStringBuilder.DEFAULT_LENIENT; boolean analyzeWildcard = SimpleQueryStringBuilder.DEFAULT_ANALYZE_WILDCARD; String quoteFieldSuffix = null; + Boolean useAllFields = null; XContentParser.Token token; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { @@ -502,6 +540,8 @@ public static Optional fromXContent(QueryParseContext minimumShouldMatch = parser.textOrNull(); } else if (parseContext.getParseFieldMatcher().match(currentFieldName, QUOTE_FIELD_SUFFIX_FIELD)) { quoteFieldSuffix = parser.textOrNull(); + } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ALL_FIELDS_FIELD)) { + useAllFields = parser.booleanValue(); } else { throw new ParsingException(parser.getTokenLocation(), "[" + SimpleQueryStringBuilder.NAME + "] unsupported field [" + parser.currentName() + "]"); @@ -517,10 +557,16 @@ public static Optional fromXContent(QueryParseContext throw new ParsingException(parser.getTokenLocation(), "[" + SimpleQueryStringBuilder.NAME + "] query text missing"); } + if ((useAllFields != null && useAllFields) && (fieldsAndWeights.size() != 0)) { + throw new ParsingException(parser.getTokenLocation(), + "cannot use [all_fields] parameter in conjunction with [fields]"); + } + SimpleQueryStringBuilder qb = new SimpleQueryStringBuilder(queryBody); qb.boost(boost).fields(fieldsAndWeights).analyzer(analyzerName).queryName(queryName).minimumShouldMatch(minimumShouldMatch); qb.flags(flags).defaultOperator(defaultOperator); qb.lenient(lenient).analyzeWildcard(analyzeWildcard).boost(boost).quoteFieldSuffix(quoteFieldSuffix); + qb.useAllFields(useAllFields); return Optional.of(qb); } @@ -531,7 +577,7 @@ public String getWriteableName() { @Override protected int doHashCode() { - return Objects.hash(fieldsAndWeights, analyzer, defaultOperator, queryText, minimumShouldMatch, settings, flags); + return Objects.hash(fieldsAndWeights, analyzer, defaultOperator, queryText, minimumShouldMatch, settings, flags, useAllFields); } @Override @@ -539,7 +585,8 @@ protected boolean doEquals(SimpleQueryStringBuilder other) { return Objects.equals(fieldsAndWeights, other.fieldsAndWeights) && Objects.equals(analyzer, other.analyzer) && Objects.equals(defaultOperator, other.defaultOperator) && Objects.equals(queryText, other.queryText) && Objects.equals(minimumShouldMatch, other.minimumShouldMatch) - && Objects.equals(settings, other.settings) && (flags == other.flags); + && Objects.equals(settings, other.settings) + && (flags == other.flags) + && (useAllFields == other.useAllFields); } } - diff --git a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java index a8a1de059e882..98c6ac00344c4 100644 --- a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java @@ -31,6 +31,8 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.TestUtil; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.test.AbstractQueryTestCase; @@ -42,6 +44,7 @@ import java.util.Map; import java.util.Set; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.instanceOf; @@ -252,7 +255,12 @@ protected void doAssertLuceneQuery(SimpleQueryStringBuilder queryBuilder, Query Map.Entry field = queryBuilder.fields().entrySet().iterator().next(); assertTermOrBoostQuery(query, field.getKey(), queryBuilder.value(), field.getValue()); } else if (queryBuilder.fields().size() == 0) { - assertTermQuery(query, MetaData.ALL, queryBuilder.value()); + MapperService ms = context.mapperService(); + if (ms.allEnabled()) { + assertTermQuery(query, MetaData.ALL, queryBuilder.value()); + } else { + assertThat(query.getClass(), equalTo(MatchNoDocsQuery.class)); + } } else { fail("Encountered lucene query type we do not have a validation implementation for in our " + SimpleQueryStringBuilderTests.class.getSimpleName()); @@ -398,4 +406,19 @@ public void testExpandedTerms() throws Exception { expected = new FuzzyQuery(new Term(STRING_FIELD_NAME, "abc"), 1); assertEquals(expected, query); } + + public void testAllFieldsWithFields() throws IOException { + String json = + "{\n" + + " \"simple_query_string\" : {\n" + + " \"query\" : \"this that thus\",\n" + + " \"fields\" : [\"foo\"],\n" + + " \"all_fields\" : true\n" + + " }\n" + + "}"; + + ParsingException e = expectThrows(ParsingException.class, () -> parseQuery(json)); + assertThat(e.getMessage(), + containsString("cannot use [all_fields] parameter in conjunction with [fields]")); + } } diff --git a/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java b/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java index b1b0da73c357d..79dea0e74b54b 100644 --- a/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.query; import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; @@ -27,6 +28,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryStringQueryBuilder; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -201,33 +203,36 @@ public void testKeywordWithWhitespace() throws Exception { assertHits(resp.getHits(), "2", "3"); assertHitCount(resp, 2L); - // Will be fixed once https://github.com/elastic/elasticsearch/pull/20965 is in - // resp = client().prepareSearch("test") - // .setQuery(queryStringQuery("Foo Bar").splitOnWhitespcae(false)) - // .get(); - // assertHits(resp.getHits(), "1", "2", "3"); - // assertHitCount(resp, 3L); + resp = client().prepareSearch("test") + .setQuery(queryStringQuery("Foo Bar").splitOnWhitespace(false)) + .get(); + assertHits(resp.getHits(), "1", "2", "3"); + assertHitCount(resp, 3L); } public void testExplicitAllFieldsRequested() throws Exception { + String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index-with-all.json"); + prepareCreate("test2").setSource(indexBody).get(); + ensureGreen("test2"); + List reqs = new ArrayList<>(); - reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo", - "f_date", "2015/09/02", - "f_float", "1.7", - "f_ip", "127.0.0.1")); - reqs.add(client().prepareIndex("test", "doc", "2").setSource("f1", "bar", - "f_date", "2015/09/01", - "f_float", "1.8", - "f_ip", "127.0.0.2")); + reqs.add(client().prepareIndex("test2", "doc", "1").setSource("f1", "foo", "f2", "eggplant")); indexRandom(true, false, reqs); - SearchResponse resp = client().prepareSearch("test").setQuery( - queryStringQuery("127.0.0.2 \"2015/09/02\"") - .field("f_ip") // Usually this would mean we wouldn't search "all" fields - .useAllFields(true)) // ... unless explicitly requested - .get(); - assertHits(resp.getHits(), "1", "2"); - assertHitCount(resp, 2L); + SearchResponse resp = client().prepareSearch("test2").setQuery( + queryStringQuery("foo eggplent").defaultOperator(Operator.AND)).get(); + assertHitCount(resp, 0L); + + resp = client().prepareSearch("test2").setQuery( + queryStringQuery("foo eggplent").defaultOperator(Operator.AND).useAllFields(true)).get(); + assertHits(resp.getHits(), "1"); + assertHitCount(resp, 1L); + + Exception e = expectThrows(Exception.class, () -> + client().prepareSearch("test2").setQuery( + queryStringQuery("blah").field("f1").useAllFields(true)).get()); + assertThat(ExceptionsHelper.detailedMessage(e), + containsString("cannot use [all_fields] parameter in conjunction with [default_field] or [fields]")); } @LuceneTestCase.AwaitsFix(bugUrl="currently can't perform phrase queries on fields that don't support positions") diff --git a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java index b98bc5d43cda8..60f89ab326eb7 100644 --- a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java @@ -19,23 +19,33 @@ package org.elasticsearch.search.query; +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.SimpleQueryStringFlag; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import java.util.concurrent.ExecutionException; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFirstHit; @@ -43,6 +53,8 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; /** @@ -352,4 +364,212 @@ public void testEmptySimpleQueryStringWithAnalysis() throws Exception { assertNoFailures(searchResponse); assertHitCount(searchResponse, 0L); } + + public void testBasicAllQuery() throws Exception { + String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json"); + prepareCreate("test").setSource(indexBody).get(); + ensureGreen("test"); + + List reqs = new ArrayList<>(); + reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo bar baz")); + reqs.add(client().prepareIndex("test", "doc", "2").setSource("f2", "Bar")); + reqs.add(client().prepareIndex("test", "doc", "3").setSource("f3", "foo bar baz")); + indexRandom(true, false, reqs); + + SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get(); + assertHitCount(resp, 2L); + assertHits(resp.getHits(), "1", "3"); + + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("bar")).get(); + assertHitCount(resp, 2L); + assertHits(resp.getHits(), "1", "3"); + + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("Bar")).get(); + assertHitCount(resp, 3L); + assertHits(resp.getHits(), "1", "2", "3"); + + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foa")).get(); + assertHitCount(resp, 1L); + assertHits(resp.getHits(), "3"); + } + + public void testWithDate() throws Exception { + String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json"); + prepareCreate("test").setSource(indexBody).get(); + ensureGreen("test"); + + List reqs = new ArrayList<>(); + reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo", "f_date", "2015/09/02")); + reqs.add(client().prepareIndex("test", "doc", "2").setSource("f1", "bar", "f_date", "2015/09/01")); + indexRandom(true, false, reqs); + + SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo bar")).get(); + assertHits(resp.getHits(), "1", "2"); + assertHitCount(resp, 2L); + + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("\"2015/09/02\"")).get(); + assertHits(resp.getHits(), "1"); + assertHitCount(resp, 1L); + + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("bar \"2015/09/02\"")).get(); + assertHits(resp.getHits(), "1", "2"); + assertHitCount(resp, 2L); + + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("\"2015/09/02\" \"2015/09/01\"")).get(); + assertHits(resp.getHits(), "1", "2"); + assertHitCount(resp, 2L); + } + + public void testWithLotsOfTypes() throws Exception { + String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json"); + prepareCreate("test").setSource(indexBody).get(); + ensureGreen("test"); + + List reqs = new ArrayList<>(); + reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo", + "f_date", "2015/09/02", + "f_float", "1.7", + "f_ip", "127.0.0.1")); + reqs.add(client().prepareIndex("test", "doc", "2").setSource("f1", "bar", + "f_date", "2015/09/01", + "f_float", "1.8", + "f_ip", "127.0.0.2")); + indexRandom(true, false, reqs); + + SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo bar")).get(); + assertHits(resp.getHits(), "1", "2"); + assertHitCount(resp, 2L); + + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("\"2015/09/02\"")).get(); + assertHits(resp.getHits(), "1"); + assertHitCount(resp, 1L); + + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("127.0.0.2 \"2015/09/02\"")).get(); + assertHits(resp.getHits(), "1", "2"); + assertHitCount(resp, 2L); + + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("127.0.0.1 1.8")).get(); + assertHits(resp.getHits(), "1", "2"); + assertHitCount(resp, 2L); + } + + public void testDocWithAllTypes() throws Exception { + String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json"); + prepareCreate("test").setSource(indexBody).get(); + ensureGreen("test"); + + List reqs = new ArrayList<>(); + String docBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-example-document.json"); + reqs.add(client().prepareIndex("test", "doc", "1").setSource(docBody)); + indexRandom(true, false, reqs); + + SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get(); + assertHits(resp.getHits(), "1"); + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("Bar")).get(); + assertHits(resp.getHits(), "1"); + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("Baz")).get(); + assertHits(resp.getHits(), "1"); + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("sbaz")).get(); + assertHits(resp.getHits(), "1"); + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("19")).get(); + assertHits(resp.getHits(), "1"); + // nested doesn't match because it's hidden + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("1476383971")).get(); + assertHits(resp.getHits(), "1"); + // bool doesn't match + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("7")).get(); + assertHits(resp.getHits(), "1"); + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("23")).get(); + assertHits(resp.getHits(), "1"); + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("1293")).get(); + assertHits(resp.getHits(), "1"); + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("42")).get(); + assertHits(resp.getHits(), "1"); + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("1.7")).get(); + assertHits(resp.getHits(), "1"); + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("1.5")).get(); + assertHits(resp.getHits(), "1"); + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("12.23")).get(); + assertHits(resp.getHits(), "1"); + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("127.0.0.1")).get(); + assertHits(resp.getHits(), "1"); + // binary doesn't match + // suggest doesn't match + // geo_point doesn't match + // geo_shape doesn't match + + resp = client().prepareSearch("test").setQuery( + simpleQueryStringQuery("foo Bar 19 127.0.0.1").defaultOperator(Operator.AND)).get(); + assertHits(resp.getHits(), "1"); + } + + public void testKeywordWithWhitespace() throws Exception { + String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json"); + prepareCreate("test").setSource(indexBody).get(); + ensureGreen("test"); + + List reqs = new ArrayList<>(); + reqs.add(client().prepareIndex("test", "doc", "1").setSource("f2", "Foo Bar")); + reqs.add(client().prepareIndex("test", "doc", "2").setSource("f1", "bar")); + reqs.add(client().prepareIndex("test", "doc", "3").setSource("f1", "foo bar")); + indexRandom(true, false, reqs); + + SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get(); + assertHits(resp.getHits(), "3"); + assertHitCount(resp, 1L); + + resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("bar")).get(); + assertHits(resp.getHits(), "2", "3"); + assertHitCount(resp, 2L); + } + + public void testExplicitAllFieldsRequested() throws Exception { + String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index-with-all.json"); + prepareCreate("test").setSource(indexBody).get(); + ensureGreen("test"); + + List reqs = new ArrayList<>(); + reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo", "f2", "eggplant")); + indexRandom(true, false, reqs); + + SearchResponse resp = client().prepareSearch("test").setQuery( + simpleQueryStringQuery("foo eggplent").defaultOperator(Operator.AND)).get(); + assertHitCount(resp, 0L); + + resp = client().prepareSearch("test").setQuery( + simpleQueryStringQuery("foo eggplent").defaultOperator(Operator.AND).useAllFields(true)).get(); + assertHits(resp.getHits(), "1"); + assertHitCount(resp, 1L); + + Exception e = expectThrows(Exception.class, () -> + client().prepareSearch("test").setQuery( + simpleQueryStringQuery("blah").field("f1").useAllFields(true)).get()); + assertThat(ExceptionsHelper.detailedMessage(e), + containsString("cannot use [all_fields] parameter in conjunction with [fields]")); + } + + @LuceneTestCase.AwaitsFix(bugUrl="currently can't perform phrase queries on fields that don't support positions") + public void testPhraseQueryOnFieldWithNoPositions() throws Exception { + String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json"); + prepareCreate("test").setSource(indexBody).get(); + ensureGreen("test"); + + List reqs = new ArrayList<>(); + reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo bar", "f4", "eggplant parmesan")); + reqs.add(client().prepareIndex("test", "doc", "2").setSource("f1", "foo bar", "f4", "chicken parmesan")); + indexRandom(true, false, reqs); + + SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("\"eggplant parmesan\"")).get(); + assertHits(resp.getHits(), "1"); + assertHitCount(resp, 1L); + } + + private void assertHits(SearchHits hits, String... ids) { + assertThat(hits.totalHits(), equalTo((long) ids.length)); + Set hitIds = new HashSet<>(); + for (SearchHit hit : hits.getHits()) { + hitIds.add(hit.id()); + } + assertThat(hitIds, containsInAnyOrder(ids)); + } } diff --git a/core/src/test/resources/org/elasticsearch/search/query/all-query-index-with-all.json b/core/src/test/resources/org/elasticsearch/search/query/all-query-index-with-all.json new file mode 100644 index 0000000000000..05de13b4261a2 --- /dev/null +++ b/core/src/test/resources/org/elasticsearch/search/query/all-query-index-with-all.json @@ -0,0 +1,35 @@ +{ + "settings": { + "index": { + "number_of_shards": 1, + "number_of_replicas": 0, + "analysis": { + "analyzer": { + "my_ngrams": { + "type": "custom", + "tokenizer": "standard", + "filter": ["my_ngrams"] + } + }, + "filter": { + "my_ngrams": { + "type": "ngram", + "min_gram": 2, + "max_gram": 2 + } + } + } + } + }, + "mappings": { + "doc": { + "_all": { + "enabled": true + }, + "properties": { + "f1": {"type": "text"}, + "f2": {"type": "text", "analyzer": "my_ngrams"} + } + } + } +} diff --git a/docs/reference/query-dsl/simple-query-string-query.asciidoc b/docs/reference/query-dsl/simple-query-string-query.asciidoc index c6f70c314160e..c67ba5cd73e5c 100644 --- a/docs/reference/query-dsl/simple-query-string-query.asciidoc +++ b/docs/reference/query-dsl/simple-query-string-query.asciidoc @@ -61,6 +61,10 @@ based just on the prefix of a term. Defaults to `false`. the query string. This allows to use a field that has a different analysis chain for exact matching. Look <> for a comprehensive example. + +|`all_fields` | Perform the query on all fields detected in the mapping that can +be queried. Will be used by default when the `_all` field is disabled and no +`default_field` is specified index settings, and no `fields` are specified. |======================================================================= [float] @@ -85,8 +89,10 @@ When not explicitly specifying the field to search on in the query string syntax, the `index.query.default_field` will be used to derive which field to search on. It defaults to `_all` field. -So, if `_all` field is disabled, it might make sense to change it to set -a different default field. +If the `_all` field is disabled and no `fields` are specified in the request`, +the `simple_query_string` query will automatically attempt to determine the +existing fields in the index's mapping that are queryable, and perform the +search on those fields. [float] ==== Multi Field