Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add "all field" execution mode to query_string query #20925

Merged
merged 1 commit into from
Nov 4, 2016
Merged

Add "all field" execution mode to query_string query #20925

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ public enum MergeReason {
private volatile FieldTypeLookup fieldTypes;
private volatile Map<String, ObjectMapper> fullPathObjectMappers = new HashMap<>();
private boolean hasNested = false; // updated dynamically to true when a nested object is added
private boolean allEnabled = false; // updated dynamically to true when _all is enabled

private final DocumentMapperParser documentParser;

Expand Down Expand Up @@ -150,6 +151,13 @@ public boolean hasNested() {
return this.hasNested;
}

/**
* Returns true if the "_all" field is enabled for the type
*/
public boolean allEnabled() {
return this.allEnabled;
}

/**
* returns an immutable iterator over current document mappers.
*
Expand Down Expand Up @@ -368,6 +376,7 @@ private synchronized DocumentMapper merge(DocumentMapper mapper, MergeReason rea
this.hasNested = hasNested;
this.fullPathObjectMappers = fullPathObjectMappers;
this.parentTypes = parentTypes;
this.allEnabled = mapper.allFieldMapper().enabled();

assert assertSerialization(newMapper);
assert assertMappersShareSameFieldType();
Expand Down
4 changes: 4 additions & 0 deletions core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,10 @@ public void setIsFilter(boolean isFilter) {
this.isFilter = isFilter;
}

/**
* Returns all the fields that match a given pattern. If prefixed with a
* type then the fields will be returned with a type prefix.
*/
public Collection<String> simpleMatchToIndexNames(String pattern) {
return mapperService.simpleMatchToIndexNames(pattern);
}
Expand Down
129 changes: 117 additions & 12 deletions core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,17 +37,30 @@
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.DateFieldMapper;
import org.elasticsearch.index.mapper.IpFieldMapper;
import org.elasticsearch.index.mapper.KeywordFieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.NumberFieldMapper;
import org.elasticsearch.index.mapper.ScaledFloatFieldMapper;
import org.elasticsearch.index.mapper.StringFieldMapper;
import org.elasticsearch.index.mapper.TextFieldMapper;
import org.elasticsearch.index.mapper.TimestampFieldMapper;
import org.elasticsearch.index.query.support.QueryParsers;
import org.joda.time.DateTimeZone;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;

/**
Expand Down Expand Up @@ -103,6 +116,24 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
.withAllDeprecated("Decision is now made by the analyzer");
private static final ParseField TIME_ZONE_FIELD = new ParseField("time_zone");
private static final ParseField SPLIT_ON_WHITESPACE = new ParseField("split_on_whitespace");
private static final ParseField ALL_FIELDS_FIELD = new ParseField("all_fields");

// Mapping types the "all-ish" query can be executed against
private static final Set<String> ALLOWED_QUERY_MAPPER_TYPES;

static {
ALLOWED_QUERY_MAPPER_TYPES = new HashSet<>();
ALLOWED_QUERY_MAPPER_TYPES.add(DateFieldMapper.CONTENT_TYPE);
ALLOWED_QUERY_MAPPER_TYPES.add(IpFieldMapper.CONTENT_TYPE);
ALLOWED_QUERY_MAPPER_TYPES.add(KeywordFieldMapper.CONTENT_TYPE);
for (NumberFieldMapper.NumberType nt : NumberFieldMapper.NumberType.values()) {
ALLOWED_QUERY_MAPPER_TYPES.add(nt.typeName());
}
ALLOWED_QUERY_MAPPER_TYPES.add(ScaledFloatFieldMapper.CONTENT_TYPE);
ALLOWED_QUERY_MAPPER_TYPES.add(StringFieldMapper.CONTENT_TYPE);
ALLOWED_QUERY_MAPPER_TYPES.add(TextFieldMapper.CONTENT_TYPE);
ALLOWED_QUERY_MAPPER_TYPES.add(TimestampFieldMapper.CONTENT_TYPE);
}

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that the Legacy field mappers are missing (at least those that are stil valid in 2.x ?).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The legacy field mappers share the same CONTENT_TYPE constant values as all the values in NumberFieldMapper.NumberType.values().

ie, we don't need LegacyLongFieldMapper because LegacyLongFieldMapper.CONTENT_TYPE is "long" and we already add NumberFieldMapper.NumberType.LONG, which is "long"

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah CONTENT_TYPE I see. Sorry for the noise ;)

private final String queryString;

Expand Down Expand Up @@ -156,6 +187,8 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue

private DateTimeZone timeZone;

private Boolean useAllFields;

/** To limit effort spent determinizing regexp queries. */
private int maxDeterminizedStates = DEFAULT_MAX_DETERMINED_STATES;

Expand Down Expand Up @@ -208,6 +241,7 @@ public QueryStringQueryBuilder(StreamInput in) throws IOException {
maxDeterminizedStates = in.readVInt();
if (in.getVersion().onOrAfter(V_5_1_0_UNRELEASED)) {
splitOnWhitespace = in.readBoolean();
useAllFields = in.readOptionalBoolean();
} else {
splitOnWhitespace = DEFAULT_SPLIT_ON_WHITESPACE;
}
Expand Down Expand Up @@ -251,6 +285,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
out.writeVInt(this.maxDeterminizedStates);
if (out.getVersion().onOrAfter(V_5_1_0_UNRELEASED)) {
out.writeBoolean(this.splitOnWhitespace);
out.writeOptionalBoolean(this.useAllFields);
}
}

Expand All @@ -271,6 +306,20 @@ public String defaultField() {
return this.defaultField;
}

/**
* Tell the query_string query to use all fields explicitly, even if _all is
* enabled. If the "default_field" parameter or "fields" are specified, they
* will be ignored.
*/
public QueryStringQueryBuilder useAllFields(Boolean useAllFields) {
this.useAllFields = useAllFields;
return this;
}

public Boolean useAllFields() {
return this.useAllFields;
}

/**
* Adds a field to run the query string against. The field will be associated with the
* default boost of {@link AbstractQueryBuilder#DEFAULT_BOOST}.
Expand Down Expand Up @@ -634,6 +683,9 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
}
builder.field(ESCAPE_FIELD.getPreferredName(), this.escape);
builder.field(SPLIT_ON_WHITESPACE.getPreferredName(), this.splitOnWhitespace);
if (this.useAllFields != null) {
builder.field(ALL_FIELDS_FIELD.getPreferredName(), this.useAllFields);
}
printBoostAndQueryName(builder);
builder.endObject();
}
Expand Down Expand Up @@ -668,6 +720,7 @@ public static Optional<QueryStringQueryBuilder> fromXContent(QueryParseContext p
String fuzzyRewrite = null;
String rewrite = null;
boolean splitOnWhitespace = DEFAULT_SPLIT_ON_WHITESPACE;
Boolean useAllFields = null;
Map<String, Float> fieldsAndWeights = new HashMap<>();
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
Expand Down Expand Up @@ -747,6 +800,8 @@ public static Optional<QueryStringQueryBuilder> fromXContent(QueryParseContext p
lenient = parser.booleanValue();
} else if (parseContext.getParseFieldMatcher().match(currentFieldName, LOCALE_FIELD)) {
// ignore, deprecated setting
} else if (parseContext.getParseFieldMatcher().match(currentFieldName, ALL_FIELDS_FIELD)) {
useAllFields = parser.booleanValue();
} else if (parseContext.getParseFieldMatcher().match(currentFieldName, TIME_ZONE_FIELD)) {
try {
timeZone = parser.text();
Expand All @@ -771,6 +826,12 @@ public static Optional<QueryStringQueryBuilder> fromXContent(QueryParseContext p
throw new ParsingException(parser.getTokenLocation(), "[" + QueryStringQueryBuilder.NAME + "] must be provided with a [query]");
}

if ((useAllFields != null && useAllFields) &&
(defaultField != null || fieldsAndWeights.size() != 0)) {
throw new ParsingException(parser.getTokenLocation(),
"cannot use [all_fields] parameter in conjunction with [default_field] or [fields]");
}

QueryStringQueryBuilder queryStringQuery = new QueryStringQueryBuilder(queryString);
queryStringQuery.fields(fieldsAndWeights);
queryStringQuery.defaultField(defaultField);
Expand Down Expand Up @@ -798,6 +859,7 @@ public static Optional<QueryStringQueryBuilder> fromXContent(QueryParseContext p
queryStringQuery.boost(boost);
queryStringQuery.queryName(queryName);
queryStringQuery.splitOnWhitespace(splitOnWhitespace);
queryStringQuery.useAllFields(useAllFields);
return Optional.of(queryStringQuery);
}

Expand Down Expand Up @@ -833,7 +895,8 @@ protected boolean doEquals(QueryStringQueryBuilder other) {
Objects.equals(timeZone.getID(), other.timeZone.getID()) &&
Objects.equals(escape, other.escape) &&
Objects.equals(maxDeterminizedStates, other.maxDeterminizedStates) &&
Objects.equals(splitOnWhitespace, other.splitOnWhitespace);
Objects.equals(splitOnWhitespace, other.splitOnWhitespace) &&
Objects.equals(useAllFields, other.useAllFields);
}

@Override
Expand All @@ -842,7 +905,29 @@ protected int doHashCode() {
quoteFieldSuffix, autoGeneratePhraseQueries, allowLeadingWildcard, analyzeWildcard,
enablePositionIncrements, fuzziness, fuzzyPrefixLength,
fuzzyMaxExpansions, fuzzyRewrite, phraseSlop, useDisMax, tieBreaker, rewrite, minimumShouldMatch, lenient,
timeZone == null ? 0 : timeZone.getID(), escape, maxDeterminizedStates, splitOnWhitespace);
timeZone == null ? 0 : timeZone.getID(), escape, maxDeterminizedStates, splitOnWhitespace, useAllFields);
}

private Map<String, Float> allQueryableDefaultFields(QueryShardContext context) {
Collection<String> allFields = context.simpleMatchToIndexNames("*");
Map<String, Float> fields = new HashMap<>();
for (String fieldName : allFields) {
if (MapperService.isMetadataField(fieldName)) {
// Ignore our metadata fields
continue;
}
MappedFieldType mft = context.fieldMapper(fieldName);
assert mft != null : "should never have a null mapper for an existing field";

// Ignore fields that are not in the allowed mapper types. Some
// types do not support term queries, and thus we cannot generate
// a special query for them.
String mappingType = mft.typeName();
if (ALLOWED_QUERY_MAPPER_TYPES.contains(mappingType)) {
fields.put(fieldName, 1.0f);
}
}
return fields;
}

@Override
Expand All @@ -855,18 +940,39 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
} else {
qpSettings = new QueryParserSettings(this.queryString);
}
qpSettings.defaultField(this.defaultField == null ? context.defaultField() : this.defaultField);

Map<String, Float> resolvedFields = new TreeMap<>();
for (Map.Entry<String, Float> fieldsEntry : fieldsAndWeights.entrySet()) {
String fieldName = fieldsEntry.getKey();
Float weight = fieldsEntry.getValue();
if (Regex.isSimpleMatchPattern(fieldName)) {
for (String resolvedFieldName : context.getMapperService().simpleMatchToIndexNames(fieldName)) {
resolvedFields.put(resolvedFieldName, weight);

// If explicitly required to use all fields, use all fields, OR:
// Automatically determine the fields (to replace the _all field) if all of the following are true:
// - The _all field is disabled,
// - and the default_field has not been changed in the settings
// - and default_field is not specified in the request
// - and no fields are specified in the request
if ((this.useAllFields != null && this.useAllFields) ||
(context.getMapperService().allEnabled() == false &&
"_all".equals(context.defaultField()) &&
this.defaultField == null &&
this.fieldsAndWeights.size() == 0)) {
// Use the automatically determined expansion of all queryable fields
resolvedFields = allQueryableDefaultFields(context);
// Automatically set leniency to "true" so mismatched fields don't cause exceptions
qpSettings.lenient(true);
} else {
qpSettings.defaultField(this.defaultField == null ? context.defaultField() : this.defaultField);

for (Map.Entry<String, Float> fieldsEntry : fieldsAndWeights.entrySet()) {
String fieldName = fieldsEntry.getKey();
Float weight = fieldsEntry.getValue();
if (Regex.isSimpleMatchPattern(fieldName)) {
for (String resolvedFieldName : context.getMapperService().simpleMatchToIndexNames(fieldName)) {
resolvedFields.put(resolvedFieldName, weight);
}
} else {
resolvedFields.put(fieldName, weight);
}
} else {
resolvedFields.put(fieldName, weight);
}
qpSettings.lenient(lenient == null ? context.queryStringLenient() : lenient);
}
qpSettings.fieldsAndWeights(resolvedFields);
qpSettings.defaultOperator(defaultOperator.toQueryParserOperator());
Expand Down Expand Up @@ -905,7 +1011,6 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
qpSettings.useDisMax(useDisMax);
qpSettings.tieBreaker(tieBreaker);
qpSettings.rewriteMethod(QueryParsers.parseRewriteMethod(context.getParseFieldMatcher(), this.rewrite));
qpSettings.lenient(lenient == null ? context.queryStringLenient() : lenient);
qpSettings.timeZone(timeZone);
qpSettings.maxDeterminizedStates(maxDeterminizedStates);
qpSettings.splitOnWhitespace(splitOnWhitespace);
Expand Down
9 changes: 9 additions & 0 deletions core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.index.mapper.MappedFieldType;
Expand Down Expand Up @@ -256,6 +257,14 @@ static Query blendTerm(QueryShardContext context, BytesRef value, Float commonTe
// of ip addresses and the value can't be parsed, so ignore this
// field
continue;
} catch (ElasticsearchParseException parseException) {
// date fields throw an ElasticsearchParseException with the
// underlying IAE as the cause, ignore this field if that is
// the case
if (parseException.getCause() instanceof IllegalArgumentException) {
continue;
}
throw parseException;
}
float boost = ft.boost;
while (query instanceof BoostQuery) {
Expand Down
1 change: 1 addition & 0 deletions core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
public static final ParseField PROFILE_FIELD = new ParseField("profile");
public static final ParseField SEARCH_AFTER = new ParseField("search_after");
public static final ParseField SLICE = new ParseField("slice");
public static final ParseField ALL_FIELDS_FIELDS = new ParseField("all_fields");

public static SearchSourceBuilder fromXContent(QueryParseContext context, AggregatorParsers aggParsers,
Suggesters suggesters, SearchExtRegistry searchExtRegistry) throws IOException {
Expand Down
30 changes: 29 additions & 1 deletion core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.lucene.all.AllTermQuery;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.search.internal.SearchContext;
Expand Down Expand Up @@ -735,5 +736,32 @@ public void testExpandedTerms() throws Exception {
.toQuery(createShardContext());
assertEquals(new TermRangeQuery(STRING_FIELD_NAME, new BytesRef("abc"), new BytesRef("bcd"), true, true), query);
}


public void testAllFieldsWithFields() throws IOException {
String json =
"{\n" +
" \"query_string\" : {\n" +
" \"query\" : \"this AND that OR thus\",\n" +
" \"fields\" : [\"foo\"],\n" +
" \"all_fields\" : true\n" +
" }\n" +
"}";

ParsingException e = expectThrows(ParsingException.class, () -> parseQuery(json));
assertThat(e.getMessage(),
containsString("cannot use [all_fields] parameter in conjunction with [default_field] or [fields]"));

String json2 =
"{\n" +
" \"query_string\" : {\n" +
" \"query\" : \"this AND that OR thus\",\n" +
" \"default_field\" : \"foo\",\n" +
" \"all_fields\" : true\n" +
" }\n" +
"}";

e = expectThrows(ParsingException.class, () -> parseQuery(json2));
assertThat(e.getMessage(),
containsString("cannot use [all_fields] parameter in conjunction with [default_field] or [fields]"));
}
}
Loading