Skip to content

Commit

Permalink
Multi-level Nested Sort with Filters
Browse files Browse the repository at this point in the history 
Allow multple levels of nested sorting where each level
can have it's own filter.  Backward compatible with
previous single-level nested sort.
  • Loading branch information
@martijnvg
martijnvg committed Aug 30, 2017
1 parent 3d07bce commit 6377afa
Show file tree
Hide file tree
Showing 13 changed files with 824 additions and 158 deletions.
9 changes: 9 additions & 0 deletions core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.index.IndexComponent;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
Expand Down Expand Up @@ -116,6 +117,14 @@ public Nested(BitSetProducer rootFilter, Query innerQuery) {
this.innerQuery = innerQuery;
}

public Query getInnerQuery() {
return innerQuery;
}

public BitSetProducer getRootFilter() {
return rootFilter;
}

/**
* Get a {@link BitDocIdSet} that matches the root documents.
*/
Expand Down
112 changes: 54 additions & 58 deletions core/src/main/java/org/elasticsearch/search/MultiValueMode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -514,42 +514,41 @@ protected long pick(SortedNumericDocValues values) throws IOException {
* NOTE: Calling the returned instance on docs that are not root docs is illegal
* The returned instance can only be evaluate the current and upcoming docs
*/
public NumericDocValues select(final SortedNumericDocValues values, final long missingValue, final BitSet rootDocs, final DocIdSetIterator innerDocs, int maxDoc) throws IOException {
if (rootDocs == null || innerDocs == null) {
public NumericDocValues select(final SortedNumericDocValues values, final long missingValue, final BitSet parentDocs, final DocIdSetIterator childDocs, int maxDoc) throws IOException {
if (parentDocs == null || childDocs == null) {
return select(DocValues.emptySortedNumeric(maxDoc), missingValue);
}

return new AbstractNumericDocValues() {

int lastSeenRootDoc = -1;
int lastSeenParentDoc = -1;
long lastEmittedValue = missingValue;

@Override
public boolean advanceExact(int rootDoc) throws IOException {
assert rootDocs.get(rootDoc) : "can only sort root documents";
assert rootDoc >= lastSeenRootDoc : "can only evaluate current and upcoming root docs";
if (rootDoc == lastSeenRootDoc) {
public boolean advanceExact(int parentDoc) throws IOException {
assert parentDoc >= lastSeenParentDoc : "can only evaluate current and upcoming parent docs";
if (parentDoc == lastSeenParentDoc) {
return true;
} else if (rootDoc == 0) {
} else if (parentDoc == 0) {
lastEmittedValue = missingValue;
return true;
}
final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1);
final int firstNestedDoc;
if (innerDocs.docID() > prevRootDoc) {
firstNestedDoc = innerDocs.docID();
final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1);
final int firstChildDoc;
if (childDocs.docID() > prevParentDoc) {
firstChildDoc = childDocs.docID();
} else {
firstNestedDoc = innerDocs.advance(prevRootDoc + 1);
firstChildDoc = childDocs.advance(prevParentDoc + 1);
}

lastSeenRootDoc = rootDoc;
lastEmittedValue = pick(values, missingValue, innerDocs, firstNestedDoc, rootDoc);
lastSeenParentDoc = parentDoc;
lastEmittedValue = pick(values, missingValue, childDocs, firstChildDoc, parentDoc);
return true;
}

@Override
public int docID() {
return lastSeenRootDoc;
return lastSeenParentDoc;
}

@Override
Expand Down Expand Up @@ -624,33 +623,32 @@ protected double pick(SortedNumericDoubleValues values) throws IOException {
* NOTE: Calling the returned instance on docs that are not root docs is illegal
* The returned instance can only be evaluate the current and upcoming docs
*/
public NumericDoubleValues select(final SortedNumericDoubleValues values, final double missingValue, final BitSet rootDocs, final DocIdSetIterator innerDocs, int maxDoc) throws IOException {
if (rootDocs == null || innerDocs == null) {
public NumericDoubleValues select(final SortedNumericDoubleValues values, final double missingValue, final BitSet parentDocs, final DocIdSetIterator childDocs, int maxDoc) throws IOException {
if (parentDocs == null || childDocs == null) {
return select(FieldData.emptySortedNumericDoubles(), missingValue);
}

return new NumericDoubleValues() {

int lastSeenRootDoc = 0;
int lastSeenParentDoc = 0;
double lastEmittedValue = missingValue;

@Override
public boolean advanceExact(int rootDoc) throws IOException {
assert rootDocs.get(rootDoc) : "can only sort root documents";
assert rootDoc >= lastSeenRootDoc : "can only evaluate current and upcoming root docs";
if (rootDoc == lastSeenRootDoc) {
public boolean advanceExact(int parentDoc) throws IOException {
assert parentDoc >= lastSeenParentDoc : "can only evaluate current and upcoming parent docs";
if (parentDoc == lastSeenParentDoc) {
return true;
}
final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1);
final int firstNestedDoc;
if (innerDocs.docID() > prevRootDoc) {
firstNestedDoc = innerDocs.docID();
final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1);
final int firstChildDoc;
if (childDocs.docID() > prevParentDoc) {
firstChildDoc = childDocs.docID();
} else {
firstNestedDoc = innerDocs.advance(prevRootDoc + 1);
firstChildDoc = childDocs.advance(prevParentDoc + 1);
}

lastSeenRootDoc = rootDoc;
lastEmittedValue = pick(values, missingValue, innerDocs, firstNestedDoc, rootDoc);
lastSeenParentDoc = parentDoc;
lastEmittedValue = pick(values, missingValue, childDocs, firstChildDoc, parentDoc);
return true;
}

Expand Down Expand Up @@ -733,8 +731,8 @@ protected BytesRef pick(SortedBinaryDocValues values) throws IOException {
* NOTE: Calling the returned instance on docs that are not root docs is illegal
* The returned instance can only be evaluate the current and upcoming docs
*/
public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef missingValue, final BitSet rootDocs, final DocIdSetIterator innerDocs, int maxDoc) throws IOException {
if (rootDocs == null || innerDocs == null) {
public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef missingValue, final BitSet parentDocs, final DocIdSetIterator childDocs, int maxDoc) throws IOException {
if (parentDocs == null || childDocs == null) {
return select(FieldData.emptySortedBinary(), missingValue);
}
final BinaryDocValues selectedValues = select(values, null);
Expand All @@ -743,27 +741,26 @@ public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef

final BytesRefBuilder builder = new BytesRefBuilder();

int lastSeenRootDoc = 0;
int lastSeenParentDoc = 0;
BytesRef lastEmittedValue = missingValue;

@Override
public boolean advanceExact(int rootDoc) throws IOException {
assert rootDocs.get(rootDoc) : "can only sort root documents";
assert rootDoc >= lastSeenRootDoc : "can only evaluate current and upcoming root docs";
if (rootDoc == lastSeenRootDoc) {
public boolean advanceExact(int parentDoc) throws IOException {
assert parentDoc >= lastSeenParentDoc : "can only evaluate current and upcoming root docs";
if (parentDoc == lastSeenParentDoc) {
return true;
}

final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1);
final int firstNestedDoc;
if (innerDocs.docID() > prevRootDoc) {
firstNestedDoc = innerDocs.docID();
final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1);
final int firstChildDoc;
if (childDocs.docID() > prevParentDoc) {
firstChildDoc = childDocs.docID();
} else {
firstNestedDoc = innerDocs.advance(prevRootDoc + 1);
firstChildDoc = childDocs.advance(prevParentDoc + 1);
}

lastSeenRootDoc = rootDoc;
lastEmittedValue = pick(selectedValues, builder, innerDocs, firstNestedDoc, rootDoc);
lastSeenParentDoc = parentDoc;
lastEmittedValue = pick(selectedValues, builder, childDocs, firstChildDoc, parentDoc);
if (lastEmittedValue == null) {
lastEmittedValue = missingValue;
}
Expand Down Expand Up @@ -850,16 +847,16 @@ protected int pick(SortedSetDocValues values) throws IOException {
* NOTE: Calling the returned instance on docs that are not root docs is illegal
* The returned instance can only be evaluate the current and upcoming docs
*/
public SortedDocValues select(final SortedSetDocValues values, final BitSet rootDocs, final DocIdSetIterator innerDocs) throws IOException {
if (rootDocs == null || innerDocs == null) {
public SortedDocValues select(final SortedSetDocValues values, final BitSet parentDocs, final DocIdSetIterator childDocs) throws IOException {
if (parentDocs == null || childDocs == null) {
return select(DocValues.emptySortedSet());
}
final SortedDocValues selectedValues = select(values);

return new AbstractSortedDocValues() {

int docID = -1;
int lastSeenRootDoc = 0;
int lastSeenParentDoc = 0;
int lastEmittedOrd = -1;

@Override
Expand All @@ -873,23 +870,22 @@ public int getValueCount() {
}

@Override
public boolean advanceExact(int rootDoc) throws IOException {
assert rootDocs.get(rootDoc) : "can only sort root documents";
assert rootDoc >= lastSeenRootDoc : "can only evaluate current and upcoming root docs";
if (rootDoc == lastSeenRootDoc) {
public boolean advanceExact(int parentDoc) throws IOException {
assert parentDoc >= lastSeenParentDoc : "can only evaluate current and upcoming root docs";
if (parentDoc == lastSeenParentDoc) {
return lastEmittedOrd != -1;
}

final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1);
final int firstNestedDoc;
if (innerDocs.docID() > prevRootDoc) {
firstNestedDoc = innerDocs.docID();
final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1);
final int firstChildDoc;
if (childDocs.docID() > prevParentDoc) {
firstChildDoc = childDocs.docID();
} else {
firstNestedDoc = innerDocs.advance(prevRootDoc + 1);
firstChildDoc = childDocs.advance(prevParentDoc + 1);
}

docID = lastSeenRootDoc = rootDoc;
lastEmittedOrd = pick(selectedValues, innerDocs, firstNestedDoc, rootDoc);
docID = lastSeenParentDoc = parentDoc;
lastEmittedOrd = pick(selectedValues, childDocs, firstChildDoc, parentDoc);
return lastEmittedOrd != -1;
}

Expand Down
53 changes: 48 additions & 5 deletions core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,15 @@

package org.elasticsearch.search.sort;

import static org.elasticsearch.search.sort.NestedSortBuilder.NESTED_FIELD;

import org.apache.lucene.search.SortField;
import org.elasticsearch.Version;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
import org.elasticsearch.common.xcontent.XContentBuilder;
Expand All @@ -45,6 +50,8 @@
* A sort builder to sort based on a document field.
*/
public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(FieldSortBuilder.class));

public static final String NAME = "field_sort";
public static final ParseField MISSING = new ParseField("missing");
public static final ParseField SORT_MODE = new ParseField("mode");
Expand All @@ -71,6 +78,8 @@ public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {

private String nestedPath;

private NestedSortBuilder nestedSort;

/** Copy constructor. */
public FieldSortBuilder(FieldSortBuilder template) {
this(template.fieldName);
Expand All @@ -82,6 +91,7 @@ public FieldSortBuilder(FieldSortBuilder template) {
}
this.setNestedFilter(template.getNestedFilter());
this.setNestedPath(template.getNestedPath());
this.setNestedSort(template.getNestedSort());
}

/**
Expand All @@ -108,6 +118,9 @@ public FieldSortBuilder(StreamInput in) throws IOException {
order = in.readOptionalWriteable(SortOrder::readFromStream);
sortMode = in.readOptionalWriteable(SortMode::readFromStream);
unmappedType = in.readOptionalString();
if (in.getVersion().onOrAfter(Version.V_6_1_0)) {
nestedSort = in.readOptionalWriteable(NestedSortBuilder::new);
}
}

@Override
Expand All @@ -119,6 +132,9 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeOptionalWriteable(order);
out.writeOptionalWriteable(sortMode);
out.writeOptionalString(unmappedType);
if (out.getVersion().onOrAfter(Version.V_6_1_0)) {
out.writeOptionalWriteable(nestedSort);
}
}

/** Returns the document field this sort should be based on. */
Expand Down Expand Up @@ -221,6 +237,15 @@ public String getNestedPath() {
return this.nestedPath;
}

public NestedSortBuilder getNestedSort() {
return this.nestedSort;
}

public FieldSortBuilder setNestedSort(final NestedSortBuilder nestedSort) {
this.nestedSort = nestedSort;
return this;
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
Expand All @@ -241,6 +266,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (nestedPath != null) {
builder.field(NESTED_PATH_FIELD.getPreferredName(), nestedPath);
}
if (nestedSort != null) {
builder.field(NESTED_FIELD.getPreferredName(), nestedSort);
}
builder.endObject();
builder.endObject();
return builder;
Expand Down Expand Up @@ -274,7 +302,14 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
localSortMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN;
}

final Nested nested = resolveNested(context, nestedPath, nestedFilter);
final Nested nested;
if (nestedSort != null) {
// new nested sorts takes priority
nested = resolveNested(context, nestedSort);
} else {
nested = resolveNested(context, nestedPath, nestedFilter);
}

IndexFieldData<?> fieldData = context.getForField(fieldType);
if (fieldData instanceof IndexNumericFieldData == false
&& (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) {
Expand All @@ -299,12 +334,13 @@ public boolean equals(Object other) {
return (Objects.equals(this.fieldName, builder.fieldName) && Objects.equals(this.nestedFilter, builder.nestedFilter)
&& Objects.equals(this.nestedPath, builder.nestedPath) && Objects.equals(this.missing, builder.missing)
&& Objects.equals(this.order, builder.order) && Objects.equals(this.sortMode, builder.sortMode)
&& Objects.equals(this.unmappedType, builder.unmappedType));
&& Objects.equals(this.unmappedType, builder.unmappedType) && Objects.equals(this.nestedSort, builder.nestedSort));
}

@Override
public int hashCode() {
return Objects.hash(this.fieldName, this.nestedFilter, this.nestedPath, this.missing, this.order, this.sortMode, this.unmappedType);
return Objects.hash(this.fieldName, this.nestedFilter, this.nestedPath, this.nestedSort, this.missing, this.order, this.sortMode,
this.unmappedType);
}

@Override
Expand All @@ -329,11 +365,18 @@ public static FieldSortBuilder fromXContent(XContentParser parser, String fieldN

static {
PARSER.declareField(FieldSortBuilder::missing, p -> p.objectText(), MISSING, ValueType.VALUE);
PARSER.declareString(FieldSortBuilder::setNestedPath , NESTED_PATH_FIELD);
PARSER.declareString((fieldSortBuilder, nestedPath) -> {
DEPRECATION_LOGGER.deprecated("[nested_path] has been deprecated in favor of the [nested] parameter");
fieldSortBuilder.setNestedPath(nestedPath);
}, NESTED_PATH_FIELD);
PARSER.declareString(FieldSortBuilder::unmappedType , UNMAPPED_TYPE);
PARSER.declareString((b, v) -> b.order(SortOrder.fromString(v)) , ORDER_FIELD);
PARSER.declareString((b, v) -> b.sortMode(SortMode.fromString(v)), SORT_MODE);
PARSER.declareObject(FieldSortBuilder::setNestedFilter, (p, c) -> SortBuilder.parseNestedFilter(p), NESTED_FILTER_FIELD);
PARSER.declareObject(FieldSortBuilder::setNestedFilter, (p, c) -> {
DEPRECATION_LOGGER.deprecated("[nested_filter] has been deprecated in favour for the [nested] parameter");
return SortBuilder.parseNestedFilter(p);
}, NESTED_FILTER_FIELD);
PARSER.declareObject(FieldSortBuilder::setNestedSort, (p, c) -> NestedSortBuilder.fromXContent(p), NESTED_FIELD);
}

@Override
Expand Down
Loading

0 comments on commit 6377afa

Please sign in to comment.