Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Override doc_value parameter in Spatial XPack module #53286

Closed
wants to merge 4 commits into from
Closed

Override doc_value parameter in Spatial XPack module #53286

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
db144f4
Override doc_value parameter in Spatial XPack module
nknize Mar 8, 2020
37d27c3
add `doc_values` mapping option to geo_shape/shape field mappers
talevy Feb 24, 2020
63cdbd6
wire up dataHandler to doc value parsing
nknize Mar 11, 2020
93c3720
remove accidental typo
nknize Mar 11, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 49 additions & 11 deletions server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.Version;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.ParseField;
Expand Down Expand Up @@ -67,9 +68,9 @@ public static class Defaults {
public static final Explicit<Boolean> COERCE = new Explicit<>(false, false);
public static final Explicit<Boolean> IGNORE_MALFORMED = new Explicit<>(false, false);
public static final Explicit<Boolean> IGNORE_Z_VALUE = new Explicit<>(true, false);
public static final Explicit<Boolean> DOC_VALUES = new Explicit<>(false, false);
}


/**
* Interface representing an preprocessor in geo-shape indexing pipeline
*/
Expand All @@ -80,6 +81,8 @@ public interface Indexer<Parsed, Processed> {
Class<Processed> processedClass();

List<IndexableField> indexShape(ParseContext context, Processed shape);

void indexDocValues(ParseContext context, Processed shape);
}

/**
Expand Down Expand Up @@ -116,15 +119,6 @@ public Builder(String name, MappedFieldType fieldType, MappedFieldType defaultFi
super(name, fieldType, defaultFieldType);
}

public Builder(String name, MappedFieldType fieldType, MappedFieldType defaultFieldType,
boolean coerce, boolean ignoreMalformed, Orientation orientation, boolean ignoreZ) {
super(name, fieldType, defaultFieldType);
this.coerce = coerce;
this.ignoreMalformed = ignoreMalformed;
this.orientation = orientation;
this.ignoreZValue = ignoreZ;
}

public Builder coerce(boolean coerce) {
this.coerce = coerce;
return this;
Expand Down Expand Up @@ -184,6 +178,27 @@ public Builder ignoreZValue(final boolean ignoreZValue) {
return this;
}

public T docValues(boolean docValuesImplemented, boolean hasDocValues) {
// TODO(talevy): see how to best make this pluggable with the DataHandler work
if (docValuesImplemented == false && hasDocValues) {
throw new ElasticsearchParseException("field [" + name + "] of type [" + fieldType().typeName()
+ "] does not support doc_values");
}
return super.docValues(hasDocValues);
}

protected Explicit<Boolean> docValues() {
// TODO(talevy): see how to best make this pluggable with the DataHandler work
// although these values can be true, an ElasticsearchParseException
// prevents this Explicit(true,true) path to ever occur in practice
if (docValuesSet && fieldType.hasDocValues()) {
return new Explicit<>(true, true);
} else if (docValuesSet) {
return new Explicit<>(false, true);
}
return Defaults.DOC_VALUES;
}

@Override
protected void setupFieldType(BuilderContext context) {
super.setupFieldType(context);
Expand Down Expand Up @@ -245,13 +260,20 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
XContentMapValues.nodeBooleanValue(fieldNode,
name + "." + GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName()));
iterator.remove();
} else if (TypeParsers.DOC_VALUES.equals(fieldName)) {
params.put(TypeParsers.DOC_VALUES, XContentMapValues.nodeBooleanValue(fieldNode, name + "." + TypeParsers.DOC_VALUES));
iterator.remove();
}
}
if (parsedDeprecatedParameters == false) {
params.remove(DEPRECATED_PARAMETERS_KEY);
}
Builder builder = newBuilder(name, params);

if (params.containsKey(TypeParsers.DOC_VALUES)) {
builder.docValues((Boolean) params.get(TypeParsers.DOC_VALUES));
}

if (params.containsKey(Names.COERCE.getPreferredName())) {
builder.coerce((Boolean)params.get(Names.COERCE.getPreferredName()));
}
Expand Down Expand Up @@ -352,15 +374,17 @@ public QueryProcessor geometryQueryBuilder() {
protected Explicit<Boolean> coerce;
protected Explicit<Boolean> ignoreMalformed;
protected Explicit<Boolean> ignoreZValue;
protected Explicit<Boolean> docValues;

protected AbstractGeometryFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce,
Explicit<Boolean> ignoreZValue, Settings indexSettings,
Explicit<Boolean> ignoreZValue, Explicit<Boolean> docValues, Settings indexSettings,
MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
this.coerce = coerce;
this.ignoreMalformed = ignoreMalformed;
this.ignoreZValue = ignoreZValue;
this.docValues = docValues;
}

@Override
Expand All @@ -376,6 +400,9 @@ protected void doMerge(Mapper mergeWith) {
if (gsfm.ignoreZValue.explicit()) {
this.ignoreZValue = gsfm.ignoreZValue;
}
if (gsfm.docValues.explicit()) {
this.docValues = gsfm.docValues;
}
}

@Override
Expand All @@ -399,6 +426,10 @@ public void doXContentBody(XContentBuilder builder, boolean includeDefaults, Par
if (includeDefaults || ignoreZValue.explicit()) {
builder.field(GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName(), ignoreZValue.value());
}

if (includeDefaults || docValues.explicit()) {
builder.field(TypeParsers.DOC_VALUES, docValues.value());
}
}

public Explicit<Boolean> coerce() {
Expand All @@ -413,6 +444,10 @@ public Explicit<Boolean> ignoreZValue() {
return ignoreZValue;
}

public Explicit<Boolean> docValues() {
return docValues;
}

public Orientation orientation() {
return ((AbstractGeometryFieldType)fieldType).orientation();
}
Expand Down Expand Up @@ -440,6 +475,9 @@ public void parse(ParseContext context) throws IOException {
for (IndexableField field : fields) {
context.doc().add(field);
}
if (fieldType.hasDocValues()) {
geometryIndexer.indexDocValues(context, shape);
}
} catch (Exception e) {
if (ignoreMalformed.value() == false) {
throw new MapperParsingException("failed to parse field [{}] of type [{}]", e, fieldType().name(),
Expand Down
104 changes: 98 additions & 6 deletions server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,18 @@
package org.elasticsearch.index.mapper;

import org.apache.lucene.document.LatLonShape;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.Version;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.geo.GeometryParser;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.geometry.Geometry;
import org.elasticsearch.index.query.VectorGeoShapeQueryProcessor;

import java.util.HashMap;
import java.util.Map;

/**
* FieldMapper for indexing {@link LatLonShape}s.
* <p>
Expand All @@ -49,16 +54,51 @@
public class GeoShapeFieldMapper extends AbstractGeometryFieldMapper<Geometry, Geometry> {
public static final String CONTENT_TYPE = "geo_shape";

public static Map<String, DataHandlerFactory> DATA_HANDLER_FACTORIES = new HashMap<>();

public static class Defaults extends AbstractGeometryFieldMapper.Defaults {
public static final Explicit<String> DATA_HANDLER = new Explicit<>("default", false);
}

public static class Builder extends AbstractGeometryFieldMapper.Builder<AbstractGeometryFieldMapper.Builder, GeoShapeFieldMapper> {
DataHandler dataHandler;

public Builder(String name) {
super (name, new GeoShapeFieldType(), new GeoShapeFieldType());
this.dataHandler = resolveDataHandler(Defaults.DATA_HANDLER.value());
}

@Override
public GeoShapeFieldMapper build(BuilderContext context) {
setupFieldType(context);
return new GeoShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context), coerce(context),
ignoreZValue(), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
ignoreZValue(), docValues(), context.indexSettings(),
multiFieldsBuilder.build(this, context), copyTo);
}

@Override
protected boolean defaultDocValues(Version indexCreated) {
return dataHandler.defaultDocValues(indexCreated);
}

@Override
public Builder docValues(boolean hasDocValues) {
return (Builder)super.docValues(dataHandler.supportsDocValues(), hasDocValues);
}

@Override
protected Explicit<Boolean> docValues() {
// TODO(talevy): see how to best make this pluggable with the DataHandler work
// although these values can be true, an ElasticsearchParseException
// prevents this Explicit(true,true) path to ever occur in practice
if (dataHandler.supportsDocValues()) {
if (docValuesSet && fieldType.hasDocValues()) {
return new Explicit<>(true, true);
} else if (docValuesSet) {
return new Explicit<>(false, true);
}
}
return AbstractGeometryFieldMapper.Defaults.DOC_VALUES;
}

@Override
Expand All @@ -68,11 +108,12 @@ protected void setupFieldType(BuilderContext context) {
GeoShapeFieldType fieldType = (GeoShapeFieldType)fieldType();
boolean orientation = fieldType.orientation == ShapeBuilder.Orientation.RIGHT;

// @todo the GeometryParser can be static since it doesn't hold state?
GeometryParser geometryParser = new GeometryParser(orientation, coerce(context).value(), ignoreZValue().value());

fieldType.setGeometryIndexer(new GeoShapeIndexer(orientation, fieldType.name()));
fieldType.setGeometryParser( (parser, mapper) -> geometryParser.parse(parser));
fieldType.setGeometryQueryBuilder(new VectorGeoShapeQueryProcessor());

fieldType.setGeometryIndexer(dataHandler.newIndexer(orientation, fieldType));
fieldType.setGeometryQueryBuilder(dataHandler.newQueryProcessor());
}
}

Expand All @@ -98,9 +139,9 @@ public String typeName() {

public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce,
Explicit<Boolean> ignoreZValue, Settings indexSettings,
Explicit<Boolean> ignoreZValue, Explicit<Boolean> docValues, Settings indexSettings,
MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, indexSettings,
super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, docValues, indexSettings,
multiFields, copyTo);
}

Expand All @@ -124,4 +165,55 @@ public GeoShapeFieldType fieldType() {
protected String contentType() {
return CONTENT_TYPE;
}

public static void registerDataHandlers(Map<String, DataHandlerFactory> dataHandlerFactories) {
DATA_HANDLER_FACTORIES.putAll(dataHandlerFactories);
}

public static DataHandler resolveDataHandler(String handlerKey) {
if (DATA_HANDLER_FACTORIES.containsKey(handlerKey)) {
return DATA_HANDLER_FACTORIES.get(handlerKey).newDataHandler();
}
throw new IllegalArgumentException("dataHandler [" + handlerKey + "] not supported");
}

public interface DataHandlerFactory {
DataHandler newDataHandler();
}

public abstract static class DataHandler {
public abstract Indexer newIndexer(boolean orientation, MappedFieldType fieldType);
public abstract QueryProcessor newQueryProcessor();
public abstract boolean defaultDocValues(Version indexCreatedVersion);
public abstract boolean supportsDocValues();
}

static {
DATA_HANDLER_FACTORIES.put(Defaults.DATA_HANDLER.value(), () ->
new DataHandler() {
@Override
public boolean supportsDocValues() {
return false;
}

@Override
public boolean defaultDocValues(Version indexCreatedVersion) {
return false;
}

@Override
public Indexer newIndexer(boolean orientation, MappedFieldType fieldType) {
return new GeoShapeIndexer(orientation, fieldType.name());
}

@Override
public QueryProcessor newQueryProcessor() {
return new VectorGeoShapeQueryProcessor();
}
});
}

public interface Extension {
Map<String, DataHandlerFactory> getDataHandlerFactories();
}
}
7 changes: 6 additions & 1 deletion server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
/**
* Utility class that converts geometries into Lucene-compatible form for indexing in a geo_shape field.
*/
public final class GeoShapeIndexer implements AbstractGeometryFieldMapper.Indexer<Geometry, Geometry> {
public class GeoShapeIndexer implements AbstractGeometryFieldMapper.Indexer<Geometry, Geometry> {

private final boolean orientation;
private final String name;
Expand Down Expand Up @@ -185,6 +185,11 @@ public List<IndexableField> indexShape(ParseContext context, Geometry shape) {
return visitor.fields();
}

@Override
public void indexDocValues(ParseContext context, Geometry shape) {
throw new UnsupportedOperationException("DocValues are not supported");
}

private static class LuceneGeometryIndexer implements GeometryVisitor<Void, RuntimeException> {
private List<IndexableField> fields = new ArrayList<>();
private String name;
Expand Down
14 changes: 10 additions & 4 deletions server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,11 @@ private void setupPrefixTrees() {
protected void setupFieldType(BuilderContext context) {
super.setupFieldType(context);

if (context.indexCreatedVersion().onOrAfter(Version.V_8_0_0) && fieldType().hasDocValues()) {
throw new ElasticsearchParseException("Field parameter [{}] is not supported for [{}] field type while using prefix trees",
TypeParsers.DOC_VALUES, CONTENT_TYPE);
}

fieldType().setGeometryIndexer(new LegacyGeoShapeIndexer(fieldType()));
fieldType().setGeometryParser(ShapeParser::parse);
fieldType().setGeometryQueryBuilder(new LegacyGeoShapeQueryProcessor(fieldType()));
Expand Down Expand Up @@ -292,7 +297,7 @@ public LegacyGeoShapeFieldMapper build(BuilderContext context) {
setupFieldType(context);

return new LegacyGeoShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context),
coerce(context), orientation(), ignoreZValue(), context.indexSettings(),
coerce(context), orientation(), ignoreZValue(), docValues(), context.indexSettings(),
multiFieldsBuilder.build(this, context), copyTo);
}
}
Expand All @@ -318,6 +323,7 @@ public GeoShapeFieldType() {
setStored(false);
setStoreTermVectors(false);
setOmitNorms(true);
setHasDocValues(false);
}

protected GeoShapeFieldType(GeoShapeFieldType ref) {
Expand Down Expand Up @@ -470,10 +476,10 @@ public PrefixTreeStrategy resolvePrefixTreeStrategy(String strategyName) {

public LegacyGeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce, Explicit<Orientation> orientation,
Explicit<Boolean> ignoreZValue, Settings indexSettings,
Explicit<Boolean> ignoreZValue, Explicit<Boolean> docValues, Settings indexSettings,
MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, indexSettings,
multiFields, copyTo);
super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, docValues,
indexSettings, multiFields, copyTo);
}

@Override
Expand Down
5 changes: 5 additions & 0 deletions server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeIndexer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,4 +69,9 @@ public List<IndexableField> indexShape(ParseContext context, Shape shape) {
}
return Arrays.asList(fieldType.defaultPrefixTreeStrategy().createIndexableFields(shape));
}

@Override
public void indexDocValues(ParseContext context, Shape shape) {
throw new UnsupportedOperationException("[" + fieldType.name() + "] does not support doc_values");
}
}
Loading