Add geo_shape mapper supporting doc-values in Spatial Plugin

distribution/build.gradle

@@ -317,6 +317,12 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
           excludePlatforms = []
         }
         from(buildModules) {
+          // geo registers the geo_shape mapper that is overridden by
+          // the geo_shape mapper registered in the x-pack-spatial plugin
+          if (oss == false) {
+            exclude "**/geo/**"
+          }
+
           for (String excludePlatform : excludePlatforms) {
             exclude "**/platform/${excludePlatform}/**"
           }

modules/geo/build.gradle

@@ -21,3 +21,7 @@ esplugin {
   description 'Placeholder plugin for geospatial features in ES. only registers geo_shape field mapper for now'
   classname 'org.elasticsearch.geo.GeoPlugin'
 }
+
+artifacts {
+  restTests(new File(projectDir, "src/test/resources/rest-api-spec/test"))
+}

modules/geo/src/main/java/org/elasticsearch/geo/GeoPlugin.java

@@ -19,7 +19,6 @@
 
 package org.elasticsearch.geo;
 
-import org.elasticsearch.index.mapper.AbstractGeometryFieldMapper;
 import org.elasticsearch.index.mapper.GeoShapeFieldMapper;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.plugins.MapperPlugin;
@@ -32,6 +31,6 @@ public class GeoPlugin extends Plugin implements MapperPlugin {
 
     @Override
     public Map<String, Mapper.TypeParser> getMappers() {
-        return Collections.singletonMap(GeoShapeFieldMapper.CONTENT_TYPE, new AbstractGeometryFieldMapper.TypeParser());
+        return Collections.singletonMap(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser());
     }
 }

server/src/main/java/org/elasticsearch/common/io/stream/ByteBufferStreamInput.java

@@ -110,6 +110,14 @@ public long readLong() throws IOException {
         }
     }
 
+    public void position(int newPosition) throws IOException {
+        buffer.position(newPosition);
+    }
+
+    public int position() throws IOException {
+        return buffer.position();
+    }
+
     @Override
     public void reset() throws IOException {
         buffer.reset();

server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java

@@ -185,7 +185,9 @@ protected void setupFieldType(BuilderContext context) {
 
     protected static final String DEPRECATED_PARAMETERS_KEY = "deprecated_parameters";
 
-    public static class TypeParser implements Mapper.TypeParser {
+    public abstract static class TypeParser implements Mapper.TypeParser {
+        protected abstract Builder newBuilder(String name, Map<String, Object> params);
+
         protected boolean parseXContentParameters(String name, Map.Entry<String, Object> entry, Map<String, Object> params)
                 throws MapperParsingException {
             if (DeprecatedParameters.parse(name, entry.getKey(), entry.getValue(),
@@ -195,13 +197,6 @@ protected boolean parseXContentParameters(String name, Map.Entry<String, Object>
             return false;
         }
 
-        protected Builder newBuilder(String name, Map<String, Object> params) {
-            if (params.containsKey(DEPRECATED_PARAMETERS_KEY)) {
-                return new LegacyGeoShapeFieldMapper.Builder(name, (DeprecatedParameters)params.get(DEPRECATED_PARAMETERS_KEY));
-            }
-            return new GeoShapeFieldMapper.Builder(name);
-        }
-
         @Override
         public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
             Map<String, Object> params = new HashMap<>();

server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java

@@ -26,6 +26,8 @@
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.index.query.VectorGeoShapeQueryProcessor;
 
+import java.util.Map;
+
 /**
  * FieldMapper for indexing {@link LatLonShape}s.
  * <p>
@@ -76,7 +78,7 @@ protected void setupFieldType(BuilderContext context) {
         }
     }
 
-    public static final class GeoShapeFieldType extends AbstractGeometryFieldType<Geometry, Geometry> {
+    public static class GeoShapeFieldType extends AbstractGeometryFieldType<Geometry, Geometry> {
         public GeoShapeFieldType() {
             super();
         }
@@ -96,6 +98,18 @@ public String typeName() {
         }
     }
 
+    public static final class TypeParser extends AbstractGeometryFieldMapper.TypeParser {
+
+        @Override
+        protected AbstractGeometryFieldMapper.Builder newBuilder(String name, Map<String, Object> params) {
+            if (params.containsKey(DEPRECATED_PARAMETERS_KEY)) {
+                return new LegacyGeoShapeFieldMapper.Builder(name,
+                    (LegacyGeoShapeFieldMapper.DeprecatedParameters)params.get(DEPRECATED_PARAMETERS_KEY));
+            }
+            return new GeoShapeFieldMapper.Builder(name);
+        }
+    }
+
     public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
                                Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce,
                                Explicit<Boolean> ignoreZValue, Settings indexSettings,

server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java

@@ -49,7 +49,7 @@
 /**
  * Utility class that converts geometries into Lucene-compatible form for indexing in a geo_shape field.
  */
-public final class GeoShapeIndexer implements AbstractGeometryFieldMapper.Indexer<Geometry, Geometry> {
+public class GeoShapeIndexer implements AbstractGeometryFieldMapper.Indexer<Geometry, Geometry> {
 
     private final boolean orientation;
     private final String name;

server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java

@@ -57,7 +57,7 @@ public String toString() {
         };
     }
 
-    static SortedBinaryDocValues replaceMissing(final SortedBinaryDocValues values, final BytesRef missing) {
+    public static SortedBinaryDocValues replaceMissing(final SortedBinaryDocValues values, final BytesRef missing) {
         return new SortedBinaryDocValues() {
 
             private int count;

test/framework/src/main/java/org/elasticsearch/test/TestGeoShapeFieldMapperPlugin.java

@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.test;
 
-import org.elasticsearch.index.mapper.AbstractGeometryFieldMapper;
 import org.elasticsearch.index.mapper.GeoShapeFieldMapper;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.plugins.MapperPlugin;
@@ -40,7 +39,7 @@ public class TestGeoShapeFieldMapperPlugin extends Plugin implements MapperPlugi
     @Override
     public Map<String, Mapper.TypeParser> getMappers() {
         Map<String, Mapper.TypeParser> mappers = new LinkedHashMap<>();
-        mappers.put(GeoShapeFieldMapper.CONTENT_TYPE, new AbstractGeometryFieldMapper.TypeParser());
+        mappers.put(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser());
         return Collections.unmodifiableMap(mappers);
     }
 }

x-pack/plugin/spatial/build.gradle

@@ -12,15 +12,24 @@ esplugin {
 dependencies {
   compileOnly project(path: xpackModule('core'), configuration: 'default')
   testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
-  testCompile project(path: ':modules:geo', configuration: 'runtime')
+  compile project(path: ':modules:geo', configuration: 'default')
+  restTestConfig project(path: ':modules:geo', configuration: 'restTests')
+}
+
+restResources {
+  restApi {
+    includeCore '_common', 'indices', 'index', 'search'
+  }
+  restTests {
+    includeCore 'geo_shape'
+  }
+}
+
+testClusters.integTest {
+  testDistribution = 'DEFAULT'
 }
 
 licenseHeaders {
   // This class was sourced from apache lucene's sandbox module tests
   excludes << 'org/apache/lucene/geo/XShapeTestUtil.java'
 }
-
-// xpack modules are installed in real clusters as the meta plugin, so
-// installing them as individual plugins for integ tests doesn't make sense,
-// so we disable integ tests
-integTest.enabled = false

x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java

@@ -7,33 +7,30 @@
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.geo.GeoPlugin;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.MapperPlugin;
-import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.spatial.index.mapper.GeoShapeWithDocValuesFieldMapper;
 import org.elasticsearch.xpack.spatial.index.mapper.PointFieldMapper;
 import org.elasticsearch.xpack.spatial.index.mapper.ShapeFieldMapper;
 import org.elasticsearch.xpack.spatial.index.query.ShapeQueryBuilder;
 import org.elasticsearch.xpack.spatial.ingest.CircleProcessor;
 
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.LinkedHashMap;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 import static java.util.Collections.singletonList;
 
-public class SpatialPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, IngestPlugin {
-
-    public SpatialPlugin(Settings settings) {
-    }
+public class SpatialPlugin extends GeoPlugin implements ActionPlugin, MapperPlugin, SearchPlugin, IngestPlugin {
 
     @Override
     public List<ActionPlugin.ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
@@ -44,9 +41,10 @@ public SpatialPlugin(Settings settings) {
 
     @Override
     public Map<String, Mapper.TypeParser> getMappers() {
-        Map<String, Mapper.TypeParser> mappers = new LinkedHashMap<>();
+        Map<String, Mapper.TypeParser> mappers = new HashMap<>(super.getMappers());
         mappers.put(ShapeFieldMapper.CONTENT_TYPE, new ShapeFieldMapper.TypeParser());
         mappers.put(PointFieldMapper.CONTENT_TYPE, new PointFieldMapper.TypeParser());
+        mappers.put(GeoShapeWithDocValuesFieldMapper.CONTENT_TYPE, new GeoShapeWithDocValuesFieldMapper.TypeParser());
         return Collections.unmodifiableMap(mappers);
     }
 

x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/AbstractAtomicGeoShapeShapeFieldData.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.spatial.index.mapper;
+
+import org.apache.lucene.util.Accountable;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
+import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
+
+import java.util.Collection;
+import java.util.Collections;
+
+public abstract class AbstractAtomicGeoShapeShapeFieldData implements LeafGeoShapeFieldData {
+
+    @Override
+    public final SortedBinaryDocValues getBytesValues() {
+        throw new UnsupportedOperationException("scripts and term aggs are not supported by geo_shape doc values");
+    }
+
+    @Override
+    public final ScriptDocValues.BytesRefs getScriptValues() {
+        throw new UnsupportedOperationException("scripts are not supported by geo_shape doc values");
+    }
+
+    public static LeafGeoShapeFieldData empty(final int maxDoc) {
+        return new AbstractAtomicGeoShapeShapeFieldData() {
+
+            @Override
+            public long ramBytesUsed() {
+                return 0;
+            }
+
+            @Override
+            public Collection<Accountable> getChildResources() {
+                return Collections.emptyList();
+            }
+
+            @Override
+            public void close() {
+            }
+
+            @Override
+            public MultiGeoShapeValues getGeoShapeValues() {
+                return MultiGeoShapeValues.EMPTY;
+            }
+        };
+    }
+}

x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/AbstractLatLonShapeDVIndexFieldData.java

@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.spatial.index.mapper;
+
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.MultiValueMode;
+import org.elasticsearch.search.sort.BucketedSort;
+import org.elasticsearch.search.sort.SortOrder;
+
+public abstract class AbstractLatLonShapeDVIndexFieldData extends DocValuesIndexFieldData implements IndexGeoShapeFieldData {
+    AbstractLatLonShapeDVIndexFieldData(Index index, String fieldName) {
+        super(index, fieldName);
+    }
+
+    @Override
+    public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested,
+            boolean reverse) {
+        throw new IllegalArgumentException("can't sort on geo_shape field without using specific sorting feature, like geo_distance");
+    }
+
+    public static class LatLonShapeDVIndexFieldData extends AbstractLatLonShapeDVIndexFieldData {
+        public LatLonShapeDVIndexFieldData(Index index, String fieldName) {
+            super(index, fieldName);
+        }
+
+        @Override
+        public LeafGeoShapeFieldData load(LeafReaderContext context) {
+            LeafReader reader = context.reader();
+            FieldInfo info = reader.getFieldInfos().fieldInfo(fieldName);
+            if (info != null) {
+                checkCompatible(info);
+            }
+            return new LatLonShapeDVAtomicShapeFieldData(reader, fieldName);
+        }
+
+        @Override
+        public LeafGeoShapeFieldData loadDirect(LeafReaderContext context) throws Exception {
+            return load(context);
+        }
+
+        @Override
+        public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, MultiValueMode sortMode,
+                                            IndexFieldData.XFieldComparatorSource.Nested nested, SortOrder sortOrder, DocValueFormat format,
+                                            int bucketSize, BucketedSort.ExtraData extra) {
+            throw new IllegalArgumentException("can't sort on geo_shape field without using specific sorting feature, like geo_distance");
+        }
+
+        /** helper: checks a fieldinfo and throws exception if its definitely not a LatLonDocValuesField */
+        static void checkCompatible(FieldInfo fieldInfo) {
+            // dv properties could be "unset", if you e.g. used only StoredField with this same name in the segment.
+            if (fieldInfo.getDocValuesType() != DocValuesType.NONE
+                && fieldInfo.getDocValuesType() != DocValuesType.BINARY) {
+                throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" was indexed with docValuesType="
+                    + fieldInfo.getDocValuesType() + " but this type has docValuesType="
+                    + DocValuesType.BINARY + ", is the field really a geo-shape field?");
+            }
+        }
+    }
+
+    public static class Builder implements IndexFieldData.Builder {
+        @Override
+        public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
+                                       CircuitBreakerService breakerService, MapperService mapperService) {
+            // ignore breaker
+            return new LatLonShapeDVIndexFieldData(indexSettings.getIndex(), fieldType.name());
+        }
+    }
+}