config: add ability to register native filters (#1193)

Description: adding templating to register native filters.
Risk Level: low - optional feature
Testing: used in example apps
Docs Changes: pending

Signed-off-by: Jose Nino <jnino@lyft.com>
Signed-off-by: JP Simard <jp@jpsim.com>

mobile/examples/kotlin/hello_world/MainActivity.kt

@@ -39,15 +39,17 @@ class MainActivity : Activity() {
   private lateinit var viewAdapter: ResponseRecyclerViewAdapter
   private lateinit var engine: Engine
 
+  @Suppress("MaxLineLength")
   override fun onCreate(savedInstanceState: Bundle?) {
     super.onCreate(savedInstanceState)
     setContentView(R.layout.activity_main)
 
     engine = AndroidEngineBuilder(application)
-      .addFilter { DemoFilter() }
-      .addFilter { BufferDemoFilter() }
-      .addFilter { AsyncDemoFilter() }
+      .addPlatformFilter { DemoFilter() }
+      .addPlatformFilter { BufferDemoFilter() }
+      .addPlatformFilter { AsyncDemoFilter() }
       .addStringAccessor("demo-accessor", DemoStringAccessor())
+      .addNativeFilter("envoy.filters.http.buffer", "{\"@type\":\"type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer\",\"max_request_bytes\":5242880}")
       .setOnEngineRunning { Log.d("MainActivity", "Envoy async internal setup completed") }
       .build()
 

mobile/examples/swift/hello_world/ViewController.swift

@@ -19,9 +19,12 @@ final class ViewController: UITableViewController {
     do {
       NSLog("starting Envoy...")
       let engine = try EngineBuilder()
-        .addFilter(factory: DemoFilter.init)
-        .addFilter(factory: BufferDemoFilter.init)
-        .addFilter(factory: AsyncDemoFilter.init)
+        .addPlatformFilter(factory: DemoFilter.init)
+        .addPlatformFilter(factory: BufferDemoFilter.init)
+        .addPlatformFilter(factory: AsyncDemoFilter.init)
+        .addNativeFilter(name: "envoy.filters.http.buffer",
+                         // swiftlint:disable:next line_length
+                         typedConfig: "{\"@type\":\"type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer\",\"max_request_bytes\":5242880}")
         .setOnEngineRunning { NSLog("Envoy async internal setup completed") }
         .build()
       self.streamClient = engine.streamClient()

mobile/library/common/config_template.cc

@@ -8,6 +8,11 @@ const char* platform_filter_template = R"(
               platform_filter_name: {{ platform_filter_name }}
 )";
 
+const char* native_filter_template = R"(
+          - name: {{ native_filter_name }}
+            typed_config: {{ native_filter_typed_config }}
+)";
+
 const char* config_template = R"(
 static_resources:
   listeners:
@@ -41,6 +46,7 @@ const char* config_template = R"(
                         max_interval: 60s
         http_filters:
 {{ platform_filter_chain }}
+{{ native_filter_chain }}
           - name: envoy.filters.http.dynamic_forward_proxy
             typed_config:
               "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig

mobile/library/common/jni/jni_interface.cc

@@ -73,13 +73,21 @@ Java_io_envoyproxy_envoymobile_engine_JniLibrary_templateString(JNIEnv* env,
 }
 
 extern "C" JNIEXPORT jstring JNICALL
-Java_io_envoyproxy_envoymobile_engine_JniLibrary_filterTemplateString(JNIEnv* env,
-                                                                      jclass // class
+Java_io_envoyproxy_envoymobile_engine_JniLibrary_platformFilterTemplateString(JNIEnv* env,
+                                                                              jclass // class
 ) {
   jstring result = env->NewStringUTF(platform_filter_template);
   return result;
 }
 
+extern "C" JNIEXPORT jstring JNICALL
+Java_io_envoyproxy_envoymobile_engine_JniLibrary_nativeFilterTemplateString(JNIEnv* env,
+                                                                            jclass // class
+) {
+  jstring result = env->NewStringUTF(native_filter_template);
+  return result;
+}
+
 extern "C" JNIEXPORT jint JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibrary_recordCounterInc(
     JNIEnv* env,
     jclass, // class

mobile/library/common/main_interface.h

@@ -23,6 +23,11 @@ extern const char* config_template;
  */
 extern const char* platform_filter_template;
 
+/**
+ * Template configuration used for dynamic creation of the native filter chain.
+ */
+extern const char* native_filter_template;
+
 /**
  * Initialize an underlying HTTP stream.
  * @param engine, handle to the engine that will manage this stream.

mobile/library/java/src/io/envoyproxy/envoymobile/engine/BUILD

@@ -28,6 +28,7 @@ java_library(
         "EnvoyEngineImpl.java",
         "EnvoyHTTPFilterCallbacksImpl.java",
         "EnvoyHTTPStream.java",
+        "EnvoyNativeFilterConfig.java",
         "EnvoyNativeResourceRegistry.java",
         "EnvoyNativeResourceReleaser.java",
         "EnvoyNativeResourceWrapper.java",

mobile/library/java/src/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java

@@ -15,11 +15,12 @@ public class EnvoyConfiguration {
   public final Integer dnsRefreshSeconds;
   public final Integer dnsFailureRefreshSecondsBase;
   public final Integer dnsFailureRefreshSecondsMax;
-  public final List<EnvoyHTTPFilterFactory> httpFilterFactories;
+  public final List<EnvoyHTTPFilterFactory> httpPlatformFilterFactories;
   public final Integer statsFlushSeconds;
   public final String appVersion;
   public final String appId;
   public final String virtualClusters;
+  public final List<EnvoyNativeFilterConfig> nativeFilterChain;
   public final Map<String, EnvoyStringAccessor> stringAccessors;
 
   private static final Pattern UNRESOLVED_KEY_PATTERN = Pattern.compile("\\{\\{ (.+) \\}\\}");
@@ -37,23 +38,27 @@ public class EnvoyConfiguration {
    * @param appVersion                   the App Version of the App using this Envoy Client.
    * @param appId                        the App ID of the App using this Envoy Client.
    * @param virtualClusters              the JSON list of virtual cluster configs.
+   * @param nativeFilterChain            the configuration for native filters.
+   * @param httpPlatformFilterFactories          the configuration for platform filters.
    * @param stringAccesssors             platform string accessors to register.
    */
   public EnvoyConfiguration(String statsDomain, int connectTimeoutSeconds, int dnsRefreshSeconds,
                             int dnsFailureRefreshSecondsBase, int dnsFailureRefreshSecondsMax,
-                            List<EnvoyHTTPFilterFactory> httpFilterFactories, int statsFlushSeconds,
-                            String appVersion, String appId, String virtualClusters,
+                            int statsFlushSeconds, String appVersion, String appId,
+                            String virtualClusters, List<EnvoyNativeFilterConfig> nativeFilterChain,
+                            List<EnvoyHTTPFilterFactory> httpPlatformFilterFactories,
                             Map<String, EnvoyStringAccessor> stringAccessors) {
     this.statsDomain = statsDomain;
     this.connectTimeoutSeconds = connectTimeoutSeconds;
     this.dnsRefreshSeconds = dnsRefreshSeconds;
     this.dnsFailureRefreshSecondsBase = dnsFailureRefreshSecondsBase;
     this.dnsFailureRefreshSecondsMax = dnsFailureRefreshSecondsMax;
-    this.httpFilterFactories = httpFilterFactories;
     this.statsFlushSeconds = statsFlushSeconds;
     this.appVersion = appVersion;
     this.appId = appId;
     this.virtualClusters = virtualClusters;
+    this.nativeFilterChain = nativeFilterChain;
+    this.httpPlatformFilterFactories = httpPlatformFilterFactories;
     this.stringAccessors = stringAccessors;
   }
 
@@ -62,19 +67,31 @@ public EnvoyConfiguration(String statsDomain, int connectTimeoutSeconds, int dns
    * configuration.
    *
    * @param templateYAML the template configuration to resolve.
+   * @param platformFilterTemplateYAML helper template to build platform http filters.
+   * @param nativeFilterTemplateYAML helper template to build native http filters.
    * @return String, the resolved template.
    * @throws ConfigurationException, when the template provided is not fully
    *                                 resolved.
    */
-  String resolveTemplate(final String templateYAML, final String filterTemplateYAML) {
+  String resolveTemplate(final String templateYAML, final String platformFilterTemplateYAML,
+                         final String nativeFilterTemplateYAML) {
     final StringBuilder filterConfigBuilder = new StringBuilder();
-    for (EnvoyHTTPFilterFactory filterFactory : httpFilterFactories) {
-      String filterConfig =
-          filterTemplateYAML.replace("{{ platform_filter_name }}", filterFactory.getFilterName());
+    for (EnvoyHTTPFilterFactory filterFactory : httpPlatformFilterFactories) {
+      String filterConfig = platformFilterTemplateYAML.replace("{{ platform_filter_name }}",
+                                                               filterFactory.getFilterName());
       filterConfigBuilder.append(filterConfig);
     }
     String filterConfigChain = filterConfigBuilder.toString();
 
+    final StringBuilder nativeFilterConfigBuilder = new StringBuilder();
+    for (EnvoyNativeFilterConfig filter : nativeFilterChain) {
+      String nativeFilterConfig =
+          nativeFilterTemplateYAML.replace("{{ native_filter_name }}", filter.name)
+              .replace("{{ native_filter_typed_config }}", filter.typedConfig);
+      nativeFilterConfigBuilder.append(nativeFilterConfig);
+    }
+    String nativeFilterConfigChain = nativeFilterConfigBuilder.toString();
+
     String resolvedConfiguration =
         templateYAML.replace("{{ stats_domain }}", statsDomain)
             .replace("{{ platform_filter_chain }}", filterConfigChain)
@@ -88,7 +105,8 @@ String resolveTemplate(final String templateYAML, final String filterTemplateYAM
             .replace("{{ device_os }}", "Android")
             .replace("{{ app_version }}", appVersion)
             .replace("{{ app_id }}", appId)
-            .replace("{{ virtual_clusters }}", virtualClusters);
+            .replace("{{ virtual_clusters }}", virtualClusters)
+            .replace("{{ native_filter_chain }}", nativeFilterConfigChain);
 
     final Matcher unresolvedKeys = UNRESOLVED_KEY_PATTERN.matcher(resolvedConfiguration);
     if (unresolvedKeys.find()) {

mobile/library/java/src/io/envoyproxy/envoymobile/engine/EnvoyEngineImpl.java

@@ -67,7 +67,8 @@ public int runWithConfig(String configurationYAML, String logLevel,
   @Override
   public int runWithConfig(EnvoyConfiguration envoyConfiguration, String logLevel,
                            EnvoyOnEngineRunning onEngineRunning) {
-    for (EnvoyHTTPFilterFactory filterFactory : envoyConfiguration.httpFilterFactories) {
+
+    for (EnvoyHTTPFilterFactory filterFactory : envoyConfiguration.httpPlatformFilterFactories) {
       JniLibrary.registerFilterFactory(filterFactory.getFilterName(),
                                        new JvmFilterFactoryContext(filterFactory));
     }
@@ -78,8 +79,9 @@ public int runWithConfig(EnvoyConfiguration envoyConfiguration, String logLevel,
                                         new JvmStringAccessorContext(entry.getValue()));
     }
 
-    return runWithConfig(envoyConfiguration.resolveTemplate(JniLibrary.templateString(),
-                                                            JniLibrary.filterTemplateString()),
+    return runWithConfig(envoyConfiguration.resolveTemplate(
+                             JniLibrary.templateString(), JniLibrary.platformFilterTemplateString(),
+                             JniLibrary.nativeFilterTemplateString()),
                          logLevel, onEngineRunning);
   }
 

mobile/library/java/src/io/envoyproxy/envoymobile/engine/EnvoyNativeFilterConfig.java

@@ -0,0 +1,18 @@
+package io.envoyproxy.envoymobile.engine;
+
+/* Datatype used by the EnvoyConfiguration to create a native http filter chain. */
+public class EnvoyNativeFilterConfig {
+  public final String name;
+  public final String typedConfig;
+
+  /**
+   * Create a new instance of the configuration
+   *
+   * @param name        the name of the filter.
+   * @param typedConfig the filter configuration.
+   */
+  public EnvoyNativeFilterConfig(String name, String typedConfig) {
+    this.name = name;
+    this.typedConfig = typedConfig;
+  }
+}

mobile/library/java/src/io/envoyproxy/envoymobile/engine/JniLibrary.java

@@ -198,9 +198,18 @@ protected static native int runEngine(long engine, String config, String logLeve
    * filter config chains.
    *
    * @return A template that may be used as a starting point for constructing
-   *         filter platform filter configuration.
+   *         platform filter configuration.
    */
-  public static native String filterTemplateString();
+  public static native String platformFilterTemplateString();
+
+  /**
+   * Provides a configuration template that may be used for building native
+   * filter config chains.
+   *
+   * @return A template that may be used as a starting point for constructing
+   *         native filter configuration.
+   */
+  public static native String nativeFilterTemplateString();
 
   /**
    * Register a string accessor to get strings from the platform.

mobile/library/java/test/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt

@@ -16,25 +16,37 @@ mock_template:
     max_interval: {{ dns_failure_refresh_rate_seconds_max }}s
   platform_filter_chain:
 {{ platform_filter_chain }}
+  native_filter_chain:
+{{ native_filter_chain }}
   stats_flush_interval: {{ stats_flush_interval_seconds }}s
   os: {{ device_os }}
   app_version: {{ app_version }}
   app_id: {{ app_id }}
   virtual_clusters: {{ virtual_clusters }}
 """
 
-private const val FILTER_CONFIG =
-  """
+private const val PLATFORM_FILTER_CONFIG =
+"""
     - platform_filter_name: {{ platform_filter_name }}
 """
 
+private const val NATIVE_FILTER_CONFIG =
+"""
+    - name: {{ native_filter_name }}
+      typed_config: {{ native_filter_typed_config }}
+"""
+
 class EnvoyConfigurationTest {
 
   @Test
   fun `resolving with default configuration resolves with values`() {
-    val envoyConfiguration = EnvoyConfiguration("stats.foo.com", 123, 234, 345, 456, emptyList(), 567, "v1.2.3", "com.mydomain.myapp", "[test]", emptyMap())
+    val envoyConfiguration = EnvoyConfiguration(
+      "stats.foo.com", 123, 234, 345, 456, 567, "v1.2.3", "com.mydomain.myapp", "[test]",
+      listOf<EnvoyNativeFilterConfig>(EnvoyNativeFilterConfig("filter_name", "test_config")),
+      emptyList(), emptyMap()
+    )
 
-    val resolvedTemplate = envoyConfiguration.resolveTemplate(TEST_CONFIG, FILTER_CONFIG)
+    val resolvedTemplate = envoyConfiguration.resolveTemplate(TEST_CONFIG, PLATFORM_FILTER_CONFIG, NATIVE_FILTER_CONFIG)
     assertThat(resolvedTemplate).contains("stats_domain: stats.foo.com")
     assertThat(resolvedTemplate).contains("connect_timeout: 123s")
     assertThat(resolvedTemplate).contains("dns_refresh_rate: 234s")
@@ -45,14 +57,19 @@ class EnvoyConfigurationTest {
     assertThat(resolvedTemplate).contains("app_version: v1.2.3")
     assertThat(resolvedTemplate).contains("app_id: com.mydomain.myapp")
     assertThat(resolvedTemplate).contains("virtual_clusters: [test]")
+    assertThat(resolvedTemplate).contains("filter_name")
+    assertThat(resolvedTemplate).contains("test_config")
   }
 
   @Test
   fun `resolve templates with invalid templates will throw on build`() {
-    val envoyConfiguration = EnvoyConfiguration("stats.foo.com", 123, 234, 345, 456, emptyList(), 567, "v1.2.3", "com.mydomain.myapp", "[test]", emptyMap())
+    val envoyConfiguration = EnvoyConfiguration(
+      "stats.foo.com", 123, 234, 345, 456, 567, "v1.2.3", "com.mydomain.myapp", "[test]",
+      emptyList(), emptyList(), emptyMap()
+    )
 
     try {
-      envoyConfiguration.resolveTemplate("{{ missing }}", "")
+      envoyConfiguration.resolveTemplate("{{ missing }}", "", "")
       fail("Unresolved configuration keys should trigger exception.")
     } catch (e: EnvoyConfiguration.ConfigurationException) {
       assertThat(e.message).contains("missing")

mobile/library/kotlin/src/io/envoyproxy/envoymobile/EngineBuilder.kt

@@ -3,6 +3,7 @@ package io.envoyproxy.envoymobile
 import io.envoyproxy.envoymobile.engine.EnvoyConfiguration
 import io.envoyproxy.envoymobile.engine.EnvoyEngine
 import io.envoyproxy.envoymobile.engine.EnvoyEngineImpl
+import io.envoyproxy.envoymobile.engine.EnvoyNativeFilterConfig
 import io.envoyproxy.envoymobile.engine.types.EnvoyHTTPFilterFactory
 import io.envoyproxy.envoymobile.engine.types.EnvoyStringAccessor
 import java.util.UUID
@@ -27,11 +28,12 @@ open class EngineBuilder(
   private var dnsRefreshSeconds = 60
   private var dnsFailureRefreshSecondsBase = 2
   private var dnsFailureRefreshSecondsMax = 10
-  private var filterChain = mutableListOf<EnvoyHTTPFilterFactory>()
   private var statsFlushSeconds = 60
   private var appVersion = "unspecified"
   private var appId = "unspecified"
   private var virtualClusters = "[]"
+  private var platformFilterChain = mutableListOf<EnvoyHTTPFilterFactory>()
+  private var nativeFilterChain = mutableListOf<EnvoyNativeFilterConfig>()
   private var stringAccessors = mutableMapOf<String, EnvoyStringAccessor>()
 
   /**
@@ -109,7 +111,7 @@ open class EngineBuilder(
   }
 
   /**
-   * Add an HTTP filter factory used to create filters for streams sent by this client.
+   * Add an HTTP filter factory used to create platform filters for streams sent by this client.
    *
    * @param name Custom name to use for this filter factory. Useful for having
    *             more meaningful trace logs, but not required. Should be unique
@@ -118,9 +120,25 @@ open class EngineBuilder(
    *
    * @return this builder.
    */
-  fun addFilter(name: String = UUID.randomUUID().toString(), factory: () -> Filter):
+  fun addPlatformFilter(name: String = UUID.randomUUID().toString(), factory: () -> Filter):
     EngineBuilder {
-      this.filterChain.add(FilterFactory(name, factory))
+      this.platformFilterChain.add(FilterFactory(name, factory))
+      return this
+    }
+
+  /**
+   * Add an HTTP filter config used to create native filters for streams sent by this client.
+   *
+   * @param name Custom name to use for this filter factory. Useful for having
+   *             more meaningful trace logs, but not required. Should be unique
+   *             per filter.
+   * @param typedConfig config string for the filter.
+   *
+   * @return this builder.
+   */
+  fun addNativeFilter(name: String = UUID.randomUUID().toString(), typedConfig: String):
+    EngineBuilder {
+      this.nativeFilterChain.add(EnvoyNativeFilterConfig(name, typedConfig))
       return this
     }
 
@@ -201,7 +219,8 @@ open class EngineBuilder(
           EnvoyConfiguration(
             statsDomain, connectTimeoutSeconds,
             dnsRefreshSeconds, dnsFailureRefreshSecondsBase, dnsFailureRefreshSecondsMax,
-            filterChain, statsFlushSeconds, appVersion, appId, virtualClusters, stringAccessors
+            statsFlushSeconds, appVersion, appId, virtualClusters, nativeFilterChain,
+            platformFilterChain, stringAccessors
           ),
           logLevel, onEngineRunning
         )