Initial commit of dynamic loader.

hydra-essentials/src/main/java/com/addthis/hydra/common/plugins/DynamicLoader.java

@@ -0,0 +1,152 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.addthis.hydra.common.plugins;
+
+import javax.annotation.Nonnull;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.base.Joiner;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.supercsv.exception.SuperCsvException;
+import org.supercsv.io.CsvListReader;
+import org.supercsv.prefs.CsvPreference;
+
+public class DynamicLoader {
+
+    private static final Logger log = LoggerFactory.getLogger(DynamicLoader.class);
+
+    private static final CsvPreference csvParserOptions =
+            new CsvPreference.Builder(CsvPreference.STANDARD_PREFERENCE)
+                    .surroundingSpacesNeedQuotes(true)
+                    .build();
+
+    /**
+     * Looks up the system property specified as input.
+     * Treats the value associated with the input as a URI.
+     * The contents of the URI should be a CSV file.
+     * Each row of the the file should have at least two columns.
+     * The first column is the fully qualified class name of
+     * a class to load. The second column specifies a URI which
+     * the classloader will use to search for the class.
+     * Executable targets should be specified with four columns:
+     * [fully qualified class name], [uri], "executable", [name]
+     * The fourth column is the execution name to be associated with
+     * the class.
+     *
+     * @param propertyName system property associated with a URI
+     * @return map of execution target names to fully qualified class names
+     */
+    public static Map<String,String> readDynamicClasses(@Nonnull String propertyName) {
+        Map<String,String> executables = new HashMap<>();
+        String urlSpecifier = System.getProperty(propertyName);
+        if (urlSpecifier == null) {
+            return executables;
+        }
+        URL url;
+        try {
+            url = new URL(urlSpecifier);
+        } catch (MalformedURLException ex) {
+            log.error("The value for the system property {} " +
+                      "is \"{}\" cannot be translated into a valid URL.",
+                    propertyName, urlSpecifier);
+            return executables;
+        }
+        Set<URL> jarLocations = new HashSet<>();
+        Set<String> classNames = new HashSet<>();
+        readDynamicClasses(urlSpecifier, url, jarLocations, classNames, executables);
+        loadDynamicClasses(jarLocations, classNames);
+        return executables;
+    }
+
+    private static void loadDynamicClasses(Set<URL> jarLocations,
+            Set<String> classNames) {
+        String jarString = Joiner.on(", ").join(jarLocations);
+        if (jarLocations.size() > 0) {
+            URLClassLoader loader = new URLClassLoader(
+                    jarLocations.toArray(new URL[jarLocations.size()]));
+            for(String className : classNames) {
+                try {
+                    loader.loadClass(className);
+                } catch (ClassNotFoundException ex) {
+                    log.error("ClassNotFoundException when attempting" +
+                              "to load {} from classpath {}",
+                            className, jarString);
+                }
+            }
+        }
+    }
+
+    private static void readDynamicClasses(String urlSpecifier,
+            URL url, Set<URL> jarLocations,
+            Set<String> classNames, Map<String,String> executables) {
+        InputStreamReader reader = null;
+        try {
+            InputStream stream = url.openStream();
+            reader = new InputStreamReader(stream);
+            CsvListReader csvParser = new CsvListReader(reader, csvParserOptions);
+            List<String> next;
+            while ((next = csvParser.read()) != null) {
+                String[] fields = next.toArray(new String[next.size()]);
+                try {
+                    if (fields.length >= 2) {
+                        classNames.add(fields[0]);
+                        jarLocations.add(new URL(fields[1]));
+                    }
+                    if (fields.length >= 4) {
+                        if (fields[2].equals("executable")) {
+                            executables.put(fields[3], fields[0]);
+                        }
+                    }
+                } catch (MalformedURLException ex) {
+                    if (fields.length >= 2) {
+                        log.error("Could not parse url \"{}\" from {}",
+                                fields[1], urlSpecifier);
+                    } else {
+                        log.error("Unexpected error in {}: {}",
+                                urlSpecifier, ex);
+                    }
+                }
+            }
+        } catch (IOException ex) {
+            log.error("Error attempting to read from {}: {}", urlSpecifier, ex);
+        } catch (SuperCsvException ex) {
+            log.error("CSV parse error in {}: {}", urlSpecifier, ex);
+        } finally {
+            if (reader != null) {
+                try {
+                    reader.close();
+                } catch(IOException ex) {
+                    log.error("Error closing stream: ", ex);
+                }
+            }
+        }
+    }
+
+
+}

hydra-essentials/src/main/java/com/addthis/hydra/common/plugins/PluginReader.java

@@ -13,6 +13,8 @@
  */
 package com.addthis.hydra.common.plugins;
 
+import javax.annotation.Nonnull;
+
 import java.io.IOException;
 import java.io.InputStreamReader;
 
@@ -35,9 +37,11 @@ public class PluginReader {
 
     private static final Logger log = LoggerFactory.getLogger(PluginReader.class);
 
-    private static final PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
+    private static final PathMatchingResourcePatternResolver resolver =
+            new PathMatchingResourcePatternResolver();
 
-    private static final CsvPreference csvParserOptions = new CsvPreference.Builder(CsvPreference.STANDARD_PREFERENCE)
+    private static final CsvPreference csvParserOptions =
+            new CsvPreference.Builder(CsvPreference.STANDARD_PREFERENCE)
             .surroundingSpacesNeedQuotes(true)
             .build();
 
@@ -50,7 +54,7 @@ public class PluginReader {
      * @param suffix filename suffix. Matches against files with this suffix in their filename.
      * @return list of String[] elements that represent CSV values
      */
-    public static List<String[]> readProperties(String suffix) {
+    public static List<String[]> readProperties(@Nonnull String suffix) {
         List<String[]> result = new ArrayList<>();
         try {
             String locationPattern = "classpath*:plugins/*" + suffix;
@@ -84,7 +88,7 @@ public static List<String[]> readProperties(String suffix) {
      * @param suffix filename suffix. Matches against files with this suffix in their filename.
      * @return list of String[] elements that represent CSV values
      */
-    public static Map<Resource, List<String[]>> readPropertiesAndMap(String suffix) {
+    public static Map<Resource, List<String[]>> readPropertiesAndMap(@Nonnull String suffix) {
         Map<Resource, List<String[]>> result = new HashMap<>();
         try {
             String locationPattern = "classpath*:plugins/*" + suffix;
@@ -113,11 +117,13 @@ public static Map<Resource, List<String[]>> readPropertiesAndMap(String suffix)
      * A workaround the plugin framework does not work in junit
      * but we do not use the plugin framework in junit tests.
      */
-    private static void ignoreJunitEnvironment(ClassNotFoundException e, String suffix, String key) {
+    private static void ignoreJunitEnvironment(ClassNotFoundException e,
+            String suffix, String key) {
         boolean junitRunning = false;
         assert (junitRunning = true);
         if (!junitRunning) {
-            log.warn("registerPlugin failure. File suffix is \"{}\", key is \"{}\", exception is {}.",
+            log.warn("registerPlugin failure. File suffix is \"{}\"," +
+                     " key is \"{}\", exception is {}.",
                     suffix, key, e.toString());
         }
     }
@@ -130,7 +136,8 @@ private static void ignoreJunitEnvironment(ClassNotFoundException e, String suff
      * @param map
      * @param parentClass
      */
-    public static void registerPlugin(String suffix, Codec.ClassMap map, Class parentClass) {
+    public static void registerPlugin(@Nonnull String suffix,
+            @Nonnull Codec.ClassMap map, @Nonnull Class parentClass) {
         List<String[]> filters = PluginReader.readProperties(suffix);
         for (String[] filter : filters) {
             if (filter.length >= 2) {
@@ -140,7 +147,8 @@ public static void registerPlugin(String suffix, Codec.ClassMap map, Class paren
         }
     }
 
-    public static Class loadClass(String suffix, String key, String className, Class parentClass) {
+    public static Class loadClass(@Nonnull String suffix, @Nonnull String key,
+            @Nonnull String className, @Nonnull Class parentClass) {
         try {
             Class clazz = Class.forName(className);
             if (parentClass.isAssignableFrom(clazz)) {
@@ -164,7 +172,8 @@ public static Class loadClass(String suffix, String key, String className, Class
      * @param suffix
      * @param map
      */
-    public static void registerLazyPlugin(String suffix, Map<String,String> map) {
+    public static void registerLazyPlugin(@Nonnull String suffix,
+            @Nonnull Map<String,String> map) {
         List<String[]> filters = PluginReader.readProperties(suffix);
         for (String[] filter : filters) {
             if (filter.length >= 2) {
@@ -181,7 +190,8 @@ public static void registerLazyPlugin(String suffix, Map<String,String> map) {
      * @param map
      * @param parentClass
      */
-    public static void registerPlugin(String suffix, Map<String, Class> map, Class parentClass) {
+    public static void registerPlugin(@Nonnull String suffix,
+            @Nonnull Map<String, Class> map, @Nonnull Class parentClass) {
         List<String[]> filters = PluginReader.readProperties(suffix);
         for (String[] filter : filters) {
             if (filter.length >= 2) {

hydra-main/src/main/java/com/addthis/hydra/Main.java

@@ -31,6 +31,7 @@
 import com.addthis.basis.util.Strings;
 
 import com.addthis.codec.Codec;
+import com.addthis.hydra.common.plugins.DynamicLoader;
 import com.addthis.hydra.common.plugins.PluginReader;
 import com.addthis.hydra.data.filter.bundle.BundleFilter;
 import com.addthis.hydra.data.query.CLIQuery;
@@ -70,7 +71,9 @@ public static void registerFilter(String name, String clazz) {
 
     /** register types */
     static {
+        Map<String,String> extras = DynamicLoader.readDynamicClasses("hydra.loader");
         PluginReader.registerLazyPlugin("-executables.classmap", cmap);
+        cmap.putAll(extras);
     }