working on data providers

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.4.2-SNAPSHOT
+
+* [FEATURE] Data providers
+
 ## 2.4.1-SNAPSHOT
 
 * [ENHANCEMENT] GUI enhancements: Targets view improved grid
@@ -7,7 +11,7 @@
 
 * [FEATURE] Version check at start
 * [FEATURE] Internal report, overview of the system, KPIs, line-charts...
-* [ENHANCEMENT] Pool of Prometheus http clients (preferably 1)
+* [ENHANCEMENT] Pool of Prometheus http clients (preferably 1 to avoid parallelism)
 * [ENHANCEMENT] GUI enhancements: scroll-up button
 
 ## 2.3.0 / 2022-04-09

alertmonitor-config.yml

@@ -0,0 +1,26 @@
+version: v1
+
+providers:
+  - name: prom1
+    source: prometheus
+    uri: /alertmonitor/webhook/prom1
+    params:
+      server: http://prom-1:9090
+      sync: 60s
+  - name: prom2
+    source: prometheus
+    uri: /alertmonitor/webhook/prom2
+    params:
+      server: http://prom-2:9090
+      clientPoolSize: 1
+      clientReadTimeout: 120s
+    data:
+      retention: 7d
+    sync:
+      interval: 60s
+  - name: cluster-1
+    source: eventlogger
+    uri: /alertmonitor/webhook/eventlogger
+    params:
+      server: http://prom-1:9090
+      sync: 60s

docs/Todo.md

@@ -7,13 +7,12 @@
 - tags: filter by operators AND or OR
 - redesigned journal (not table, but a list with show details)
 - copy to clipboard button
-- show targets by job
 - configure labels for CID (if needed also from external_labels)
-- store alarms in MongoDB
 - function time_of_max
 - downsample - from 1 min (ie 4 metrics) create 1 metric
 - generate report, configure data in report, configure query for report
-- add buttons when selecting dates for query_range (1h ago, 7d ago, ...,  now)
+- metric simulator
+- tenants, different providers, data sources
 
 
 https://www.primefaces.org/showcase/ui/data/timeline/basic.xhtml

docs/config/config.yml

@@ -7,9 +7,10 @@ global:
 providers:
   - source: prometheus
     name: prom1
-    server: http://prom-1:9090
-    sync:
-      interval: 60s
+    params:
+      server: http://prom-1:9090
+      sync:
+        interval: 60s
   - source: prometheus
     name: prom2
     server: http://prom-2:9090

pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>si.matjazcerkvenik</groupId>
     <artifactId>alertmonitor</artifactId>
-    <version>2.0.0-SNAPSHOT</version>
+    <version>3.0.0-SNAPSHOT</version>
     <packaging>war</packaging>
 
     <name>alertmonitor</name>
@@ -107,6 +107,12 @@
             <artifactId>mongo-java-driver</artifactId>
             <version>${mongodb-driver.version}</version>
         </dependency>
+<!--        parsing yaml-->
+        <dependency>
+            <groupId>org.yaml</groupId>
+            <artifactId>snakeyaml</artifactId>
+            <version>1.33</version>
+        </dependency>
     </dependencies>
 
     <build>

src/main/java/si/matjazcerkvenik/alertmonitor/data/DAO.java

@@ -16,6 +16,7 @@
 package si.matjazcerkvenik.alertmonitor.data;
 
 import si.matjazcerkvenik.alertmonitor.model.*;
+import si.matjazcerkvenik.alertmonitor.model.config.ProviderConfig;
 import si.matjazcerkvenik.alertmonitor.model.prometheus.*;
 import si.matjazcerkvenik.alertmonitor.util.*;
 import si.matjazcerkvenik.alertmonitor.util.Formatter;
@@ -34,6 +35,8 @@ public class DAO {
 
     private IDataManager dataManager;
 
+    private Map<String, DataProvider> dataProviders = new HashMap<>();
+
     /** Map of active alerts. Key is correlation id */
     private Map<String, DEvent> activeAlerts = new HashMap<>();
 
@@ -46,6 +49,22 @@ public class DAO {
 
 
     private DAO() {
+        if (AmProps.yamlConfig != null) {
+            for (ProviderConfig pc : AmProps.yamlConfig.getProviders()) {
+                DataProvider dp = new DataProvider();
+                dp.setProviderConfig(pc);
+                dataProviders.put(pc.getUri(), dp);
+            }
+        }
+        // create default provider if not configured
+        if (!dataProviders.containsKey("/alertmonitor/webhook")) {
+            DataProvider defaultDP = new DataProvider();
+            defaultDP.getProviderConfig().setName(".default");
+            defaultDP.getProviderConfig().setSource("prometheus");
+            defaultDP.getProviderConfig().setUri("/alertmonitor/webhook");
+            dataProviders.put("/alertmonitor/webhook", defaultDP);
+        }
+
         if (AmProps.ALERTMONITOR_MONGODB_ENABLED) {
             dataManager = new MongoDbDataManager();
         } else {
@@ -76,6 +95,10 @@ public void resetDataManager() {
         TaskManager.getInstance().startDbMaintenanceTimer();
     }
 
+    public DataProvider getDataProvider(String name) {
+        return dataProviders.getOrDefault(name, null);
+    }
+
     /**
      * Add new webhook message to the list. Also delete oldest messages.
      * @param message incoming message

src/main/java/si/matjazcerkvenik/alertmonitor/model/DataProvider.java

@@ -0,0 +1,22 @@
+package si.matjazcerkvenik.alertmonitor.model;
+
+import si.matjazcerkvenik.alertmonitor.model.config.ProviderConfig;
+import si.matjazcerkvenik.alertmonitor.web.WebhookMessage;
+
+public class DataProvider {
+
+    private ProviderConfig providerConfig;
+
+    public ProviderConfig getProviderConfig() {
+        return providerConfig;
+    }
+
+    public void setProviderConfig(ProviderConfig providerConfig) {
+        this.providerConfig = providerConfig;
+    }
+
+    public void processIncomingEvent(WebhookMessage m) {
+
+    }
+
+}

src/main/java/si/matjazcerkvenik/alertmonitor/model/config/Config.java

@@ -0,0 +1,35 @@
+package si.matjazcerkvenik.alertmonitor.model.config;
+
+import java.util.List;
+
+public class Config {
+
+    private String version;
+
+    private List<ProviderConfig> providers;
+
+
+    public String getVersion() {
+        return version;
+    }
+
+    public void setVersion(String version) {
+        this.version = version;
+    }
+
+
+    public List<ProviderConfig> getProviders() {
+        return providers;
+    }
+
+    public void setProviders(List<ProviderConfig> providers) {
+        this.providers = providers;
+    }
+
+    @Override
+    public String toString() {
+        return "Config{" +
+                "providers=" + providers +
+                '}';
+    }
+}

src/main/java/si/matjazcerkvenik/alertmonitor/model/config/ConfigReader.java

@@ -0,0 +1,30 @@
+package si.matjazcerkvenik.alertmonitor.model.config;
+
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.constructor.Constructor;
+import si.matjazcerkvenik.alertmonitor.util.LogFactory;
+
+import java.io.*;
+
+public class ConfigReader {
+
+    public static void main(String... args) {
+        ConfigReader.loadYaml("alertmonitor-config.yml");
+    }
+
+    public static Config loadYaml(String path) {
+        Yaml yaml = new Yaml(new Constructor(Config.class));
+        File f = new File(path);
+        InputStream inputStream;
+        try {
+            inputStream = new FileInputStream(f);
+        } catch (FileNotFoundException e) {
+            LogFactory.getLogger().warn("no alertmonitor-config.yml found at " +  path);
+            return null;
+        }
+        Config config = yaml.load(inputStream);
+        LogFactory.getLogger().info("config loaded: " + config.toString());
+        return config;
+    }
+
+}

src/main/java/si/matjazcerkvenik/alertmonitor/model/config/ProviderConfig.java

@@ -0,0 +1,63 @@
+package si.matjazcerkvenik.alertmonitor.model.config;
+
+import java.util.Map;
+
+public class ProviderConfig {
+
+    private String id;
+    private String name;
+    private String source;
+    private String uri;
+    private Map<String, Object> params;
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getSource() {
+        return source;
+    }
+
+    public void setSource(String source) {
+        this.source = source;
+    }
+
+    public String getUri() {
+        return uri;
+    }
+
+    public void setUri(String uri) {
+        this.uri = uri;
+    }
+
+    public Map<String, Object> getParams() {
+        return params;
+    }
+
+    public void setParams(Map<String, Object> params) {
+        this.params = params;
+    }
+
+    @Override
+    public String toString() {
+        return "ProviderConfig{" +
+                "id='" + id + '\'' +
+                ", name='" + name + '\'' +
+                ", source='" + source + '\'' +
+                ", uri='" + uri + '\'' +
+                ", params=" + params +
+                '}';
+    }
+}

src/main/java/si/matjazcerkvenik/alertmonitor/util/AmProps.java

@@ -15,6 +15,9 @@
  */
 package si.matjazcerkvenik.alertmonitor.util;
 
+import si.matjazcerkvenik.alertmonitor.model.config.Config;
+import si.matjazcerkvenik.alertmonitor.model.config.ConfigReader;
+
 import java.io.File;
 
 public class AmProps {
@@ -26,9 +29,12 @@ public class AmProps {
     public static boolean IS_CONTAINERIZED = false;
     public static String LOCAL_IP_ADDRESS;
 
-    public static boolean devEnv = false;
+    public static boolean DEV_ENV = false;
+
+    public static Config yamlConfig;
 
     /** Environment variables */
+    public static String ALERTMONITOR_DATAPROVIDERS_CONFIG_FILE = "/opt/alertmonitor-config.yml";
     public static int ALERTMONITOR_DATA_RETENTION_DAYS = 7;
     public static int ALERTMONITOR_PSYNC_INTERVAL_SEC = 300;
     public static String ALERTMONITOR_PROMETHEUS_SERVER = "http://localhost:9090";
@@ -45,6 +51,10 @@ public class AmProps {
 
     public static void loadProps() {
 
+        // first load config file
+        if (DEV_ENV) ALERTMONITOR_DATAPROVIDERS_CONFIG_FILE = "alertmonitor-config.yml";
+        yamlConfig = ConfigReader.loadYaml(ALERTMONITOR_DATAPROVIDERS_CONFIG_FILE);
+
         // read configuration from environment variables
         ALERTMONITOR_DATA_RETENTION_DAYS = Integer.parseInt(System.getenv().getOrDefault("ALERTMONITOR_DATA_RETENTION_DAYS", "30").trim());
         ALERTMONITOR_PSYNC_INTERVAL_SEC = Integer.parseInt(System.getenv().getOrDefault("ALERTMONITOR_PSYNC_INTERVAL_SEC", "60").trim());
@@ -64,7 +74,7 @@ public static void loadProps() {
         ALERTMONITOR_MONGODB_CONNECTION_STRING = System.getenv().getOrDefault("ALERTMONITOR_MONGODB_CONNECTION_STRING", "mongodb://admin:mongodbpassword@promvm:27017/?authSource=admin").trim();
 
         // set development environment, override default configuration
-        if (new File("/Users/matjaz").exists()) {
+        if (DEV_ENV) {
             ALERTMONITOR_PROMETHEUS_SERVER = "https://elasticvm/prometheus";
             ALERTMONITOR_MONGODB_ENABLED = false;
             ALERTMONITOR_MONGODB_CONNECTION_STRING = "mongodb://admin:mongodbpassword@elasticvm:27017/?authSource=admin";

src/main/java/si/matjazcerkvenik/alertmonitor/util/LogFactory.java

@@ -18,7 +18,6 @@ public static SimpleLogger getLogger() {
                 // if env variable would be set, logger will be already configured
                 // so if it was not set, I will assume this is a development environment
                 // and write logs to file in working directory
-                AmProps.devEnv = true;
                 logger.setFilename("./alertmonitor.log");
             }
         }
@@ -27,7 +26,7 @@ public static SimpleLogger getLogger() {
 
     public static SimpleLogger getAlertLog() {
         if (alertLog == null) {
-            if (AmProps.devEnv) {
+            if (AmProps.DEV_ENV) {
                 // write file in local working directory
                 alertLog = new SimpleLogger("./alerts.log");
             } else {

src/main/java/si/matjazcerkvenik/alertmonitor/util/OnStartListener.java

@@ -33,6 +33,7 @@ public void contextInitialized(ServletContextEvent servletContextEvent) {
 
         AmProps.START_UP_TIME = System.currentTimeMillis();
         AmProps.RUNTIME_ID = UUID.randomUUID().toString();
+        if (new File("/Users/matjaz").exists()) AmProps.DEV_ENV = true;
 
         LogFactory.getLogger().info("\n");
         LogFactory.getLogger().info("************************************************");