Skip to content

Commit

Permalink
moving stats to providers
Browse files Browse the repository at this point in the history
  • Loading branch information
@matjaz99
matjaz99 committed Dec 18, 2022
1 parent 3814bce commit 7285369
Show file tree
Hide file tree
Showing 12 changed files with 164 additions and 97 deletions.
21 changes: 8 additions & 13 deletions src/main/java/si/matjazcerkvenik/alertmonitor/data/DAO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
*/
package si.matjazcerkvenik.alertmonitor.data;

import si.matjazcerkvenik.alertmonitor.model.config.ConfigReader;
import si.matjazcerkvenik.alertmonitor.model.config.ProviderConfig;
import si.matjazcerkvenik.alertmonitor.providers.AbstractDataProvider;
import si.matjazcerkvenik.alertmonitor.providers.EventloggerDataProvider;
Expand All @@ -38,10 +37,6 @@ public class DAO {
/** A map of dataproviders. Key is name. */
private Map<String, AbstractDataProvider> dataProviders = new HashMap<>();

/** Map of warnings in the alertmonitor. */
private Map<String, String> warnings = new HashMap<>();



private DAO() {
if (AmProps.yamlConfig != null) {
Expand Down Expand Up @@ -163,16 +158,16 @@ public List<AbstractDataProvider> getAllDataProviders() {
// }


public void addWarning(String msgKey, String msg) {
warnings.put(msgKey, msg);
}

public void removeWarning(String msgKey) {
warnings.remove(msgKey);
public void addWarningToAllProviders(String msgKey, String msg) {
for (AbstractDataProvider adp : dataProviders.values()) {
adp.addWarning(msgKey, msg);
}
}

public List<String> getWarnings() {
return new ArrayList<>(warnings.values());
public void removeWarningFromAllProviders(String msgKey) {
for (AbstractDataProvider adp : dataProviders.values()) {
adp.removeWarning(msgKey);
}
}

}
36 changes: 18 additions & 18 deletions src/main/java/si/matjazcerkvenik/alertmonitor/data/MongoDbDataManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,12 +94,12 @@ public void addWebhookMessage(WebhookMessage message) {
// insert one doc
collection.insertOne(doc);

DAO.getInstance().removeWarning("mongo");
DAO.getInstance().removeWarningFromAllProviders("mongo");
AmMetrics.alertmonitor_db_inserts_total.labels("webhook").inc();

} catch (Exception e) {
logger.error("MongoDbDataManager: addWebhookMessage: Exception: " + e.getMessage());
DAO.getInstance().addWarning("mongo", "No connection to DB");
DAO.getInstance().addWarningToAllProviders("mongo", "No connection to DB");
AmMetrics.alertmonitor_db_failures_total.labels().inc();
}

Expand Down Expand Up @@ -149,14 +149,14 @@ public List<WebhookMessage> getWebhookMessages() {
webhookMessageList.add(m);
}

DAO.getInstance().removeWarning("mongo");
DAO.getInstance().removeWarningFromAllProviders("mongo");
AmMetrics.alertmonitor_db_queries_total.labels("webhook").inc();

return webhookMessageList;

} catch (Exception e) {
logger.error("MongoDbDataManager: getWebhookMessages: Exception: ", e);
DAO.getInstance().addWarning("mongo", "No connection to DB");
DAO.getInstance().addWarningToAllProviders("mongo", "No connection to DB");
AmMetrics.alertmonitor_db_failures_total.labels().inc();
}
return null;
Expand All @@ -183,12 +183,12 @@ public void addToJournal(List<DEvent> events) {

collection.insertMany(list, new InsertManyOptions().ordered(false));

DAO.getInstance().removeWarning("mongo");
DAO.getInstance().removeWarningFromAllProviders("mongo");
AmMetrics.alertmonitor_db_inserts_total.labels("journal").inc();

} catch (Exception e) {
logger.error("MongoDbDataManager: addToJournal(): Exception: " + e.getMessage());
DAO.getInstance().addWarning("mongo", "No connection to DB");
DAO.getInstance().addWarningToAllProviders("mongo", "No connection to DB");
AmMetrics.alertmonitor_db_failures_total.labels().inc();
}

Expand Down Expand Up @@ -216,14 +216,14 @@ public List<DEvent> getJournal() {
eventList.add(e);
}

DAO.getInstance().removeWarning("mongo");
DAO.getInstance().removeWarningFromAllProviders("mongo");
AmMetrics.alertmonitor_db_queries_total.labels("journal").inc();

return eventList;

} catch (Exception e) {
logger.error("MongoDbDataManager: getJournal: Exception: " + e.getMessage());
DAO.getInstance().addWarning("mongo", "No connection to DB");
DAO.getInstance().addWarningToAllProviders("mongo", "No connection to DB");
AmMetrics.alertmonitor_db_failures_total.labels().inc();
}
return null;
Expand All @@ -237,14 +237,14 @@ public long getJournalSize() {
MongoDatabase db = mongoClient.getDatabase(AmProps.ALERTMONITOR_MONGODB_DB_NAME);
MongoCollection<Document> collection = db.getCollection("journal");

DAO.getInstance().removeWarning("mongo");
DAO.getInstance().removeWarningFromAllProviders("mongo");
AmMetrics.alertmonitor_db_queries_total.labels("journal").inc();

return collection.countDocuments();

} catch (Exception e) {
logger.error("MongoDbDataManager: getJournalSize: Exception: " + e.getMessage());
DAO.getInstance().addWarning("mongo", "No connection to DB");
DAO.getInstance().addWarningToAllProviders("mongo", "No connection to DB");
AmMetrics.alertmonitor_db_failures_total.labels().inc();
}
return -1;
Expand All @@ -260,14 +260,14 @@ public int getNumberOfAlertsInLastHour() {

Bson filter = Filters.gte("timestamp", System.currentTimeMillis() - 3600 * 1000);

DAO.getInstance().removeWarning("mongo");
DAO.getInstance().removeWarningFromAllProviders("mongo");
AmMetrics.alertmonitor_db_queries_total.labels("journal").inc();

return (int) collection.countDocuments(filter);

} catch (Exception e) {
logger.error("MongoDbDataManager: getNumberOfAlertsInLastHour: Exception: " + e.getMessage());
DAO.getInstance().addWarning("mongo", "No connection to DB");
DAO.getInstance().addWarningToAllProviders("mongo", "No connection to DB");
AmMetrics.alertmonitor_db_failures_total.labels().inc();
}
return -1;
Expand All @@ -292,13 +292,13 @@ public DEvent getEvent(String id) {

Document doc = collection.find(Filters.eq("uid", id)).first();
DEvent event = convertToDEvent(doc);
DAO.getInstance().removeWarning("mongo");
DAO.getInstance().removeWarningFromAllProviders("mongo");
AmMetrics.alertmonitor_db_queries_total.labels("journal").inc();
return event;

} catch (Exception e) {
logger.error("MongoDbDataManager: getEvent: Exception: " + e.getMessage());
DAO.getInstance().addWarning("mongo", "No connection to DB");
DAO.getInstance().addWarningToAllProviders("mongo", "No connection to DB");
AmMetrics.alertmonitor_db_failures_total.labels().inc();
}
return null;
Expand Down Expand Up @@ -360,11 +360,11 @@ public void cleanDB() {
logger.info("MongoDbDataManager: cleanDB [journal]: result" + resultDeleteMany2);
AmMetrics.alertmonitor_db_deletes_total.labels("journal").inc();

DAO.getInstance().removeWarning("mongo");
DAO.getInstance().removeWarningFromAllProviders("mongo");

} catch (Exception e) {
logger.error("MongoDbDataManager: cleanDB: Exception: " + e.getMessage());
DAO.getInstance().addWarning("mongo", "No connection to DB");
DAO.getInstance().addWarningToAllProviders("mongo", "No connection to DB");
AmMetrics.alertmonitor_db_failures_total.labels().inc();
}
}
Expand All @@ -385,11 +385,11 @@ public void handleAlarmClearing(DEvent clearEvent) {
logger.info("MongoDbDataManager: handleAlarmClearing: result" + updateResult);
AmMetrics.alertmonitor_db_updates_total.labels("journal").inc();

DAO.getInstance().removeWarning("mongo");
DAO.getInstance().removeWarningFromAllProviders("mongo");

} catch (Exception e) {
logger.error("MongoDbDataManager: handleAlarmClearing: Exception: " + e.getMessage());
DAO.getInstance().addWarning("mongo", "No connection to DB");
DAO.getInstance().addWarningToAllProviders("mongo", "No connection to DB");
AmMetrics.alertmonitor_db_failures_total.labels().inc();
}
}
Expand Down
38 changes: 20 additions & 18 deletions src/main/java/si/matjazcerkvenik/alertmonitor/model/prometheus/PrometheusApiClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
import com.google.gson.GsonBuilder;
import okhttp3.*;
import si.matjazcerkvenik.alertmonitor.data.DAO;
import si.matjazcerkvenik.alertmonitor.providers.AbstractDataProvider;
import si.matjazcerkvenik.alertmonitor.providers.PrometheusDataProvider;
import si.matjazcerkvenik.alertmonitor.util.AmMetrics;
import si.matjazcerkvenik.alertmonitor.util.AmProps;
import si.matjazcerkvenik.alertmonitor.util.HttpClientFactory;
Expand All @@ -41,33 +43,33 @@ public class PrometheusApiClient {

private final SimpleLogger logger = LogFactory.getLogger();

/** Name of this client - provider name */
private String name = ".default";

private final String HTTP_CLIENT_USER_AGENT = "Alertmonitor/v1";

private static long requestCount;

private AbstractDataProvider dataProvider;

/* Extracted from providerConfig */
/** Name of this client - provider name */
private String name = ".default";
private String server;
private boolean secureClient = false;
private int connectTimeout = 10;
private int readTimeout = 120;

public PrometheusApiClient(boolean secure, int connectTimeout, int readTimeout, String server) {
this.secureClient = secure;
this.connectTimeout = connectTimeout;
this.readTimeout = readTimeout;
this.server = server;
public PrometheusApiClient(AbstractDataProvider dataProvider) {
this.dataProvider = dataProvider;
name = dataProvider.getProviderConfig().getName();
server = dataProvider.getProviderConfig().getParam(PrometheusDataProvider.DP_PARAM_KEY_SERVER);
secureClient = server.startsWith("https");
connectTimeout = Integer.parseInt(dataProvider.getProviderConfig().getParam(PrometheusDataProvider.DP_PARAM_KEY_CLIENT_CONNECT_TIMEOUT_SEC));
readTimeout = Integer.parseInt(dataProvider.getProviderConfig().getParam(PrometheusDataProvider.DP_PARAM_KEY_CLIENT_READ_TIMEOUT_SEC));
}

public String getName() {
return name;
}

public void setName(String name) {
this.name = name;
}

/**
* Execute a simple query
* @param query the query
Expand Down Expand Up @@ -271,32 +273,32 @@ private String execute(Request request) throws PrometheusApiException {

response.close();

DAO.getInstance().removeWarning("prom_api");
dataProvider.removeWarning("prom_api");

} catch (UnknownHostException e) {
logger.error("PrometheusApi[" + name + "]: request[" + requestCount + "] failed: UnknownHostException: " + e.getMessage());
code = "0";
DAO.getInstance().addWarning("prom_api", "Prometheus API not reachable");
dataProvider.addWarning("prom_api", "Prometheus API not reachable");
throw new PrometheusApiException("Unknown Host");
} catch (SocketTimeoutException e) {
logger.error("PrometheusApi[" + name + "]: request[" + requestCount + "] failed: SocketTimeoutException: " + e.getMessage());
code = "0";
DAO.getInstance().addWarning("prom_api", "Prometheus API not reachable");
dataProvider.addWarning("prom_api", "Prometheus API not reachable");
throw new PrometheusApiException("Timeout");
} catch (SocketException e) {
logger.error("PrometheusApi[" + name + "]: request[" + requestCount + "] failed: SocketException: " + e.getMessage());
code = "0";
DAO.getInstance().addWarning("prom_api", "Prometheus API not reachable");
dataProvider.addWarning("prom_api", "Prometheus API not reachable");
throw new PrometheusApiException("Socket Error");
} catch (SSLException e) {
logger.error("PrometheusApi[" + name + "]: request[" + requestCount + "] failed: SSLException: " + e.getMessage());
code = "0";
DAO.getInstance().addWarning("prom_api", "Prometheus API not reachable");
dataProvider.addWarning("prom_api", "Prometheus API not reachable");
throw new PrometheusApiException("SSL Exception");
} catch (Exception e) {
logger.error("PrometheusApi[" + name + "]: request[" + requestCount + "] failed: Exception: ", e);
code = "0";
DAO.getInstance().addWarning("prom_api", "Prometheus API not reachable");
dataProvider.addWarning("prom_api", "Prometheus API not reachable");
throw new PrometheusApiException("Unknown Exception");
} finally {
double duration = (System.currentTimeMillis() - before) * 1.0 / 1000;
Expand Down
8 changes: 5 additions & 3 deletions src/main/java/si/matjazcerkvenik/alertmonitor/model/prometheus/PrometheusApiClientPool.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
*/
package si.matjazcerkvenik.alertmonitor.model.prometheus;

import si.matjazcerkvenik.alertmonitor.providers.AbstractDataProvider;
import si.matjazcerkvenik.alertmonitor.providers.PrometheusDataProvider;
import si.matjazcerkvenik.alertmonitor.util.LogFactory;

import java.util.ArrayList;
Expand All @@ -25,10 +27,10 @@ public class PrometheusApiClientPool {
private int count = 0;
private List<PrometheusApiClient> pool = new ArrayList<>();

public PrometheusApiClientPool(String name, int poolSize, boolean secure, int connectTimeout, int readTimeout, String server) {
public PrometheusApiClientPool(AbstractDataProvider dataProvider) {
Integer poolSize = Integer.parseInt(dataProvider.getProviderConfig().getParam(PrometheusDataProvider.DP_PARAM_KEY_CLIENT_POOL_SIZE));
for (int i = 0; i < poolSize; i++) {
PrometheusApiClient c = new PrometheusApiClient(secure, connectTimeout, readTimeout, server);
c.setName(name);
PrometheusApiClient c = new PrometheusApiClient(dataProvider);
pool.add(c);
}
}
Expand Down
31 changes: 20 additions & 11 deletions src/main/java/si/matjazcerkvenik/alertmonitor/providers/AbstractDataProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@ public abstract class AbstractDataProvider {

protected Timer syncTimer = null;

/** Map of warnings in this data provider. */
private Map<String, String> warnings = new HashMap<>();

protected long webhookMessagesReceivedCount = 0;
protected long journalReceivedCount = 0;
protected long lastEventTimestamp = 0;
Expand All @@ -66,13 +69,7 @@ public void setProviderConfig(ProviderConfig providerConfig) {
public void init() {
// TODO error handling!
LogFactory.getLogger().info(providerConfig.toString());
String server = providerConfig.getParam(PrometheusDataProvider.DP_PARAM_KEY_SERVER);
Boolean secure = server.startsWith("https");
Integer poolSize = Integer.parseInt(providerConfig.getParam(PrometheusDataProvider.DP_PARAM_KEY_CLIENT_POOL_SIZE));
Integer connTimeout = Integer.parseInt(providerConfig.getParam(PrometheusDataProvider.DP_PARAM_KEY_CLIENT_CONNECT_TIMEOUT_SEC));
Integer readTimeout = Integer.parseInt(providerConfig.getParam(PrometheusDataProvider.DP_PARAM_KEY_CLIENT_READ_TIMEOUT_SEC));

prometheusApiClientPool = new PrometheusApiClientPool(providerConfig.getName(), poolSize, secure, connTimeout, readTimeout, server);
prometheusApiClientPool = new PrometheusApiClientPool(this);
restartSyncTimer();
}

Expand Down Expand Up @@ -102,10 +99,6 @@ public List<DEvent> getJournal() {
return DAO.getInstance().getDataManager().getJournal();
}

public long getJournalSize() {
return DAO.getInstance().getDataManager().getJournalSize();
}

public DEvent getEvent(String id) {
DEvent event = DAO.getInstance().getDataManager().getEvent(id);
return event;
Expand Down Expand Up @@ -350,6 +343,18 @@ public List<String> getActiveTargets() {
return new ArrayList<>(map.keySet());
}

public void addWarning(String msgKey, String msg) {
warnings.put(msgKey, msg);
}

public void removeWarning(String msgKey) {
warnings.remove(msgKey);
}

public List<String> getWarnings() {
return new ArrayList<>(warnings.values());
}

public abstract void restartSyncTimer();

public long getWebhookMessagesReceivedCount() {
Expand All @@ -360,6 +365,10 @@ public long getJournalCount() {
return journalReceivedCount;
}

public long getJournalSize() {
return DAO.getInstance().getDataManager().getJournalSize();
}

public long getLastEventTimestamp() {
return lastEventTimestamp;
}
Expand Down
6 changes: 3 additions & 3 deletions src/main/java/si/matjazcerkvenik/alertmonitor/providers/PrometheusSyncTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public void run() {
logger.error("SYNC: null response returned");
logger.info("SYNC: === Periodic synchronization complete ===");
dataProvider.syncFailedCount++;
DAO.getInstance().addWarning("sync", "Synchronization is failing");
dataProvider.addWarning("sync", "Synchronization is failing");
return;
}

Expand Down Expand Up @@ -138,13 +138,13 @@ public void run() {

dataProvider.syncSuccessCount++;
AmMetrics.alertmonitor_sync_success.labels(dataProvider.providerConfig.getName()).set(1);
DAO.getInstance().removeWarning("sync");
dataProvider.removeWarning("sync");

} catch (Exception e) {
logger.error("SYNC: failed to synchronize alarms; root cause: " + e.getMessage());
dataProvider.syncFailedCount++;
AmMetrics.alertmonitor_sync_success.labels(dataProvider.providerConfig.getName()).set(0);
DAO.getInstance().addWarning("sync", "Synchronization is failing");
dataProvider.addWarning("sync", "Synchronization is failing");
} finally {
dataProvider.getPrometheusApiClientPool().returnClient(api);
}
Expand Down
Loading

0 comments on commit 7285369

Please sign in to comment.