Merge branch 'master' into ibm-cos-sdk

app/index.js

@@ -25,6 +25,7 @@ const _ = require('lodash');
 const addRequestId = require('express-request-id')();
 const {router, initialize} = require('./routes/index.js');
 const log = require('./log').createLogger('razeedash-api/app/index');
+const DefaultProbes = require('./utils/probes/probe-default.js');
 const port = 3333;
 
 // Set ipv4first (changed in Node 18)
@@ -151,6 +152,7 @@ function onListening() {
   const addr = server.address();
   const bind = typeof addr === 'string' ? `pipe ${addr}` : `port ${addr.port}`;
   log.info(`🏄 razeedash-api listening on ${bind}/api`);
+  DefaultProbes.setStartupComplete(true);
 }
 
 function onError(error) {

app/routes/index.js

@@ -48,9 +48,11 @@ router.get('/v1/health', (req, res)=>{
   });
 });
 
-router.use('/kube', Kube);
 router.use(createExpressLogger('razeedash-api/api'));
 
+// Respond to /api/kube/[liveness|readiness|startup]
+router.use('/kube', Kube);
+
 router.use(asyncHandler(async (req, res, next) => {
   const db = req.app.get('db');
   req.db = db;

app/routes/kube/kube.js

@@ -1,5 +1,5 @@
 /**
-* Copyright 2019 IBM Corp. All Rights Reserved.
+* Copyright 2019,2024 IBM Corp. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -15,36 +15,41 @@
 */
 const express = require('express');
 const asyncHandler = require('express-async-handler');
+const probeUtil = require('../../utils/probes');
+
 const router = express.Router();
-const { GraphqlPubSub } = require('../../apollo/subscription');
-const pubSub = GraphqlPubSub.getInstance();
-const logger = require('../../log').createLogger('razeedash-api/kube/liveness');
-const timeInterval = 300000; //5 mintues
 
-// /kube/liveness
-router.get('/liveness', asyncHandler(async(req, res) => {
-  // does a db call to make sure we didnt disconnect
+const startupHandler = asyncHandler(async (req, res) => {
   try {
-    await require('../../apollo/models').models.Organization.findOne({});
-  } catch (err) {
-    logger.error(err, 'razeedash-api liveness probe failed due to a mongo connection issue');
-    return res.sendStatus(503);
+    const payload = await probeUtil.getStartupPayload(req);
+    return res.status(200).send(payload);
+  }
+  catch (e) {
+    return res.status(503).send('service unavailable');
   }
+});
+router.get('/startup', startupHandler);
 
-  // TODO: not real pub-sub liveness test yet, will add later
-  if (pubSub.initRetries > 5) {
-    // if the remote redis is not ready after 5 initial retries, then
-    // it is better to restart this pod, return 500 error
-    logger.error('Razeedash Api is down due to Redis pubsub connection issue, please check logs.');
-    return res.sendStatus(503);
+const readinessHandler = asyncHandler(async (req, res) => {
+  try {
+    const payload = await probeUtil.getReadinessPayload(req);
+    return res.status(200).send(payload);
   }
+  catch (e) {
+    return res.status(503).send('service unavailable');
+  }
+});
+router.get('/readiness', readinessHandler);
 
-  if (pubSub.lastPubSubMessage !== null && Date.now()- pubSub.lastPubSubMessage.time > timeInterval) {
-    // check if the most recent message received is within ${timeInterval/60000} minitue
-    logger.error(`Razeedash Api is down, haven't received any published messages within ${timeInterval/60000} minitue, please check logs.`);
-    return res.sendStatus(503);
+const livenessHandler = asyncHandler(async(req, res) => {
+  try {
+    const payload = await probeUtil.getLivenessPayload(req);
+    return res.status(200).send(payload);
+  }
+  catch (e) {
+    return res.status(503).send('service unavailable');
   }
-  return res.sendStatus(200);
-}));
+});
+router.get('/liveness', livenessHandler);
 
 module.exports = router;

app/routes/kube/kube.tests.js

@@ -0,0 +1,103 @@
+/* eslint-env node, mocha */
+/**
+ * Copyright 2024 IBM Corp. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const assert = require('assert');
+const mongodb = require('mongo-mock');
+var httpMocks = require('node-mocks-http');
+const log = require('../../log').log;
+
+const probeUtil = require('../../utils/probes');
+const defaultProbe = require('../../utils/probes/probe-default.js');
+
+const rewire = require('rewire');
+let kube = rewire('./kube');
+let db = {};
+
+describe('probes', () => {
+
+  before(async function () {
+    mongodb.max_delay = 0;
+    const MongoClient = mongodb.MongoClient;
+    db = await MongoClient.connect('someconnectstring', {});
+    db.collection('orgs');
+  });
+
+  after(function () {
+    db.close();
+  });
+
+  describe('startupProbe', () => {
+    it('should pass the default startup probe after setStartupComplete is called', async () => {
+      const startupHandler = kube.__get__('startupHandler');
+
+      const request = httpMocks.createRequest({
+        method: 'GET',
+        url: '/startup',
+        params: {},
+        log: log
+      });
+      const response = httpMocks.createResponse();
+
+      // Default impl returns failure before 'setStartupComplete' is called
+      await startupHandler(request, response);
+      assert.equal(response.statusCode, 503);
+
+      defaultProbe.setStartupComplete(true);
+
+      // Default impl returns success after 'setStartupComplete' is called
+      await startupHandler(request, response);
+      assert.equal(response.statusCode, 200);
+    });
+
+    it('should fail if the custom startup probe fails', async () => {
+      const startupHandler = kube.__get__('startupHandler');
+
+      const request = httpMocks.createRequest({
+        method: 'GET',
+        url: '/startup',
+        params: {},
+        log: log
+      });
+      const response = httpMocks.createResponse();
+
+      // Note: default probe setStartupComplete has already been called by earlier test
+
+      probeUtil.setImpl('./probe-testFailure.js');
+      await startupHandler(request, response);
+
+      assert.equal(response.statusCode, 503);
+    });
+
+    it('should succeed if the custom startup probe succeeds', async () => {
+      const startupHandler = kube.__get__('startupHandler');
+
+      const request = httpMocks.createRequest({
+        method: 'GET',
+        url: '/startup',
+        params: {},
+        log: log
+      });
+      const response = httpMocks.createResponse();
+
+      // Note: default probe setStartupComplete has already been called by earlier test
+
+      probeUtil.setImpl('./probe-testSuccess.js');
+      await startupHandler(request, response);
+
+      assert.equal(response.statusCode, 200);
+    });
+  });
+});

app/utils/probes/index.js

@@ -0,0 +1,58 @@
+/**
+ * Copyright 2024 IBM Corp. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const PROBE_DEFAULT_IMPL = require( './probe-default.js' );
+let PROBE_CUSTOM_IMPL = require( process.env.PROBE_IMPL || './probe-none.js' );
+
+/*
+Return an impl for each of the probe types:
+  Get the default probe payload.
+    If default probe impl throws an error, throw an error.
+  If module specified by PROBE_IMPL implements a probe, get the custom probe payload.
+    If custom probe impl throws an error, throw an error.
+  Return the custom payload, or the default payload if there is none.
+*/
+const PROBE_IMPL = {
+  getStartupPayload: async function( req ) {
+    const method = 'getStartupPayload';
+    const defaultPayload = await PROBE_DEFAULT_IMPL[method](req);
+    if( Object.prototype.hasOwnProperty.call(PROBE_CUSTOM_IMPL, method) ) {
+      return( await PROBE_CUSTOM_IMPL[method](req) );
+    }
+    return defaultPayload;
+  },
+  getReadinessPayload: async function( req ) {
+    const method = 'getReadinessPayload';
+    const defaultPayload = await PROBE_DEFAULT_IMPL[method](req);
+    if( Object.prototype.hasOwnProperty.call(PROBE_CUSTOM_IMPL, method) ) {
+      return( await PROBE_CUSTOM_IMPL[method](req) );
+    }
+    return defaultPayload;
+  },
+  getLivenessPayload: async function( req ) {
+    const method = 'getLivenessPayload';
+    const defaultPayload = await PROBE_DEFAULT_IMPL[method](req);
+    if( Object.prototype.hasOwnProperty.call(PROBE_CUSTOM_IMPL, method) ) {
+      return( await PROBE_CUSTOM_IMPL[method](req) );
+    }
+    return defaultPayload;
+  },
+  setImpl: function( newImpl ) {
+    PROBE_CUSTOM_IMPL = require( newImpl || './probe-none.js' );
+  }
+};
+
+module.exports = PROBE_IMPL;

app/utils/probes/probe-default.js

@@ -0,0 +1,60 @@
+/**
+ * Copyright 2024 IBM Corp. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const Models = require('../../apollo/models');
+const { GraphqlPubSub } = require('../../apollo/subscription');
+const pubSub = GraphqlPubSub.getInstance();
+const timeInterval = 300000; //5 mintues
+
+let STARTUP_COMPLETE = false;
+async function getStartupPayload() {
+  if( !STARTUP_COMPLETE ) {
+    throw new Error('startup incomplete');
+  }
+  return('startup probe successful');
+}
+
+async function getReadinessPayload() {
+  return('readiness probe successful');
+}
+
+async function getLivenessPayload() {
+  // does a db call to make sure we didnt disconnect
+  try {
+    await Models.models.Organization.findOne({});
+  } catch (err) {
+    throw new Error(`Razeedash-api liveness probe failed due to a mongo connection issue: ${err.message}`);
+  }
+
+  // TODO: not real pub-sub liveness test yet, will add later
+  if (pubSub.initRetries > 5) {
+    // if the remote redis is not ready after 5 initial retries, then
+    // it is better to restart this pod, return 500 error
+    throw new Error('Razeedash-api liveness probe failed due to Redis pubsub connection issue, please check logs');
+  }
+
+  if (pubSub.lastPubSubMessage !== null && Date.now()- pubSub.lastPubSubMessage.time > timeInterval) {
+    // check if the most recent message received is within ${timeInterval/60000} minitue
+    throw new Error(`Razeedash-api is down, haven't received any published messages within ${timeInterval/60000} minutes, please check logs`);
+  }
+}
+
+// Called from app/index.js when server is ready to receive traffic
+function setStartupComplete(b) {
+  STARTUP_COMPLETE = b;
+}
+
+module.exports = { getLivenessPayload, getReadinessPayload, getStartupPayload, setStartupComplete };

app/utils/probes/probe-none.js

@@ -0,0 +1,19 @@
+/**
+ * Copyright 2024 IBM Corp. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// empty implementation to be used if PROBE_IMPL is not specified
+
+module.exports = {};