From 5cb7475ce825c1f64b25723303798c37ef0bda24 Mon Sep 17 00:00:00 2001 From: Adam King Date: Tue, 23 Mar 2021 14:04:53 -0400 Subject: [PATCH 1/2] Update onHealthCheck. Allow longer for liveness to respond --- app/apollo/index.js | 29 ++++++++++++++------------ app/routes/kube/kube.js | 5 ++--- kubernetes/razeedash-api/resource.yaml | 19 +++++++++-------- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/app/apollo/index.js b/app/apollo/index.js index eedd6df5b..d2f1ae998 100644 --- a/app/apollo/index.js +++ b/app/apollo/index.js @@ -92,7 +92,7 @@ const buildCommonApolloContext = async ({ models, req, res, connection, logger } context = { apiKey: apiKey, req: upgradeReq, req_id: upgradeReq ? upgradeReq.id : undefined, userToken, recoveryHintsMap, orgId, ...context }; } else if (req) { context = { req, req_id: req.id, recoveryHintsMap, ...context }; - } + } return context; }; @@ -112,6 +112,9 @@ const loadCustomPlugins = () => { return []; }; +var SIGTERM = false; +process.on('SIGTERM', () => SIGTERM = true); + const createApolloServer = () => { const customPlugins = loadCustomPlugins(); if (process.env.GRAPHQL_ENABLE_TRACING === 'true') { @@ -165,15 +168,15 @@ const createApolloServer = () => { logger.trace({ req_id, connectionParams, context }, 'subscriptions:onConnect'); const me = await models.User.getMeFromConnectionParams( connectionParams, {req_id, models, logger, ...context},); - + logger.debug({ me }, 'subscriptions:onConnect upgradeReq getMe'); if (me === undefined) { throw Error( 'Can not find the session for this subscription request.', ); } - - // add original upgrade request to the context + + // add original upgrade request to the context return { me, upgradeReq: webSocket.upgradeReq, logger, orgKey, orgId }; }, onDisconnect: (webSocket, context) => { @@ -212,14 +215,14 @@ const apollo = async (options = {}) => { server.applyMiddleware({ app, path: GRAPHQL_PATH, - onHealthCheck: () => { - return new Promise((resolve, reject) => { - if (pubSub.enabled) { - resolve(); - } else { - reject(); - } - }); + onHealthCheck: async () => { + if (SIGTERM) { + throw 'SIGTERM received. Not accepting additional requests'; + } else if (!pubSub.enabled){ + throw '!pubSub.enabled'; + } else { + return 200; + } } }); @@ -239,7 +242,7 @@ const apollo = async (options = {}) => { port = options.graphql_port; } httpServer.listen({ port }); - } + } return { db, server, httpServer, stop}; } catch (err) { logger.error(err, 'Apollo api error'); diff --git a/app/routes/kube/kube.js b/app/routes/kube/kube.js index 7b5998862..df033e0a5 100644 --- a/app/routes/kube/kube.js +++ b/app/routes/kube/kube.js @@ -24,7 +24,7 @@ const logger = bunyan.createLogger(getBunyanConfig('kube/liveness')); const timeInterval = 300000; //5 mintues // /kube/liveness -const kube = router.get('/liveness', asyncHandler(async(req, res) => { +router.get('/liveness', asyncHandler(async(req, res) => { // does a db call to make sure we didnt disconnect try { await require('../../apollo/models').models.Organization.findOne({}); @@ -40,7 +40,6 @@ const kube = router.get('/liveness', asyncHandler(async(req, res) => { logger.error('Razeedash Api is down due to Redis pubsub connection issue, please check logs.'); return res.sendStatus(503); } - if (pubSub.lastPubSubMessage !== null && Date.now()- pubSub.lastPubSubMessage.time > timeInterval) { // check if the most recent message received is within ${timeInterval/60000} minitue logger.error(`Razeedash Api is down, haven't received any published messages within ${timeInterval/60000} minitue, please check logs.`); @@ -49,4 +48,4 @@ const kube = router.get('/liveness', asyncHandler(async(req, res) => { return res.sendStatus(200); })); -module.exports = kube; +module.exports = router; diff --git a/kubernetes/razeedash-api/resource.yaml b/kubernetes/razeedash-api/resource.yaml index 2d42fd284..ecc4f87a8 100644 --- a/kubernetes/razeedash-api/resource.yaml +++ b/kubernetes/razeedash-api/resource.yaml @@ -105,21 +105,22 @@ items: path: /api/kube/liveness port: 3333 initialDelaySeconds: 5 - periodSeconds: 20 - timeoutSeconds: 2 + periodSeconds: 30 + timeoutSeconds: 10 readinessProbe: - exec: - command: ["npm", "run", "wait-mongo"] + httpGet: + path: /.well-known/apollo/server-health + port: 3333 initialDelaySeconds: 5 - periodSeconds: 15 + periodSeconds: 30 timeoutSeconds: 10 resources: requests: - cpu: 100m - memory: 80Mi - limits: cpu: 500m - memory: 256Mi + memory: 512Mi + limits: + cpu: 2000m + memory: 1Gi volumeMounts: - name: add-cluster-webhook-headers-vol mountPath: /var/run/secrets/razeeio/razeedash-api/add-cluster-webhook-headers From 92f9e2ca0e3c890f104f40e1fae9c658ca09c006 Mon Sep 17 00:00:00 2001 From: Adam King Date: Tue, 23 Mar 2021 14:36:49 -0400 Subject: [PATCH 2/2] Use existing request id where it exists --- app/apollo/utils/bunyan.js | 13 +++++++++---- app/utils/bunyan.js | 3 +++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/app/apollo/utils/bunyan.js b/app/apollo/utils/bunyan.js index 2e646e291..fcea2cda1 100644 --- a/app/apollo/utils/bunyan.js +++ b/app/apollo/utils/bunyan.js @@ -18,7 +18,7 @@ const responseCodeMapper = (status, err, meta) => { if (meta['req-headers'] && meta['req-headers']['authorization']) { meta['req-headers']['authorization'] = 'Bearer [HIDDEN]'; meta['req-headers']['x-auth-refresh-token'] = 'Bearer [HIDDEN]'; - } + } if (meta.method === 'OPTIONS' && status === 204) { // skip OPTION request 204 response return 'trace'; @@ -37,16 +37,21 @@ const responseCodeMapper = (status, err, meta) => { }; const getBunyanConfig = (route) => { - return { + const result = { name: route, parseUA: false, - excludes: ['referer', 'url', 'short-body', 'user-agent', 'req', 'res'], + excludes: ['referer', 'body', 'short-body'], levelFn: responseCodeMapper, + obfuscate: ['req.headers.razee-org-key'], + genReqId: function (req) { + return req.request_id; + }, streams: [{ level: process.env.LOG_LEVEL || 'info', stream: process.stdout - }] + }] }; + return result; }; module.exports = { getBunyanConfig }; diff --git a/app/utils/bunyan.js b/app/utils/bunyan.js index 22df2c865..617ead6b0 100644 --- a/app/utils/bunyan.js +++ b/app/utils/bunyan.js @@ -33,6 +33,9 @@ const getBunyanConfig = (route) => { excludes: ['referer', 'body', 'short-body'], levelFn: responseCodeMapper, obfuscate: ['req.headers.razee-org-key'], + genReqId: function (req) { + return req.request_id; + }, streams: [{ level: process.env.LOG_LEVEL || 'info', stream: process.stdout