Implementing logging using priority queues

Endpoints fail, it's inevitable. Although we can try to resend failed messages, we need to realize at some point that there is an issue on our end and stop bombarding the server with requests to forward the messages. This is where priority queues can come in handy.

We will replace the existing logic to use a priority queue so that we detect when to stop trying to resend the message and notify the support team instead.

The biggest change is in the triggerFailureProtocol() method where we check whether the message has failed more times than the preset retryThreshold; if it has, then we add the message to the queue with critical priority, which we later use to prevent subsequent bombardment of the server until the support team resolves the issue. This solution although rather naive is very efficient when it comes to preserving server resources.

So, the updated code with the priority queue is as follows:

function triggerFailureProtocol() {

    console.log('trigger failure protocol');

    // get front message from queue
    var frontMsgNode = failedMessageQueue.front();

    // low priority and hasnt hit retry threshold
    if (frontMsgNode.priority === 0 
        && failureTriggerCount <= failureTriggerCountThreshold) {

        // try to send message
        msgUtils.sendMessage(frontMsgNode.message)
            .then(function() {

                console.log('resend success');
                // success, so remove from queue
                failedMessageQueue.remove();

                // inform user
                res.send('OK!');

             }, function() {

                console.log('resend failure');
            
                // increment counter
                failureTriggerCount++;

                //retry failure protocol
                triggerFailureProtocol();

             });

    } else {

        console.log('resend failed too many times');
      
        // replace top message with higher priority message
        let prevMsgNode = failedMessageQueue.remove();

        prevMsgNode.priority = 1;

        // gets added to front
        failedMessageQueue.add(prevMsgNode);

        res.status(500).send('Critical Server Error! Failed to send
        message');

    }
}

In the preceding code, we wrapped the same login in an if-else block to be able to retry sending the message or create a critical error and stop our retry efforts.

So, the next time a new message for that channel comes in, you can verify that there already exists a critical error and reject the request directly rather than going through the hassle of trying to send the messages and failing, which keeps bloating the failure queue.

This is certainly one approach to solving this problem, but a more suitable approach, which is outside the scope of this example, is to notify the user of any critical errors when the user tries to access the channel rather than doing it when the users posts a message to it.

The following is the complete code including the priority queue:

var express = require('express');
var router = express.Router();
var Utils = require('../utils/messaging-utils');
const msgUtils = new Utils();

router.route('/')
    .post(function(req, res) {
        const message = req.body.message;
        let failedMessageQueue;
        let failureTriggerCount = 0;
        let failureTriggerCountThreshold = 3;
        let newMsgNode = {
            message: message,
            priority: 0
        };


        // try to send the message
        msgUtils.sendMessage(req.body)
            .then(function() {
            
                console.log('send success');

                // success
                res.send(`Message received from: ${req.body.from} to ${req.body.to} with message ${req.body.message}`);

         }, function() {

                console.log('send failed');

                // get unique queue
                failedMessageQueue = 
                    msgUtils.getUniqueFailureQueue(req.body.from,
                    req.body.to);

               // get front message in queue
               var frontMsgNode = failedMessageQueue.front();

               // already has a critical failure
               if (frontMsgNode && frontMsgNode.priority === 1) {

                   // notify support

                   // notify user
                   res.status(500)
                      .send('Critical Server Error! Failed to send
                      message');

               } else {

                   // add more
                   failedMessageQueue.add(newMsgNode);

                   // increment count
                   failureTriggerCount++;

                   // trigger failure protocol
                   triggerFailureProtocol();

               }
        });


        function triggerFailureProtocol() {

            console.log('trigger failure protocol');

            // get front message from queue
            var frontMsgNode = failedMessageQueue.front();

            // low priority and hasnt hit retry threshold
           if (frontMsgNode.priority === 0 
               && failureTriggerCount <= failureTriggerCountThreshold) {

               // try to send message
               msgUtils.sendMessage(frontMsgNode.message)
                   .then(function() {

                       console.log('resend success');
                       // success, so remove from queue
                       failedMessageQueue.remove();

                       // inform user
                       res.send('OK!');

                    }, function() {

                       console.log('resend failure');
                  
                       // increment counter
                       failureTriggerCount++;

                       //retry failure protocol
                       triggerFailureProtocol();

                     });

            } else {

                console.log('resend failed too many times');
            
                // replace top message with higher priority message
                let prevMsgNode = failedMessageQueue.remove();

                prevMsgNode.priority = 1;

                // gets added to front
                failedMessageQueue.add(prevMsgNode);

                res.status(500)
                   .send('Critical Server Error! Failed to send 
                   message');

           }
        }
});

module.exports = router;