Saturday, August 27, 2011

Apache log4j queue appender

Hi friends,

Starting another techie post. This is regarding apache log4j appenders. I was using log4j for quite some time. It is excellent one. It has got almost everything required for normal to advanced logging, except if we use it extensively then the application tends to slow down a bit, like if we use the mail and DB appender then the application slows down. Thats not a fault of the system. It will slow down. To solve this we have JMS appender. put every thing in the JMS queue and then read from there and again resend to respective appender. In this we have a small catch. That is, the utility have the JMS appender but it does not have a reader. and to avail this we need to be on J2EE container.

So to solve this I just made one small class which can take care of this situation. This is having a internal JDK5 queue and reader as well. So application does not slows down and all kind of logging we can do.


The Java class:

import java.util.Enumeration;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;

import org.apache.log4j.Appender;
import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.LoggingEvent;

/**
 * @author: Aniruddha Dutta Chowdhury
 * @since: 11-Aug-2011 : 4:35:01 PM
 */

public class QuedAppender extends AppenderSkeleton {

    private Appender[]            targetAppender        = null;
    private String                adapter                = null;
    private Queue    loggingEventQueue    = null;
    private boolean                isActive            = false;
    private Executor            executor            = null;

    public QuedAppender() {
        loggingEventQueue = new ConcurrentLinkedQueue();
        System.out.println("QuedAppender.QuedAppender()");
        startQueueCheck();
    }

    private void startQueueCheck() {
        Thread l_th = new Thread(new Runnable() {
            @Override
            public void run() {
                callTargetAppender();
            }
        });
        // l_th.setDaemon(true);
        l_th.setName("QuedAppenderChecker");
        l_th.start();
    }

    public void setQueueSize(String a_queueSize) {
        try {
            executor = Executors.newFixedThreadPool(Integer.parseInt(a_queueSize), new ThreadFactory() {

                @Override
                public Thread newThread(Runnable r) {
                    Thread l_th = new Thread(r);
                    l_th.setDaemon(true);
                    return l_th;
                }
            });
        } catch (NumberFormatException a_numberFormatExceptionIgnore) {
        }
    }

    public String getTargetAppender() {
        return adapter;
    }

    public void setTargetAppender(String a_adapter) {
        // System.out.println("QuedAppender.setTargetAppender(" + a_adapter +")");
        if (targetAppender != null) {
            return;
        }

        adapter = a_adapter;
        String[] l_appenders = adapter.split(",");
        targetAppender = new Appender[l_appenders.length];
        int iCounter = 0;
        for (String l_ap : l_appenders) {
            targetAppender[iCounter] = getAppenderById(l_ap.trim());

            if (targetAppender[iCounter] == null) {
                targetAppender = null;
                return;
            }
            iCounter++;
        }
        // targetAppender = getAppenderById(a_adapter);
    }

    public static Appender getAppenderById(String a_strAppenderName) {
        Logger logger = LogManager.getLogger("app.logger.DummyLogger");
        Appender l_targetAppender = logger.getAppender(a_strAppenderName);
        return l_targetAppender;
    }

    private static void printAllAppenders() {
        Enumeration allLoggers = LogManager.getCurrentLoggers();
        while (allLoggers.hasMoreElements()) {
            Logger l = (Logger) allLoggers.nextElement();
            Enumeration appernders = l.getAllAppenders();
            while (appernders.hasMoreElements()) {
                Appender l_ap = (Appender) appernders.nextElement();
                System.out.println("QuedAppender.getAppenderById()" + l_ap.getName());
            }
        }
    }

    @Override
    protected void append(LoggingEvent loggingEvent) {
        if (executor == null) {
            setQueueSize("5");
        }

        if (targetAppender == null) {
            setTargetAppender(adapter);
        }

        if (targetAppender == null || targetAppender.length <= 0) {
            throw new RuntimeException("Target appender is properly not initialised. Please do that first before you start this");
        }

        System.out.println("QuedAppender.append(" + loggingEvent.getLoggerName() + ")");
        loggingEventQueue.offer(loggingEvent);

        try {
            Thread.sleep(100);
        } catch (Throwable a_th) {
        }

        startQueueCheck();
    }

    /**
     * nothing much to do, so just keep it like this
     */
    @Override
    public void close() {
    }

    @Override
    public boolean requiresLayout() {
        return false;
    }

    private void callTargetAppender() {
        LoggingEvent loggingEvent = null;

        /**
         * check is already running then just ignore it
         */
        if (isActive) {
            return;
        }

        /**
         * now mark as this method in action
         */
        isActive = true;

        while ((loggingEvent = loggingEventQueue.poll()) != null) {
            executor.execute(new TargetLogger(loggingEvent, targetAppender));
            try {
                Thread.sleep(100);
            } catch (Throwable a_th) {
            }
        }

        /**
         * lets set it false for others to access
         */
        isActive = false;
    }

    private class TargetLogger implements Runnable {
        private LoggingEvent    loggingEvent    = null;
        private Appender[]        targetAppenders    = null;

        public TargetLogger(LoggingEvent loggingEvent, Appender[] a_targetAppenders) {
            this.loggingEvent = loggingEvent;
            targetAppenders = a_targetAppenders;
        }

        @Override
        public void run() {
            for (Appender targetAppender : targetAppenders) {
                if (((AppenderSkeleton) targetAppender).getThreshold() == null || ((AppenderSkeleton) targetAppender).getThreshold().toInt() <= loggingEvent.getLevel().toInt()) {
                    targetAppender.doAppend(loggingEvent);
                }
            }
        }
    }
}

The Sample Log4j.properties:

log4j.rootLogger=DEBUG,quedAppender

#a dummy logger to initialized, so that all appenders can be loaded
log4j.logger.app.logger.DummyLogger=DEBUG,sendMail,CA,FA

log4j.appender.sendMail=org.apache.log4j.net.SMTPAppender
log4j.appender.sendMail.Threshold=ERROR
log4j.appender.sendMail.To=to@target.com
log4j.appender.sendMail.From=from@source.com
log4j.appender.sendMail.SMTPHost=smtp.source.com
log4j.appender.sendMail.Subject=iSource @ on []
log4j.appender.sendMail.layout=org.apache.log4j.PatternLayout
log4j.appender.sendMail.layout.ConversionPattern=[%d{ISO8601}]%n%n%-5p%n%n%c%n%n%m%n%n
log4j.appender.sendMail.BufferSize=1
#log4j.appender.sendMail.SMTPDebug=true

log4j.appender.quedAppender=QuedAppender
log4j.appender.quedAppender.Threshold=DEBUG
log4j.appender.quedAppender.targetAppender=sendMail, CA, FA

log4j.appender.RquedAppender=QuedAppender
log4j.appender.RquedAppender.Threshold=DEBUG
log4j.appender.RquedAppender.targetAppender=FA

#Console Appender
log4j.appender.CA=org.apache.log4j.ConsoleAppender
log4j.appender.CA.layout=org.apache.log4j.PatternLayout
log4j.appender.CA.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CA.Threshold=DEBUG

#File Appender
log4j.appender.FA=org.apache.log4j.FileAppender
log4j.appender.FA.File=sample.log
log4j.appender.FA.layout=org.apache.log4j.PatternLayout
log4j.appender.FA.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.FA.Threshold=INFO

In this sample log4j.properties the dummy logger is required for finding all required appender. I could not come up with any other mechanism.

If any one has any suggestion or modification proposal, please put forward. I would be glad to know that.