Log4j2: Dynamic creation of log files for multiple logs

Question

I am currently creating a system that can have modules (think of them as plugins), where each one of them can have their own log, dedicated.

I would like to use the log4j2 project for logging, but I seem to have some trouble with the file appenders.

The main project (the module loader and "core" of the whole thing) should have its own log file, while the modules should have their own (like mod_XXXXXXXX.log ).

By reading the documentation about the appenders I discovered the FileAppender class, and I was going to use that. Until I found out that I can't just simple add the appender to the default logger created by LogManager.getLog() .

The logger returned by the LogManager is a different logger than the Logger interface.

Even searching did not give me any near solution, all I found was predefined file logs in the xml configuration - which is not what I want.

Thank you for reading; even the slightest clue is welcome :)

Answer 1

if you really need to determine the log file dynamically, take a look at the Log4J2 RoutingAppender . A longer example is in theFAQ and these stackoverflow questions may be of interest: Wildcard pattern for RoutingAppender of Log4j2 and How to write different logs in different files with log4j2 (MDC in xml)?

Note that you need to set values in the ThreadContext map that the RoutingAppender uses to decide which appender to route the log event to. This means that you would need to put some value in the ThreadContext map every time your code enters a different plugin.

However, do you really need it to be this dynamic? If you know in advance what plugins you have, you can just declare a logger for each plugin (using the package name of the plugin is a common way to do this), and map each such logger to a separate appender.

<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="warn" name="MyApp" packages="">
  <Appenders>
    <File name="MyFile" fileName="logs/app.log">
      <PatternLayout>
        <Pattern>%d %p %c{1.} [%t] %m%n</Pattern>
      </PatternLayout>
    </File>
    <File name="plugin1" fileName="logs/plugin1.log">
      <PatternLayout>
        <Pattern>%d %p %c{1.} [%t] %m%n</Pattern>
      </PatternLayout>
    </File>
    <File name="plugin2" fileName="logs/plugin2.log">
      <PatternLayout>
        <Pattern>%d %p %c{1.} [%t] %m%n</Pattern>
      </PatternLayout>
    </File>
  </Appenders>
  <Loggers>
    <Logger name="com.mycomp.project.plugin1" level="debug">
      <AppenderRef ref="plugin1" level="debug" />
    </Logger>
    <Logger name="com.mycomp.project.plugin2" level="debug">
      <AppenderRef ref="plugin2" level="debug" />
    </Logger>
    <Root level="trace">
      <AppenderRef ref="MyFile" level="trace" />
    </Root>
  </Loggers>
</Configuration>

Answer 2

I'm assuming you want your module management code define the logger configuration, right? If so, you may want to take a look at this portion of the manual which talks about extending LoggerConfig which based on what your asking is what I think you are looking for.

http://logging.apache.org/log4j/2.x/manual/extending.html

For what it's worth, I've been involved in large plug-in based systems before (using OSGi) and we honestly haven't taken this route. It's usually easier to just grep the class or package you are interested in from a single log file.

Answer 3

Even though Remko Popma's answer might be the most efficient way to do the logging, I built a small class that can create log files on it's own.

I think I will use the accepted answer's solution, so here is the code I wrote to work around the XML file stuff:

import gnu.trove.map.hash.THashMap;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.core.Logger;
import org.apache.logging.log4j.core.LoggerContext;
import org.apache.logging.log4j.core.appender.FileAppender;
import org.apache.logging.log4j.core.async.AsyncLoggerContext;
import org.apache.logging.log4j.core.layout.PatternLayout;
import org.apache.logging.log4j.message.FormattedMessageFactory;
import org.apache.logging.log4j.message.MessageFactory;

import java.io.File;
import java.io.IOException;
import java.util.Map;

/**
 * Represents a manager for custom log files stored inside a log folder.
 */
public class LoggingManager {
    /** The default log file extension */
    public static final String FILE_EXTENSION = "log";

    /** The global context used for all loggers */
    private final LoggerContext context;

    /** The global message factory used for all loggers */
    private final MessageFactory msgFactory;

    /** A map of all created logs */
    private final Map<String, Logger> logCache;

    /** The folder containing the log files */
    private final File logFolder;


    public LoggingManager(String name, File logFolder) throws IOException {
        this.logFolder = logFolder;

        if(!logFolder.exists()) {
            if(!logFolder.mkdirs()) {
                throw new IOException("Could not create log folder");
            }
        }

        this.logCache = new THashMap<String, Logger>();

        // Create logger context
        this.context = new AsyncLoggerContext(name);

        // Create formatted message factory
        this.msgFactory = new FormattedMessageFactory();
    }

    public Logger getLogger(String name) {
        Logger logger = logCache.get(name);

        // Create a new one
        if(logger == null) {
            logger = new SimpleLogger(name);

            FileAppender appender = FileAppender.createAppender(
                    new File(logFolder, name + "." + FILE_EXTENSION).getAbsolutePath(),
                    "true",
                    "false",
                    "file_appender-" + name,
                    "true",
                    "false",
                    "true",
                    PatternLayout.createLayout(PatternLayout.SIMPLE_CONVERSION_PATTERN, null, null, "UTF-8", "true"),
                    null,
                    "false",
                    null,
                    null
            );

            appender.start();
            logger.getContext().getConfiguration().getLoggerConfig("root").addAppender(appender, Level.ALL, null);

            // Add to log cache
            logCache.put(name, logger);
        }

        // Return the logger
        return logger;
    }

    private class SimpleLogger extends Logger {

        public SimpleLogger(String name) {
            super(context, name, msgFactory);

            // Set to all levels
            this.setLevel(Level.ALL);
        }

    }

}

If you don't use trove you can replace it with a normal java HashMap if you want.

Answer 4

***** Creating Dynamic Multiple Log Files using RoutingAppender and
                RollingFileAppender based on MDC *****

1) In the below file I am creating the three files dynamically 
       a) specialspecial.log
       b) rollingtest.log
       c) Queue.Name.log

 Note: we can create as many as you want just need change the keyname
 like ThreadContext.remove("keyname"); //removing previous key
      ThreadContext.put("keyname","fileNameWithPath") //adding new file
       log.info("Message"); //now onwards log the messages into above file

2)  I have added the IdlePurgePolicy( 1min) - if the log events are not coming for 1 min then 
    it released all resources attached with the appender and deletes the dormant appender

 and after deleting the appender if again log event is coming then it 
 will create the new appender and log the messages into our desired file 
 I have created my own and tested working fine in debug mode
 ---------------------------------------------------------------------------------------
log4j2.properties

status=debug
name=PropertiesConfig

filter.threshold.type=ThresholdFilter
filter.threshold.level=debug

appenders=routing
appender.routing.type=Routing
appender.routing.name=Routing

appender.routing.purge.type=IdlePurgePolicy
appender.routing.purge.timeToLive=1
appender.routing.purge.timeUnit=minutes

appender.routing.routes.type=Routes
appender.routing.routes.pattern=$${ctx:keyname}

appender.routing.routes.route.type=Route
appender.routing.routes.route.rolling.type=RollingFile
appender.routing.routes.route.rolling.name=RollingFile
appender.routing.routes.route.rolling.fileName=${ctx:keyname}.log
appender.routing.routes.route.rolling.filePattern=${ctx:keyname}-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz
appender.routing.routes.route.rolling.layout.type=PatternLayout
appender.routing.routes.route.rolling.layout.pattern=%m%n
appender.routing.routes.route.rolling.policies.type=Policies
#appender.routing.routes.route.rolling.policies.time.type=TimeBasedTriggeringPolicy
#appender.routing.routes.route.rolling.policies.time.interval=2
#appender.routing.routes.route.rolling.policies.time.modulate=true
appender.routing.routes.route.rolling.policies.size.type=SizeBasedTriggeringPolicy
appender.routing.routes.route.rolling.policies.size.size=1KB
appender.routing.routes.route.rolling.strategy.type=DefaultRolloverStrategy
appender.routing.routes.route.rolling.strategy.max=5
appender.routing.routes.route2.type=Route
appender.routing.routes.route2.key=P:/TestLogging/specialspecial
#appender.routing.routes.route.ref=Routes

appender.routing.routes.route2.rolling.type=RollingFile
appender.routing.routes.route2.rolling.name=RollingFile
appender.routing.routes.route2.rolling.fileName=${ctx:keyname}.log
appender.routing.routes.route2.rolling.filePattern=${ctx:keyname}-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz
appender.routing.routes.route2.rolling.layout.type=PatternLayout
appender.routing.routes.route2.rolling.layout.pattern=%d %p %C{1.} [%t] %m%n
appender.routing.routes.route2.rolling.policies.type=Policies
#appender.routing.routes.route2.rolling.policies.time.type=TimeBasedTriggeringPolicy
#appender.routing.routes.route2.rolling.policies.time.interval=2
#appender.routing.routes.route2.rolling.policies.time.modulate=true
appender.routing.routes.route2.rolling.policies.size.type=SizeBasedTriggeringPolicy
appender.routing.routes.route2.rolling.policies.size.size=1KB
appender.routing.routes.route2.rolling.strategy.type=DefaultRolloverStrategy
appender.routing.routes.route2.rolling.strategy.max = 5
loggers=routing
logger.routing.level=debug
logger.routing.name=com.infy.demo
logger.routing.additivity=false
logger.routing.appenderRef.routing.ref=Routing
rootLogger.level=debug
------------------------------------------------------------------------------------------------------------------
TestLog4j2Logging.class

 package com.infy.demo;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.ThreadContext;
import org.apache.logging.log4j.core.config.ConfigurationSource;
import org.apache.logging.log4j.core.config.Configurator;

public class TestLog4j2Logging {

private static final Logger log = LogManager.getLogger(TestLog4j2Logging.class);

public TestLog4j2Logging() {
    logInfo("Logging has been initalized");
}

static {
    ThreadContext.put("keyname","P:/TestLogging/TestLog4j2Logging");
    Configurator.initialize(null, "./properties/log4j2.properties");
}

public  void logTestLog4j2LoggingMesg()
{
    
    String[] arr = {"Msg1111111111111","Msg222222222222222","Msg3333333333333","Msg44444444444"};
    for (String string : arr) {
        logInfo(string);
        logError(string);
        log.debug(string);
    }
}

public void logMqMsg() throws Exception
{
    
    
    String[] mqArr = {"MQ111111111111111111","MQ222222222222222222222222","MQ3333333333333333333333"};
    for (String string : mqArr) {
        log.info(string);
    }
}


public void logSpecialKeyRouteMsgAppender()
{
    String[] mqArr = {"Special111111111111111111","Special222222222222222222222222","Special3333333333333333333333"};
    for (String string : mqArr) {
        log.info(string);
    }
}

public static void main(String[] args) throws Exception
{
    TestLog4j2Logging testLog4j2Logging = new TestLog4j2Logging();
 System.out.println("=================file TestLog4j2Logging.log creating and logging the messages==============================================");
//Logging the messages for the this class level Logging
    testLog4j2Logging.logTestLog4j2LoggingMesg();
    
    
 System.out.println("=================file Queue.Name.log creating and logging the messages c==============================================");
//Logging the messages for the MQ Logging file Queue.Name   
    ThreadContext.remove("keyname");
    ThreadContext.put("keyname", "P:/TestLogging/Queue.Name");
    testLog4j2Logging.logMqMsg();
    ThreadContext.remove("keyname");
    ThreadContext.put("keyname", "P:/TestLogging/TestLog4j2Logging");
    
    
System.out.println("=================file TestLog4j2Logging logging the messages==============================================");
    //Logging the messages for special key in the log4j.properties file 
    ThreadContext.remove("keyname");
    ThreadContext.put("keyname", "P:/TestLogging/specialspecial");
    testLog4j2Logging.logSpecialKeyRouteMsgAppender();
    ThreadContext.remove("keyname");
    ThreadContext.put("keyname", "P:/TestLogging/TestLog4j2Logging");
    
    Thread.sleep(61000);
    logInfo("Logging is Complted sucessfully");
    System.out.println("Messages are getting Logged");
}

private static String getCurrentDateAndTime() {
     DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");  

    return dtf.format(LocalDateTime.now());
}

private static void logInfo(String msg) {
    log.info("{} INFO  {} - {}",getCurrentDateAndTime(),TestLog4j2Logging.class.getName(),msg);
}

private static void logError(String msg) {
    log.info("{} ERROR {} - {}",getCurrentDateAndTime(),TestLog4j2Logging.class.getName(),msg);
}

}