This is 2nd part of log4j configuration for spark application. For more understanding about log4j you can follow below link.

https://www.linkedin.com/pulse/databricks-log4j-configuration-ashish-kumar/

Spark-submit

/usr/hdp/3.0.1.0-187/spark2/bin/spark-submit --master yarn --queue dev --deploy-mode client --class com.riveriq.log4jExample --driver-java-options "-Dlog4j.configuration=file:/home/riveriq/log4j/conf/log4j.xml" --conf "spark.executor.extraJavaOptions=-Dlog4j.configuration=file:/home/riveriq/log4j/conf/log4j.xml" --num-executors 10 --executor-memory 2g --executor-cores 4 home/riveriq/log4j/jar/log4jExample.jar

log4j.xml

<?xml version="1.0" encoding="UTF-8"?>

<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">

<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/" debug="true">

    <!-- Logging Properties -->

    <Properties>

        <Property name="log-pattern">%d{yyyy-MM-dd HH:mm:ss} | %-5p | [%t] %C{2} (%F:%L) | %m%n</Property>

        <Property name="log-file-path">/tmp/riveriq/logs</Property>

    </Properties>

<Appenders>

<!-- Console Appender -->

        <Console name="console" target="SYSTEM_OUT" follow="true">

            <PatternLayout pattern="${LOG_PATTERN}"/>

        </Console>

<!-- Info Log File Appender -->

<appender name="info-log" class="org.apache.log4j.DailyRollingFileAppender">

<param name="File" value="${log-file-path}/info.log" />

<layout class="org.apache.log4j.PatternLayout">

<param name="ConversionPattern" value="${log-pattern}" />

</layout>

<filter class="org.apache.log4j.varia.LevelRangeFilter">

<param name="LevelMin" value="info" />

<param name="LevelMax" value="info" />

<param name="AcceptOnMatch" value="true" />

</filter>

</appender>

<!-- Error Log File Appender -->

<appender name="error-log" class="org.apache.log4j.DailyRollingFileAppender">

<param name="MaxFileSize" value="500KB"/>

<param name="MaxBackupIndex" value="1"/>

<param name="File" value="${log-file-path}/error.log" />

<layout class="org.apache.log4j.PatternLayout">

<param name="ConversionPattern" value="${log-pattern}" />

</layout>

<filter class="org.apache.log4j.varia.LevelRangeFilter">

<param name="LevelMin" value="error" />

<param name="LevelMax" value="error" />

<param name="AcceptOnMatch" value="true" />

</filter>

</appender>

<!-- debug Log File Appender -->

<appender name="debug-log" class="org.apache.log4j.DailyRollingFileAppender">

<param name="MaxFileSize" value="500KB"/>

<param name="MaxBackupIndex" value="1"/>

<param name="File" value="${log-file-path}/debug.log" />

<layout class="org.apache.log4j.PatternLayout">

<param name="ConversionPattern" value="${log-pattern}" />

</layout>

<filter class="org.apache.log4j.varia.LevelRangeFilter">

<param name="LevelMin" value="debug" />

<param name="LevelMax" value="debug" />

<param name="AcceptOnMatch" value="true" />

</filter>

</appender>

<!-- Perf log Log File Appender -->

<appender name="perf-log" class="org.apache.log4j.DailyRollingFileAppender">

<param name="MaxFileSize" value="500KB"/>

<param name="MaxBackupIndex" value="1"/>

<param name="File" value="${log-file-path}/perf.log" />

<layout class="org.apache.log4j.PatternLayout">

<param name="ConversionPattern" value="${log-pattern}" />

</layout>

</appender>

<!-- Trace Log File Appender -->

<appender name="trace-log" class="org.apache.log4j.DailyRollingFileAppender">

<param name="MaxFileSize" value="500KB"/>

<param name="MaxBackupIndex" value="1"/>

<param name="File" value="${log-file-path}/trace.log" />

<layout class="org.apache.log4j.PatternLayout">

<param name="ConversionPattern" value="${log-pattern}" />

</layout>

</appender>

</Appenders>

<Loggers>

<logger name="com.riveriq.pkg1" additivity="false" level="trace">

<appender-ref ref="trace-log" />

<appender-ref ref="console" />

</logger>

<logger name="com.riveriq.pkg2" additivity="false" level="debug">

<appender-ref ref="debug-log" />

<appender-ref ref="info-log" />

<appender-ref ref="error-log" />

<appender-ref ref="console" />

</logger>

<logger name="com.riveriq.app" additivity="false" level="info">

<appender-ref ref="perf-log" />

<appender-ref ref="console" />

</logger>

<root>

<level value="info" />

<appender-ref ref="console" />

</root>

</Loggers>

</log4j:configuration>