こんにちは皆さん、Apache Sparkのlog4jに関する質問があります。 「log4j.rootCategory = WARN」を使用すると、すべての警告ログが除外されますが、「スパーク」ログと「YARN」ログのみをフィルタリングする必要がある場合は、どのように設定するのがよいでしょうか？log4j.propertiesフィルタスパークとYARNログ
log4j.rootCategory=WARN, console
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.out
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
下記のいくつかのspark log4j構成パッケージの例です。
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Set everything to be logged to the file target/unit-tests.log
#log4j.rootCategory=DEBUG, file
log4j.rootLogger=INFO,stdout
log4j.appender.file=org.apache.log4j.FileAppender
log4j.appender.file.append=true
log4j.appender.file.file=target/unit-tests.log
log4j.appender.file.layout=org.apache.log4j.PatternLayout
log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c (%F:%M(%L)) - %m%n
log4j.threshhold=ALL
# Ignore messages below warning level from a few verbose libraries.
log4j.logger.com.sun.jersey=WARN
log4j.logger.org.eclipse.jetty=WARN
log4j.logger.org.mortbay=WARN
log4j.logger.org.spark-project.jetty=WARN
# test package itself
log4j.logger.org.apache.spark.deploy.history.yarn=DEBUG
log4j.logger.org.apache.spark.deploy.history=DEBUG
log4j.logger.org.apache.spark.deploy.history.yarn.YarnTimelineUtils=INFO
log4j.logger.org.mortbay.jetty=WARN
# Spark noise
log4j.logger.org.apache.spark.ContextCleaner=WARN
log4j.logger.org.apache.spark.storage=WARN
log4j.logger.org.apache.spark.scheduler=WARN
log4j.logger.org.apache.spark.sparkEnv=WARN
log4j.logger.org.apache.spark.SecurityManager=WARN
log4j.logger.org.apache.spark.MapOutputTrackerMasterActor=WARN
log4j.logger.org.apache.spark.HttpFileServer=WARN
log4j.logger.org.apache.spark.util.AkkaUtils=WARN
log4j.logger.org.apache.spark.network.netty=WARN
log4j.logger.org.apache.spark.MapOutputTrackerMasterEndpoint=WARN
# Akka
log4j.logger.akka.remote=WARN
log4j.logger.akka.remote.Remoting=WARN
# Hadoop source debugging
log4j.logger.org.apache.hadoop.yarn.client.api=DEBUG
log4j.logger.org.apache.hadoop.yarn.server.timeline=DEBUG
#crank back on some Hadooop noise
log4j.logger.org.apache.hadoop=INFO
log4j.logger.org.apache.hadoop.http=ERROR
log4j.logger.org.apache.hadoop.ipc=ERROR
log4j.logger.org.apache.hadoop.util.JvmPauseMonitor=ERROR
log4j.logger.org.apache.hadoop.util.Shell=ERROR
log4j.logger.org.apache.hadoop.util.LightWeightGSet=ERROR
log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
log4j.logger.org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner=WARN
log4j.logger.org.apache.hadoop.hdfs.server.blockmanagement=WARN
log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=WARN
log4j.logger.org.apache.hadoop.hdfs=WARN
log4j.logger.BlockStateChange=WARN
log4j.logger.org.apache.hadoop.yarn.factories.impl=WARN
log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.security=WARN
log4j.logger.org.apache.hadoop.metrics2=ERROR
log4j.logger.org.apache.hadoop.util.HostsFileReader=WARN
log4j.logger.org.apache.hadoop.yarn.event.AsyncDispatcher=WARN
log4j.logger.org.apache.hadoop.security.token.delegation=WARN
log4j.logger.org.apache.hadoop.yarn.util.AbstractLivelinessMonitor=WARN
log4j.logger.org.apache.hadoop.yarn.server.nodemanager.security=WARN
log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMNMInfo=WARN
log4j.logger.org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager=FATAL
log4j.logger.org.apache.hadoop.yarn.server.timeline.security.TimelineDelegationTokenSecretManagerService=INFO