0
私は新しいSparkとIntellijです。私は実行アプリケーションを実行するとき、私はまた、いくつかの理由のsrc /メイン/ Scalaの-2.11/TestMain.scalaSparkContext：SparkContextの初期化中にエラーが発生しました。 java.net.UnknownHostException：master
import org.apache.spark.{SparkContext,SparkConf}
/**
* Created by tuannv5 on 24/08/2016.
*/
object TestMain {
def main(args: Array[String]) {
val conf = new SparkConf().setMaster("local[*]").setAppName("Test Spark").set("spark.executor.memory","2g")
val sc = new SparkContext(conf)
val data = sc.parallelize(1 to 1000000).filter(_<10000)
data.foreach(println)
}
}
でTestMain.scalaを持って
name := "TestSpark"
version := "1.0"
scalaVersion := "2.11.8"
// https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.11
libraryDependencies ++= Seq("org.apache.spark" % "spark-core_2.11" % "1.6.2","org.apache.hadoop" % "hadoop-client" % "2.6.2")
：私は以下の私のbuild.sbtファイルを持っていますIntellij。私はエラーが表示されます：
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
16/08/24 10:29:28 INFO SparkContext: Running Spark version 1.6.2
16/08/24 10:29:29 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
16/08/24 10:29:29 ERROR SparkContext: Error initializing SparkContext.
java.net.UnknownHostException: master: master: unknown error
at java.net.InetAddress.getLocalHost(InetAddress.java:1505)
at org.apache.spark.util.Utils$.findLocalInetAddress(Utils.scala:789)
at org.apache.spark.util.Utils$.org$apache$spark$util$Utils$$localIpAddress$lzycompute(Utils.scala:782)
at org.apache.spark.util.Utils$.org$apache$spark$util$Utils$$localIpAddress(Utils.scala:782)
at org.apache.spark.util.Utils$$anonfun$localHostName$1.apply(Utils.scala:839)
at org.apache.spark.util.Utils$$anonfun$localHostName$1.apply(Utils.scala:839)
at scala.Option.getOrElse(Option.scala:121)
at org.apache.spark.util.Utils$.localHostName(Utils.scala:839)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:420)
at TestMain$.main(TestMain.scala:8)
at TestMain.main(TestMain.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:147)
Caused by: java.net.UnknownHostException: master: unknown error
at java.net.Inet6AddressImpl.lookupAllHostAddr(Native Method)
at java.net.InetAddress$2.lookupAllHostAddr(InetAddress.java:928)
at java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1323)
at java.net.InetAddress.getLocalHost(InetAddress.java:1500)
... 15 more
16/08/24 10:29:29 INFO SparkContext: Successfully stopped SparkContext
Exception in thread "main" java.net.UnknownHostException: master: master: unknown error
at java.net.InetAddress.getLocalHost(InetAddress.java:1505)
at org.apache.spark.util.Utils$.findLocalInetAddress(Utils.scala:789)
at org.apache.spark.util.Utils$.org$apache$spark$util$Utils$$localIpAddress$lzycompute(Utils.scala:782)
at org.apache.spark.util.Utils$.org$apache$spark$util$Utils$$localIpAddress(Utils.scala:782)
at org.apache.spark.util.Utils$$anonfun$localHostName$1.apply(Utils.scala:839)
at org.apache.spark.util.Utils$$anonfun$localHostName$1.apply(Utils.scala:839)
at scala.Option.getOrElse(Option.scala:121)
at org.apache.spark.util.Utils$.localHostName(Utils.scala:839)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:420)
at TestMain$.main(TestMain.scala:8)
at TestMain.main(TestMain.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:147)
Caused by: java.net.UnknownHostException: master: unknown error
at java.net.Inet6AddressImpl.lookupAllHostAddr(Native Method)
at java.net.InetAddress$2.lookupAllHostAddr(InetAddress.java:928)
at java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1323)
at java.net.InetAddress.getLocalHost(InetAddress.java:1500)
... 15 more
Process finished with exit code 1
このエラーに記載されている問題を説明してもらえますか？これは、依存関係が正しくインストールされなかったか、別の理由が原因ですか？
あなたは 'hostname'とチェックを実行することができます出力？ホスト名が正しく設定されていないようです。 – zsxwing
hostnameコマンドを実行すると、出力はmasterです。しかし、私はこれを修復するか分からない。私に説明することはできますか？ – TuanMang
'/ etc/hosts'には' 127.0.0.1 master'という行を1行追加するだけです – zsxwing