我试图用Flink从S3下载一些Avro文件。但我得到下面的错误不知道为什么。我已经将aws-java-sdk-core-1.11.342.jar和hadoop-aws-2.7.4.jar jar文件添加到flink lib目录中。当没有s3交互时,程序运行良好,但是在尝试与s3交互时,程序会给出错误消息。有什么问题吗?
scalaVersion in ThisBuild := "2.11.11"
val flink_version = "1.3.2"
val slf4j_version = "1.7.25"
val playJson_version = "2.6.3"
val avro_version = "1.8.2"
val cassandra_version = "3.3.0"
val mongodb_version = "3.5.0"
val commons_version = "1.40"
val aws_sdk_version = "1.11.342"
val dependencies = Seq(
"org.apache.flink" %% "flink-scala" % flink_version,
"org.apache.flink" %% "flink-clients" % flink_version,
"org.apache.flink" %% "flink-avro" % flink_version,
"org.apache.hadoop" % "hadoop-aws" % "2.7.4" % Provided,
"org.apache.flink" %% "flink-hadoop-compatibility" % flink_version,
"org.apache.flink" %% "flink-connector-cassandra" % flink_version,
"org.mongodb" % "mongo-java-driver" % mongodb_version,
"com.typesafe.play" %% "play-json" % playJson_version,
"org.apache.avro" % "avro-mapred" % "1.7.1",
"com.paulgoldbaum" %% "scala-influxdb-client" % "0.4.3",
"com.amazonaws" % "aws-java-sdk-dynamodb" % aws_sdk_version,
"com.amazonaws" % "aws-java-sdk-core" % aws_sdk_version % Provided
) ++ Seq(
"org.specs2" %% "specs2-core" % "3.9.4" % Test,
"org.mockito" % "mockito-all" % "1.10.19" % Test,
"com.amazonaws" % "DynamoDBLocal" % "[1.11,2.0)" % Test,
"org.slf4j" % "slf4j-api" % slf4j_version,
"org.slf4j" % "slf4j-log4j12" % slf4j_version
)
错误:-
at org.apache.flink.runtime.jobmanager.JobManager.aroundReceive(JobManager.scala:125)
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516)
at akka.actor.ActorCell.invoke(ActorCell.scala:487)
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238)
at akka.dispatch.Mailbox.run(Mailbox.scala:220)
at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397)
at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
Caused by: org.apache.flink.runtime.JobException: Creating the input splits caused an error: com/amazonaws/services/s3/AmazonS3
at org.apache.flink.runtime.executiongraph.ExecutionJobVertex.<init>(ExecutionJobVertex.java:247)
at org.apache.flink.runtime.executiongraph.ExecutionGraph.attachJobGraph(ExecutionGraph.java:780)
at org.apache.flink.runtime.executiongraph.ExecutionGraphBuilder.buildGraph(ExecutionGraphBuilder.java:166)
at org.apache.flink.runtime.jobmanager.JobManager.org$apache$flink$runtime$jobmanager$JobManager$$submitJob(JobManager.scala:1315)
... 19 more
Caused by: java.lang.NoClassDefFoundError: com/amazonaws/services/s3/AmazonS3
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:348)
at org.apache.hadoop.conf.Configuration.getClassByNameOrNull(Configuration.java:1821)
at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:1786)
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1880)
at org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2298)
at org.apache.flink.runtime.fs.hdfs.HadoopFileSystem.getHadoopWrapperClassNameForFileSystem(HadoopFileSystem.java:486)
at org.apache.flink.core.fs.FileSystem.getHadoopWrapperClassNameForFileSystem(FileSystem.java:437)
at org.apache.flink.core.fs.FileSystem.getUnguardedFileSystem(FileSystem.java:339)
at org.apache.flink.core.fs.FileSystem.get(FileSystem.java:389)
at org.apache.flink.core.fs.Path.getFileSystem(Path.java:293)
at org.apache.flink.api.common.io.FileInputFormat.createInputSplits(FileInputFormat.java:472)
at org.apache.flink.api.common.io.FileInputFormat.createInputSplits(FileInputFormat.java:62)
at org.apache.flink.runtime.executiongraph.ExecutionJobVertex.<init>(ExecutionJobVertex.java:233)
... 22 more
Caused by: java.lang.ClassNotFoundException: com.amazonaws.services.s3.AmazonS3
at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 36 more
