From a62fb9a85f7bcb2a1781216bac419cc3665a0512 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Tue, 4 Apr 2017 16:37:51 +0200 Subject: [PATCH] Get both jar and bundle artifacts from spark-submit command For the JAR list passed to spark.yarn.jars. The bundle ones are missing since 1.0.0-M15-2 else. --- cli/src/main/scala-2.11/coursier/cli/SparkSubmit.scala | 6 ++++-- cli/src/main/scala-2.11/coursier/cli/spark/Assembly.scala | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cli/src/main/scala-2.11/coursier/cli/SparkSubmit.scala b/cli/src/main/scala-2.11/coursier/cli/SparkSubmit.scala index 67d6c2637..99a4e8ddf 100644 --- a/cli/src/main/scala-2.11/coursier/cli/SparkSubmit.scala +++ b/cli/src/main/scala-2.11/coursier/cli/SparkSubmit.scala @@ -100,7 +100,8 @@ final case class SparkSubmit( options.defaultAssemblyDependencies.getOrElse(options.autoAssembly), options.assemblyDependencies.flatMap(_.split(",")).filter(_.nonEmpty) ++ options.sparkAssemblyDependencies.flatMap(_.split(",")).filter(_.nonEmpty).map(_ + s":$sparkVersion"), - options.common + options.common, + options.artifactOptions.artifactTypes(sources = false, javadoc = false) ) val extraConf = @@ -120,7 +121,8 @@ final case class SparkSubmit( options.yarnVersion, options.defaultAssemblyDependencies.getOrElse(true), options.assemblyDependencies.flatMap(_.split(",")).filter(_.nonEmpty), - options.common + options.common, + options.artifactOptions.artifactTypes(sources = false, javadoc = false) ) val (assembly, assemblyJars) = assemblyAndJarsOrError match { diff --git a/cli/src/main/scala-2.11/coursier/cli/spark/Assembly.scala b/cli/src/main/scala-2.11/coursier/cli/spark/Assembly.scala index 8ddb6b674..9d73bf57c 100644 --- a/cli/src/main/scala-2.11/coursier/cli/spark/Assembly.scala +++ b/cli/src/main/scala-2.11/coursier/cli/spark/Assembly.scala @@ -191,7 +191,7 @@ object Assembly { default: Boolean, extraDependencies: Seq[String], options: CommonOptions, - artifactTypes: Set[String] = Set("jar") + artifactTypes: Set[String] ): Seq[File] = { val helper = sparkJarsHelper(scalaVersion, sparkVersion, yarnVersion, default, extraDependencies, options) @@ -206,7 +206,7 @@ object Assembly { default: Boolean, extraDependencies: Seq[String], options: CommonOptions, - artifactTypes: Set[String] = Set("jar"), + artifactTypes: Set[String], checksumSeed: Array[Byte] = "v1".getBytes("UTF-8") ): Either[String, (File, Seq[File])] = {