luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Get both jar and bundle artifacts from spark-submit command 

For the JAR list passed to spark.yarn.jars. The bundle ones are missing since 1.0.0-M15-2 else.
This commit is contained in:
Alexandre Archambault 2017-04-04 16:37:51 +02:00
parent 375eeca48b
commit a62fb9a85f
2 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cli/src/main/scala-2.11/coursier/cli/SparkSubmit.scala
View File

@ -100,7 +100,8 @@ final case class SparkSubmit(
options.defaultAssemblyDependencies.getOrElse(options.autoAssembly),
options.assemblyDependencies.flatMap(_.split(",")).filter(_.nonEmpty) ++
options.sparkAssemblyDependencies.flatMap(_.split(",")).filter(_.nonEmpty).map(_ + s":$sparkVersion"),
options.common
options.common,
options.artifactOptions.artifactTypes(sources = false, javadoc = false)
)
val extraConf =
@ -120,7 +121,8 @@ final case class SparkSubmit(
options.yarnVersion,
options.defaultAssemblyDependencies.getOrElse(true),
options.assemblyDependencies.flatMap(_.split(",")).filter(_.nonEmpty),
options.common
options.common,
options.artifactOptions.artifactTypes(sources = false, javadoc = false)
)
val (assembly, assemblyJars) = assemblyAndJarsOrError match {

4
cli/src/main/scala-2.11/coursier/cli/spark/Assembly.scala
View File

@ -191,7 +191,7 @@ object Assembly {
default: Boolean,
extraDependencies: Seq[String],
options: CommonOptions,
artifactTypes: Set[String] = Set("jar")
artifactTypes: Set[String]
): Seq[File] = {
val helper = sparkJarsHelper(scalaVersion, sparkVersion, yarnVersion, default, extraDependencies, options)
@ -206,7 +206,7 @@ object Assembly {
default: Boolean,
extraDependencies: Seq[String],
options: CommonOptions,
artifactTypes: Set[String] = Set("jar"),
artifactTypes: Set[String],
checksumSeed: Array[Byte] = "v1".getBytes("UTF-8")
): Either[String, (File, Seq[File])] = {