luke/sbt
1
0
Fork 0
mirror of https://github.com/sbt/sbt.git synced 2026-08-22 06:07:24 +02:00
Code Issues Packages Projects Releases Wiki Activity

Get both jar and bundle artifacts from spark-submit command

Browse Source 
For the JAR list passed to spark.yarn.jars. The bundle ones are missing since 1.0.0-M15-2 else.
This commit is contained in:
Alexandre Archambault
2017-04-04 16:37:51 +02:00
parent 375eeca48b
commit a62fb9a85f
2 changed files with 6 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/src/main/scala-2.11/coursier/cli/SparkSubmit.scala
+4 -2
View File
@@ -100,7 +100,8 @@ final case class SparkSubmit(
options.defaultAssemblyDependencies.getOrElse(options.autoAssembly), options.defaultAssemblyDependencies.getOrElse(options.autoAssembly),
options.assemblyDependencies.flatMap(_.split(",")).filter(_.nonEmpty) ++ options.assemblyDependencies.flatMap(_.split(",")).filter(_.nonEmpty) ++
options.sparkAssemblyDependencies.flatMap(_.split(",")).filter(_.nonEmpty).map(_ + s":$sparkVersion"), options.sparkAssemblyDependencies.flatMap(_.split(",")).filter(_.nonEmpty).map(_ + s":$sparkVersion"),
options.common options.common,
options.artifactOptions.artifactTypes(sources = false, javadoc = false)
) )
val extraConf = val extraConf =
@@ -120,7 +121,8 @@ final case class SparkSubmit(
options.yarnVersion, options.yarnVersion,
options.defaultAssemblyDependencies.getOrElse(true), options.defaultAssemblyDependencies.getOrElse(true),
options.assemblyDependencies.flatMap(_.split(",")).filter(_.nonEmpty), options.assemblyDependencies.flatMap(_.split(",")).filter(_.nonEmpty),
options.common options.common,
options.artifactOptions.artifactTypes(sources = false, javadoc = false)
) )
val (assembly, assemblyJars) = assemblyAndJarsOrError match { val (assembly, assemblyJars) = assemblyAndJarsOrError match {
cli/src/main/scala-2.11/coursier/cli/spark/Assembly.scala
+2 -2
View File
@@ -191,7 +191,7 @@ object Assembly {
default: Boolean, default: Boolean,
extraDependencies: Seq[String], extraDependencies: Seq[String],
options: CommonOptions, options: CommonOptions,
artifactTypes: Set[String] = Set("jar") artifactTypes: Set[String]
): Seq[File] = { ): Seq[File] = {
val helper = sparkJarsHelper(scalaVersion, sparkVersion, yarnVersion, default, extraDependencies, options) val helper = sparkJarsHelper(scalaVersion, sparkVersion, yarnVersion, default, extraDependencies, options)
@@ -206,7 +206,7 @@ object Assembly {
default: Boolean, default: Boolean,
extraDependencies: Seq[String], extraDependencies: Seq[String],
options: CommonOptions, options: CommonOptions,
artifactTypes: Set[String] = Set("jar"), artifactTypes: Set[String],
checksumSeed: Array[Byte] = "v1".getBytes("UTF-8") checksumSeed: Array[Byte] = "v1".getBytes("UTF-8")
): Either[String, (File, Seq[File])] = { ): Either[String, (File, Seq[File])] = {