luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #478 from coursier/topic/tweak-spark-submit 

Tweak spark-submit command options
This commit is contained in:
Alexandre Archambault 2017-04-03 16:12:03 +02:00 committed by GitHub
parent 3b717f8be0 e7f74dda0d
commit 7c68e3cd5e
2 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cli/src/main/scala-2.11/coursier/cli/Options.scala
View File

@ -268,6 +268,7 @@ final case class SparkSubmitOptions(
@Help("Include default dependencies in Spark Yarn assembly or jars (see --auto-assembly). If --auto-assembly is false, the corresponding dependencies will still be shunted from the job classpath if this option is true. (Default: same as --auto-assembly)")
defaultAssemblyDependencies: Option[Boolean] = None,
assemblyDependencies: List[String] = Nil,
sparkAssemblyDependencies: List[String] = Nil,
noDefaultSubmitDependencies: Boolean = false,
submitDependencies: List[String] = Nil,
@Help("Spark version - if empty, deduced from the job classpath. (Default: empty)")

3
cli/src/main/scala-2.11/coursier/cli/SparkSubmit.scala
View File

@ -98,7 +98,8 @@ final case class SparkSubmit(
sparkVersion,
options.yarnVersion,
options.defaultAssemblyDependencies.getOrElse(options.autoAssembly),
options.assemblyDependencies.flatMap(_.split(",")).filter(_.nonEmpty),
options.assemblyDependencies.flatMap(_.split(",")).filter(_.nonEmpty) ++
options.sparkAssemblyDependencies.flatMap(_.split(",")).filter(_.nonEmpty).map(_ + s":$sparkVersion"),
options.common
)