luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Tweak spark-submit command options

This commit is contained in:
Alexandre Archambault 2017-04-03 15:18:10 +02:00
parent 3b717f8be0
commit e7f74dda0d
2 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cli/src/main/scala-2.11/coursier/cli/Options.scala
View File

@ -268,6 +268,7 @@ final case class SparkSubmitOptions(
@Help("Include default dependencies in Spark Yarn assembly or jars (see --auto-assembly). If --auto-assembly is false, the corresponding dependencies will still be shunted from the job classpath if this option is true. (Default: same as --auto-assembly)") @Help("Include default dependencies in Spark Yarn assembly or jars (see --auto-assembly). If --auto-assembly is false, the corresponding dependencies will still be shunted from the job classpath if this option is true. (Default: same as --auto-assembly)")
defaultAssemblyDependencies: Option[Boolean] = None, defaultAssemblyDependencies: Option[Boolean] = None,
assemblyDependencies: List[String] = Nil, assemblyDependencies: List[String] = Nil,
sparkAssemblyDependencies: List[String] = Nil,
noDefaultSubmitDependencies: Boolean = false, noDefaultSubmitDependencies: Boolean = false,
submitDependencies: List[String] = Nil, submitDependencies: List[String] = Nil,
@Help("Spark version - if empty, deduced from the job classpath. (Default: empty)") @Help("Spark version - if empty, deduced from the job classpath. (Default: empty)")

3
cli/src/main/scala-2.11/coursier/cli/SparkSubmit.scala
View File

@ -98,7 +98,8 @@ final case class SparkSubmit(
sparkVersion, sparkVersion,
options.yarnVersion, options.yarnVersion,
options.defaultAssemblyDependencies.getOrElse(options.autoAssembly), options.defaultAssemblyDependencies.getOrElse(options.autoAssembly),
options.assemblyDependencies.flatMap(_.split(",")).filter(_.nonEmpty), options.assemblyDependencies.flatMap(_.split(",")).filter(_.nonEmpty) ++
options.sparkAssemblyDependencies.flatMap(_.split(",")).filter(_.nonEmpty).map(_ + s":$sparkVersion"),
options.common options.common
) )