mirror of https://github.com/sbt/sbt.git
Merge pull request #850 from coursier/topic/bootstrap-assembly
Allow to generate assemblies via the bootstrap command
This commit is contained in:
Alexandre Archambault
GitHub
commit
5c6d719598
|
|
@ -1,22 +1,276 @@
|
|||
package coursier
|
||||
package cli
|
||||
|
||||
import java.io.{ByteArrayInputStream, ByteArrayOutputStream, File, FileInputStream, IOException}
|
||||
import java.io.{ByteArrayInputStream, ByteArrayOutputStream, File, FileInputStream, FileOutputStream, IOException}
|
||||
import java.nio.charset.StandardCharsets.UTF_8
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.attribute.PosixFilePermission
|
||||
import java.util.Properties
|
||||
import java.util.jar.{JarFile, Attributes => JarAttributes}
|
||||
import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream}
|
||||
|
||||
import caseapp._
|
||||
import coursier.cli.options.BootstrapOptions
|
||||
import coursier.cli.util.Zip
|
||||
import coursier.cli.util.{Assembly, Zip}
|
||||
import coursier.internal.FileUtil
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
|
||||
object Bootstrap extends CaseApp[BootstrapOptions] {
|
||||
|
||||
private def createNativeBootstrap(
|
||||
options: BootstrapOptions,
|
||||
helper: Helper,
|
||||
mainClass: String
|
||||
): Unit = {
|
||||
|
||||
val files = helper.fetch(
|
||||
sources = false,
|
||||
javadoc = false,
|
||||
artifactTypes = options.artifactOptions.artifactTypes(sources = false, javadoc = false)
|
||||
)
|
||||
|
||||
val log: String => Unit =
|
||||
if (options.options.common.verbosityLevel >= 0)
|
||||
s => Console.err.println(s)
|
||||
else
|
||||
_ => ()
|
||||
|
||||
val tmpDir = new File(options.options.target)
|
||||
|
||||
try {
|
||||
coursier.extra.Native.create(
|
||||
mainClass,
|
||||
files,
|
||||
new File(options.options.output),
|
||||
tmpDir,
|
||||
log,
|
||||
verbosity = options.options.common.verbosityLevel
|
||||
)
|
||||
} finally {
|
||||
if (!options.options.keepTarget)
|
||||
coursier.extra.Native.deleteRecursive(tmpDir)
|
||||
}
|
||||
}
|
||||
|
||||
private def createJarBootstrap(javaOpts: Seq[String], output: File, content: Array[Byte]): Unit = {
|
||||
|
||||
val javaCmd = Seq("java") ++
|
||||
javaOpts
|
||||
// escaping possibly a bit loose :-|
|
||||
.map(s => "'" + s.replace("'", "\\'") + "'") ++
|
||||
Seq(
|
||||
"-jar",
|
||||
"\"$0\"",
|
||||
"\"$@\""
|
||||
)
|
||||
|
||||
val shellPreamble = Seq(
|
||||
"#!/usr/bin/env sh",
|
||||
"exec " + javaCmd.mkString(" ")
|
||||
).mkString("", "\n", "\n")
|
||||
|
||||
try Files.write(output.toPath, shellPreamble.getBytes(UTF_8) ++ content)
|
||||
catch { case e: IOException =>
|
||||
Console.err.println(s"Error while writing $output${Option(e.getMessage).fold("")(" (" + _ + ")")}")
|
||||
sys.exit(1)
|
||||
}
|
||||
|
||||
try {
|
||||
val perms = Files.getPosixFilePermissions(output.toPath).asScala.toSet
|
||||
|
||||
var newPerms = perms
|
||||
if (perms(PosixFilePermission.OWNER_READ))
|
||||
newPerms += PosixFilePermission.OWNER_EXECUTE
|
||||
if (perms(PosixFilePermission.GROUP_READ))
|
||||
newPerms += PosixFilePermission.GROUP_EXECUTE
|
||||
if (perms(PosixFilePermission.OTHERS_READ))
|
||||
newPerms += PosixFilePermission.OTHERS_EXECUTE
|
||||
|
||||
if (newPerms != perms)
|
||||
Files.setPosixFilePermissions(
|
||||
output.toPath,
|
||||
newPerms.asJava
|
||||
)
|
||||
} catch {
|
||||
case e: UnsupportedOperationException =>
|
||||
// Ignored
|
||||
case e: IOException =>
|
||||
Console.err.println(
|
||||
s"Error while making $output executable" +
|
||||
Option(e.getMessage).fold("")(" (" + _ + ")")
|
||||
)
|
||||
sys.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
private def createOneJarLikeJarBootstrap(
|
||||
options: BootstrapOptions,
|
||||
helper: Helper,
|
||||
mainClass: String,
|
||||
javaOpts: Seq[String],
|
||||
urls: Seq[String],
|
||||
files: Seq[File],
|
||||
output: File
|
||||
): Unit = {
|
||||
|
||||
val bootstrapJar =
|
||||
Option(Thread.currentThread().getContextClassLoader.getResourceAsStream("bootstrap.jar")) match {
|
||||
case Some(is) => FileUtil.readFully(is)
|
||||
case None =>
|
||||
Console.err.println(s"Error: bootstrap JAR not found")
|
||||
sys.exit(1)
|
||||
}
|
||||
|
||||
val isolatedDeps = options.options.isolated.isolatedDeps(options.options.common.scalaVersion)
|
||||
|
||||
val (_, isolatedArtifactFiles) =
|
||||
options.options.isolated.targets.foldLeft((Vector.empty[String], Map.empty[String, (Seq[String], Seq[File])])) {
|
||||
case ((done, acc), target) =>
|
||||
|
||||
// TODO Add non regression test checking that optional artifacts indeed land in the isolated loader URLs
|
||||
|
||||
val m = helper.fetchMap(
|
||||
sources = false,
|
||||
javadoc = false,
|
||||
artifactTypes = options.artifactOptions.artifactTypes(sources = false, javadoc = false),
|
||||
subset = isolatedDeps.getOrElse(target, Seq.empty).toSet
|
||||
)
|
||||
|
||||
val (done0, subUrls, subFiles) =
|
||||
if (options.options.standalone) {
|
||||
val subFiles0 = m.values.toSeq
|
||||
(done, Nil, subFiles0)
|
||||
} else {
|
||||
val filteredSubArtifacts = m.keys.toSeq.diff(done)
|
||||
(done ++ filteredSubArtifacts, filteredSubArtifacts, Nil)
|
||||
}
|
||||
|
||||
val updatedAcc = acc + (target -> (subUrls, subFiles))
|
||||
|
||||
(done0, updatedAcc)
|
||||
}
|
||||
|
||||
val isolatedUrls = isolatedArtifactFiles.map { case (k, (v, _)) => k -> v }
|
||||
val isolatedFiles = isolatedArtifactFiles.map { case (k, (_, v)) => k -> v }
|
||||
|
||||
val buffer = new ByteArrayOutputStream
|
||||
|
||||
val bootstrapZip = new ZipInputStream(new ByteArrayInputStream(bootstrapJar))
|
||||
val outputZip = new ZipOutputStream(buffer)
|
||||
|
||||
for ((ent, data) <- Zip.zipEntries(bootstrapZip)) {
|
||||
outputZip.putNextEntry(ent)
|
||||
outputZip.write(data)
|
||||
outputZip.closeEntry()
|
||||
}
|
||||
|
||||
|
||||
val time = System.currentTimeMillis()
|
||||
|
||||
def putStringEntry(name: String, content: String): Unit = {
|
||||
val entry = new ZipEntry(name)
|
||||
entry.setTime(time)
|
||||
|
||||
outputZip.putNextEntry(entry)
|
||||
outputZip.write(content.getBytes(UTF_8))
|
||||
outputZip.closeEntry()
|
||||
}
|
||||
|
||||
def putEntryFromFile(name: String, f: File): Unit = {
|
||||
val entry = new ZipEntry(name)
|
||||
entry.setTime(f.lastModified())
|
||||
|
||||
outputZip.putNextEntry(entry)
|
||||
outputZip.write(FileUtil.readFully(new FileInputStream(f)))
|
||||
outputZip.closeEntry()
|
||||
}
|
||||
|
||||
putStringEntry("bootstrap-jar-urls", urls.mkString("\n"))
|
||||
|
||||
if (options.options.isolated.anyIsolatedDep) {
|
||||
putStringEntry("bootstrap-isolation-ids", options.options.isolated.targets.mkString("\n"))
|
||||
|
||||
for (target <- options.options.isolated.targets) {
|
||||
val urls = isolatedUrls.getOrElse(target, Nil)
|
||||
val files = isolatedFiles.getOrElse(target, Nil)
|
||||
putStringEntry(s"bootstrap-isolation-$target-jar-urls", urls.mkString("\n"))
|
||||
putStringEntry(s"bootstrap-isolation-$target-jar-resources", files.map(pathFor).mkString("\n"))
|
||||
}
|
||||
}
|
||||
|
||||
def pathFor(f: File) = s"jars/${f.getName}"
|
||||
|
||||
for (f <- files)
|
||||
putEntryFromFile(pathFor(f), f)
|
||||
|
||||
putStringEntry("bootstrap-jar-resources", files.map(pathFor).mkString("\n"))
|
||||
|
||||
val propsEntry = new ZipEntry("bootstrap.properties")
|
||||
propsEntry.setTime(time)
|
||||
|
||||
val properties = new Properties
|
||||
properties.setProperty("bootstrap.mainClass", mainClass)
|
||||
|
||||
outputZip.putNextEntry(propsEntry)
|
||||
properties.store(outputZip, "")
|
||||
outputZip.closeEntry()
|
||||
|
||||
outputZip.close()
|
||||
|
||||
createJarBootstrap(
|
||||
javaOpts,
|
||||
output,
|
||||
buffer.toByteArray
|
||||
)
|
||||
}
|
||||
|
||||
private def defaultRules = Seq(
|
||||
Assembly.Rule.Append("reference.conf"),
|
||||
Assembly.Rule.AppendPattern("META-INF/services/.*"),
|
||||
Assembly.Rule.Exclude("log4j.properties"),
|
||||
Assembly.Rule.Exclude(JarFile.MANIFEST_NAME),
|
||||
Assembly.Rule.ExcludePattern("META-INF/.*\\.[sS][fF]"),
|
||||
Assembly.Rule.ExcludePattern("META-INF/.*\\.[dD][sS][aA]"),
|
||||
Assembly.Rule.ExcludePattern("META-INF/.*\\.[rR][sS][aA]")
|
||||
)
|
||||
|
||||
private def createAssemblyJar(
|
||||
options: BootstrapOptions,
|
||||
files: Seq[File],
|
||||
javaOpts: Seq[String],
|
||||
mainClass: String,
|
||||
output: File
|
||||
): Unit = {
|
||||
|
||||
val parsedRules = options.options.rule.map { s =>
|
||||
s.split(":", 2) match {
|
||||
case Array("append", v) => Assembly.Rule.Append(v)
|
||||
case Array("append-pattern", v) => Assembly.Rule.AppendPattern(v)
|
||||
case Array("exclude", v) => Assembly.Rule.Exclude(v)
|
||||
case Array("exclude-pattern", v) => Assembly.Rule.ExcludePattern(v)
|
||||
case _ =>
|
||||
sys.error(s"Malformed assembly rule: $s")
|
||||
}
|
||||
}
|
||||
|
||||
val rules =
|
||||
(if (options.options.defaultRules) defaultRules else Nil) ++ parsedRules
|
||||
|
||||
val attrs = Seq(
|
||||
JarAttributes.Name.MAIN_CLASS -> mainClass
|
||||
)
|
||||
|
||||
val baos = new ByteArrayOutputStream
|
||||
Assembly.make(files, baos, attrs, rules)
|
||||
|
||||
createJarBootstrap(
|
||||
javaOpts,
|
||||
output,
|
||||
baos.toByteArray
|
||||
)
|
||||
}
|
||||
|
||||
def run(options: BootstrapOptions, args: RemainingArgs): Unit = {
|
||||
|
||||
val helper = new Helper(
|
||||
|
|
@ -38,36 +292,9 @@ object Bootstrap extends CaseApp[BootstrapOptions] {
|
|||
else
|
||||
options.options.mainClass
|
||||
|
||||
if (options.options.native) {
|
||||
|
||||
val files = helper.fetch(
|
||||
sources = false,
|
||||
javadoc = false,
|
||||
artifactTypes = options.artifactOptions.artifactTypes(sources = false, javadoc = false)
|
||||
)
|
||||
|
||||
val log: String => Unit =
|
||||
if (options.options.common.verbosityLevel >= 0)
|
||||
s => Console.err.println(s)
|
||||
else
|
||||
_ => ()
|
||||
|
||||
val tmpDir = new File(options.options.target)
|
||||
|
||||
try {
|
||||
coursier.extra.Native.create(
|
||||
mainClass,
|
||||
files,
|
||||
output0,
|
||||
tmpDir,
|
||||
log,
|
||||
verbosity = options.options.common.verbosityLevel
|
||||
)
|
||||
} finally {
|
||||
if (!options.options.keepTarget)
|
||||
coursier.extra.Native.deleteRecursive(tmpDir)
|
||||
}
|
||||
} else {
|
||||
if (options.options.native)
|
||||
createNativeBootstrap(options, helper, mainClass)
|
||||
else {
|
||||
|
||||
val (validProperties, wrongProperties) = options.options.property.partition(_.contains("="))
|
||||
if (wrongProperties.nonEmpty) {
|
||||
|
|
@ -76,165 +303,39 @@ object Bootstrap extends CaseApp[BootstrapOptions] {
|
|||
}
|
||||
|
||||
val properties0 = validProperties.map { s =>
|
||||
val idx = s.indexOf('=')
|
||||
assert(idx >= 0)
|
||||
(s.take(idx), s.drop(idx + 1))
|
||||
s.split("=", 2) match {
|
||||
case Array(k, v) => k -> v
|
||||
case _ => sys.error("Cannot possibly happen")
|
||||
}
|
||||
}
|
||||
|
||||
val bootstrapJar =
|
||||
Option(Thread.currentThread().getContextClassLoader.getResourceAsStream("bootstrap.jar")) match {
|
||||
case Some(is) => FileUtil.readFully(is)
|
||||
case None =>
|
||||
Console.err.println(s"Error: bootstrap JAR not found")
|
||||
sys.exit(1)
|
||||
}
|
||||
|
||||
val isolatedDeps = options.options.isolated.isolatedDeps(options.options.common.scalaVersion)
|
||||
|
||||
val (_, isolatedArtifactFiles) =
|
||||
options.options.isolated.targets.foldLeft((Vector.empty[String], Map.empty[String, (Seq[String], Seq[File])])) {
|
||||
case ((done, acc), target) =>
|
||||
|
||||
// TODO Add non regression test checking that optional artifacts indeed land in the isolated loader URLs
|
||||
|
||||
val m = helper.fetchMap(
|
||||
sources = false,
|
||||
javadoc = false,
|
||||
artifactTypes = options.artifactOptions.artifactTypes(sources = false, javadoc = false),
|
||||
subset = isolatedDeps.getOrElse(target, Seq.empty).toSet
|
||||
)
|
||||
|
||||
val (done0, subUrls, subFiles) =
|
||||
if (options.options.standalone) {
|
||||
val subFiles0 = m.values.toSeq
|
||||
(done, Nil, subFiles0)
|
||||
} else {
|
||||
val filteredSubArtifacts = m.keys.toSeq.diff(done)
|
||||
(done ++ filteredSubArtifacts, filteredSubArtifacts, Nil)
|
||||
}
|
||||
|
||||
val updatedAcc = acc + (target -> (subUrls, subFiles))
|
||||
|
||||
(done0, updatedAcc)
|
||||
}
|
||||
val javaOpts = options.options.javaOpt ++
|
||||
properties0.map { case (k, v) => s"-D$k=$v" }
|
||||
|
||||
val (urls, files) =
|
||||
helper.fetchMap(
|
||||
sources = false,
|
||||
javadoc = false,
|
||||
artifactTypes = options.artifactOptions.artifactTypes(sources = false, javadoc = false)
|
||||
).toList.foldLeft((List.empty[String], List.empty[File])){
|
||||
case ((urls, files), (url, file)) =>
|
||||
if (options.options.standalone) (urls, file :: files)
|
||||
else if (url.startsWith("file:/")) (urls, file :: files)
|
||||
else (url :: urls, files)
|
||||
}
|
||||
|
||||
val isolatedUrls = isolatedArtifactFiles.map { case (k, (v, _)) => k -> v }
|
||||
val isolatedFiles = isolatedArtifactFiles.map { case (k, (_, v)) => k -> v }
|
||||
|
||||
val buffer = new ByteArrayOutputStream
|
||||
|
||||
val bootstrapZip = new ZipInputStream(new ByteArrayInputStream(bootstrapJar))
|
||||
val outputZip = new ZipOutputStream(buffer)
|
||||
|
||||
for ((ent, data) <- Zip.zipEntries(bootstrapZip)) {
|
||||
outputZip.putNextEntry(ent)
|
||||
outputZip.write(data)
|
||||
outputZip.closeEntry()
|
||||
}
|
||||
|
||||
|
||||
val time = System.currentTimeMillis()
|
||||
|
||||
def putStringEntry(name: String, content: String): Unit = {
|
||||
val entry = new ZipEntry(name)
|
||||
entry.setTime(time)
|
||||
|
||||
outputZip.putNextEntry(entry)
|
||||
outputZip.write(content.getBytes(UTF_8))
|
||||
outputZip.closeEntry()
|
||||
}
|
||||
|
||||
def putEntryFromFile(name: String, f: File): Unit = {
|
||||
val entry = new ZipEntry(name)
|
||||
entry.setTime(f.lastModified())
|
||||
|
||||
outputZip.putNextEntry(entry)
|
||||
outputZip.write(FileUtil.readFully(new FileInputStream(f)))
|
||||
outputZip.closeEntry()
|
||||
}
|
||||
|
||||
putStringEntry("bootstrap-jar-urls", urls.mkString("\n"))
|
||||
|
||||
if (options.options.isolated.anyIsolatedDep) {
|
||||
putStringEntry("bootstrap-isolation-ids", options.options.isolated.targets.mkString("\n"))
|
||||
|
||||
for (target <- options.options.isolated.targets) {
|
||||
val urls = isolatedUrls.getOrElse(target, Nil)
|
||||
val files = isolatedFiles.getOrElse(target, Nil)
|
||||
putStringEntry(s"bootstrap-isolation-$target-jar-urls", urls.mkString("\n"))
|
||||
putStringEntry(s"bootstrap-isolation-$target-jar-resources", files.map(pathFor).mkString("\n"))
|
||||
sources = false,
|
||||
javadoc = false,
|
||||
artifactTypes = options.artifactOptions.artifactTypes(sources = false, javadoc = false)
|
||||
).toList.foldLeft((List.empty[String], List.empty[File])){
|
||||
case ((urls, files), (url, file)) =>
|
||||
if (options.options.assembly || options.options.standalone) (urls, file :: files)
|
||||
else if (url.startsWith("file:/")) (urls, file :: files)
|
||||
else (url :: urls, files)
|
||||
}
|
||||
}
|
||||
|
||||
def pathFor(f: File) = s"jars/${f.getName}"
|
||||
|
||||
for (f <- files)
|
||||
putEntryFromFile(pathFor(f), f)
|
||||
|
||||
putStringEntry("bootstrap-jar-resources", files.map(pathFor).mkString("\n"))
|
||||
|
||||
val propsEntry = new ZipEntry("bootstrap.properties")
|
||||
propsEntry.setTime(time)
|
||||
|
||||
val properties = new Properties
|
||||
properties.setProperty("bootstrap.mainClass", mainClass)
|
||||
|
||||
outputZip.putNextEntry(propsEntry)
|
||||
properties.store(outputZip, "")
|
||||
outputZip.closeEntry()
|
||||
|
||||
outputZip.close()
|
||||
|
||||
// escaping of javaOpt possibly a bit loose :-|
|
||||
val shellPreamble = Seq(
|
||||
"#!/usr/bin/env sh",
|
||||
"exec java -jar " + options.options.javaOpt.map(s => "'" + s.replace("'", "\\'") + "'").mkString(" ") + " \"$0\" \"$@\""
|
||||
).mkString("", "\n", "\n")
|
||||
|
||||
try Files.write(output0.toPath, shellPreamble.getBytes(UTF_8) ++ buffer.toByteArray)
|
||||
catch { case e: IOException =>
|
||||
Console.err.println(s"Error while writing $output0${Option(e.getMessage).fold("")(" (" + _ + ")")}")
|
||||
sys.exit(1)
|
||||
}
|
||||
|
||||
try {
|
||||
val perms = Files.getPosixFilePermissions(output0.toPath).asScala.toSet
|
||||
|
||||
var newPerms = perms
|
||||
if (perms(PosixFilePermission.OWNER_READ))
|
||||
newPerms += PosixFilePermission.OWNER_EXECUTE
|
||||
if (perms(PosixFilePermission.GROUP_READ))
|
||||
newPerms += PosixFilePermission.GROUP_EXECUTE
|
||||
if (perms(PosixFilePermission.OTHERS_READ))
|
||||
newPerms += PosixFilePermission.OTHERS_EXECUTE
|
||||
|
||||
if (newPerms != perms)
|
||||
Files.setPosixFilePermissions(
|
||||
output0.toPath,
|
||||
newPerms.asJava
|
||||
)
|
||||
} catch {
|
||||
case e: UnsupportedOperationException =>
|
||||
// Ignored
|
||||
case e: IOException =>
|
||||
Console.err.println(
|
||||
s"Error while making $output0 executable" +
|
||||
Option(e.getMessage).fold("")(" (" + _ + ")")
|
||||
)
|
||||
sys.exit(1)
|
||||
}
|
||||
if (options.options.assembly)
|
||||
createAssemblyJar(options, files, javaOpts, mainClass, output0)
|
||||
else
|
||||
createOneJarLikeJarBootstrap(
|
||||
options,
|
||||
helper,
|
||||
mainClass,
|
||||
javaOpts,
|
||||
urls,
|
||||
files,
|
||||
output0
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import java.net.URLClassLoader
|
|||
import caseapp._
|
||||
import coursier.Dependency
|
||||
import coursier.cli.options.SparkSubmitOptions
|
||||
import coursier.cli.spark.{Assembly, Submit}
|
||||
import coursier.cli.spark.{SparkAssembly, Submit}
|
||||
|
||||
|
||||
/**
|
||||
|
|
@ -85,7 +85,7 @@ object SparkSubmit extends CaseApp[SparkSubmitOptions] {
|
|||
val (sparkYarnExtraConf, sparkBaseJars) =
|
||||
if (!options.autoAssembly || sparkVersion.startsWith("2.")) {
|
||||
|
||||
val assemblyJars = Assembly.sparkJars(
|
||||
val assemblyJars = SparkAssembly.sparkJars(
|
||||
scalaVersion,
|
||||
sparkVersion,
|
||||
options.yarnVersion,
|
||||
|
|
@ -107,7 +107,7 @@ object SparkSubmit extends CaseApp[SparkSubmitOptions] {
|
|||
(extraConf, assemblyJars)
|
||||
} else {
|
||||
|
||||
val assemblyAndJarsOrError = Assembly.spark(
|
||||
val assemblyAndJarsOrError = SparkAssembly.spark(
|
||||
scalaVersion,
|
||||
sparkVersion,
|
||||
options.yarnVersion,
|
||||
|
|
|
|||
|
|
@ -29,6 +29,15 @@ final case class BootstrapSpecificOptions(
|
|||
target: String = "native-target",
|
||||
@Help("Don't wipe native compilation target directory (for debug purposes)")
|
||||
keepTarget: Boolean = false,
|
||||
@Help("Generate an assembly rather than a bootstrap jar")
|
||||
@Short("a")
|
||||
assembly: Boolean = false,
|
||||
@Help("Add assembly rule")
|
||||
@Value("append:$path|append-pattern:$pattern|exclude:$path|exclude-pattern:$pattern")
|
||||
@Short("R")
|
||||
rule: List[String] = Nil,
|
||||
@Help("Add default rules to assembly rule list")
|
||||
defaultRules: Boolean = true,
|
||||
@Recurse
|
||||
isolated: IsolatedLoaderOptions = IsolatedLoaderOptions(),
|
||||
@Recurse
|
||||
|
|
|
|||
|
|
@ -1,144 +1,28 @@
|
|||
package coursier.cli.spark
|
||||
|
||||
import java.io.{File, FileInputStream, FileOutputStream}
|
||||
import java.io.{File, FileOutputStream}
|
||||
import java.math.BigInteger
|
||||
import java.nio.charset.StandardCharsets.UTF_8
|
||||
import java.nio.file.{Files, StandardCopyOption}
|
||||
import java.security.MessageDigest
|
||||
import java.util.jar.{Attributes, JarFile, JarOutputStream, Manifest}
|
||||
import java.util.regex.Pattern
|
||||
import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream}
|
||||
import java.util.jar.JarFile
|
||||
|
||||
import coursier.Cache
|
||||
import coursier.cli.Helper
|
||||
import coursier.cli.options.CommonOptions
|
||||
import coursier.cli.util.Zip
|
||||
import coursier.cli.util.Assembly
|
||||
|
||||
import scala.collection.mutable
|
||||
object SparkAssembly {
|
||||
|
||||
object Assembly {
|
||||
|
||||
sealed abstract class Rule extends Product with Serializable
|
||||
|
||||
object Rule {
|
||||
sealed abstract class PathRule extends Rule {
|
||||
def path: String
|
||||
}
|
||||
|
||||
final case class Exclude(path: String) extends PathRule
|
||||
final case class ExcludePattern(path: Pattern) extends Rule
|
||||
|
||||
object ExcludePattern {
|
||||
def apply(s: String): ExcludePattern =
|
||||
ExcludePattern(Pattern.compile(s))
|
||||
}
|
||||
|
||||
// TODO Accept a separator: Array[Byte] argument in these
|
||||
// (to separate content with a line return in particular)
|
||||
final case class Append(path: String) extends PathRule
|
||||
final case class AppendPattern(path: Pattern) extends Rule
|
||||
|
||||
object AppendPattern {
|
||||
def apply(s: String): AppendPattern =
|
||||
AppendPattern(Pattern.compile(s))
|
||||
}
|
||||
}
|
||||
|
||||
def make(jars: Seq[File], output: File, rules: Seq[Rule]): Unit = {
|
||||
|
||||
val rulesMap = rules.collect { case r: Rule.PathRule => r.path -> r }.toMap
|
||||
val excludePatterns = rules.collect { case Rule.ExcludePattern(p) => p }
|
||||
val appendPatterns = rules.collect { case Rule.AppendPattern(p) => p }
|
||||
|
||||
val manifest = new Manifest
|
||||
manifest.getMainAttributes.put(Attributes.Name.MANIFEST_VERSION, "1.0")
|
||||
|
||||
var fos: FileOutputStream = null
|
||||
var zos: ZipOutputStream = null
|
||||
|
||||
try {
|
||||
fos = new FileOutputStream(output)
|
||||
zos = new JarOutputStream(fos, manifest)
|
||||
|
||||
val concatenedEntries = new mutable.HashMap[String, ::[(ZipEntry, Array[Byte])]]
|
||||
|
||||
var ignore = Set.empty[String]
|
||||
|
||||
for (jar <- jars) {
|
||||
var fis: FileInputStream = null
|
||||
var zis: ZipInputStream = null
|
||||
|
||||
try {
|
||||
fis = new FileInputStream(jar)
|
||||
zis = new ZipInputStream(fis)
|
||||
|
||||
for ((ent, content) <- Zip.zipEntries(zis)) {
|
||||
|
||||
def append() =
|
||||
concatenedEntries += ent.getName -> ::((ent, content), concatenedEntries.getOrElse(ent.getName, Nil))
|
||||
|
||||
rulesMap.get(ent.getName) match {
|
||||
case Some(Rule.Exclude(_)) =>
|
||||
// ignored
|
||||
|
||||
case Some(Rule.Append(_)) =>
|
||||
append()
|
||||
|
||||
case None =>
|
||||
if (!excludePatterns.exists(_.matcher(ent.getName).matches())) {
|
||||
if (appendPatterns.exists(_.matcher(ent.getName).matches()))
|
||||
append()
|
||||
else if (!ignore(ent.getName)) {
|
||||
ent.setCompressedSize(-1L)
|
||||
zos.putNextEntry(ent)
|
||||
zos.write(content)
|
||||
zos.closeEntry()
|
||||
|
||||
ignore += ent.getName
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} finally {
|
||||
if (zis != null)
|
||||
zis.close()
|
||||
if (fis != null)
|
||||
fis.close()
|
||||
}
|
||||
}
|
||||
|
||||
for ((_, entries) <- concatenedEntries) {
|
||||
val (ent, _) = entries.head
|
||||
|
||||
ent.setCompressedSize(-1L)
|
||||
|
||||
if (entries.tail.nonEmpty)
|
||||
ent.setSize(entries.map(_._2.length).sum)
|
||||
|
||||
zos.putNextEntry(ent)
|
||||
// for ((_, b) <- entries.reverse)
|
||||
// zos.write(b)
|
||||
zos.write(entries.reverse.toArray.flatMap(_._2))
|
||||
zos.closeEntry()
|
||||
}
|
||||
} finally {
|
||||
if (zos != null)
|
||||
zos.close()
|
||||
if (fos != null)
|
||||
fos.close()
|
||||
}
|
||||
}
|
||||
|
||||
val assemblyRules = Seq[Rule](
|
||||
Rule.Append("META-INF/services/org.apache.hadoop.fs.FileSystem"),
|
||||
Rule.Append("reference.conf"),
|
||||
Rule.AppendPattern("META-INF/services/.*"),
|
||||
Rule.Exclude("log4j.properties"),
|
||||
Rule.Exclude(JarFile.MANIFEST_NAME),
|
||||
Rule.ExcludePattern("META-INF/.*\\.[sS][fF]"),
|
||||
Rule.ExcludePattern("META-INF/.*\\.[dD][sS][aA]"),
|
||||
Rule.ExcludePattern("META-INF/.*\\.[rR][sS][aA]")
|
||||
val assemblyRules = Seq[Assembly.Rule](
|
||||
Assembly.Rule.Append("META-INF/services/org.apache.hadoop.fs.FileSystem"),
|
||||
Assembly.Rule.Append("reference.conf"),
|
||||
Assembly.Rule.AppendPattern("META-INF/services/.*"),
|
||||
Assembly.Rule.Exclude("log4j.properties"),
|
||||
Assembly.Rule.Exclude(JarFile.MANIFEST_NAME),
|
||||
Assembly.Rule.ExcludePattern("META-INF/.*\\.[sS][fF]"),
|
||||
Assembly.Rule.ExcludePattern("META-INF/.*\\.[dD][sS][aA]"),
|
||||
Assembly.Rule.ExcludePattern("META-INF/.*\\.[rR][sS][aA]")
|
||||
)
|
||||
|
||||
def sparkBaseDependencies(
|
||||
|
|
@ -272,7 +156,14 @@ object Assembly {
|
|||
dest.getParentFile.mkdirs()
|
||||
val tmpDest = new File(dest.getParentFile, s".${dest.getName}.part")
|
||||
// FIXME Acquire lock on tmpDest
|
||||
Assembly.make(jars, tmpDest, assemblyRules)
|
||||
var fos: FileOutputStream = null
|
||||
try {
|
||||
fos = new FileOutputStream(tmpDest)
|
||||
Assembly.make(jars, fos, Nil, assemblyRules)
|
||||
} finally {
|
||||
if (fos != null)
|
||||
fos.close()
|
||||
}
|
||||
Files.move(tmpDest.toPath, dest.toPath, StandardCopyOption.ATOMIC_MOVE)
|
||||
Right((dest, jars))
|
||||
}.left.map(_.describe)
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
package coursier.cli.util
|
||||
|
||||
import java.io.{File, FileInputStream, OutputStream}
|
||||
import java.util.jar.{Attributes, JarOutputStream, Manifest}
|
||||
import java.util.regex.Pattern
|
||||
import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream}
|
||||
|
||||
import scala.collection.mutable
|
||||
|
||||
object Assembly {
|
||||
|
||||
sealed abstract class Rule extends Product with Serializable
|
||||
|
||||
object Rule {
|
||||
sealed abstract class PathRule extends Rule {
|
||||
def path: String
|
||||
}
|
||||
|
||||
final case class Exclude(path: String) extends PathRule
|
||||
final case class ExcludePattern(path: Pattern) extends Rule
|
||||
|
||||
object ExcludePattern {
|
||||
def apply(s: String): ExcludePattern =
|
||||
ExcludePattern(Pattern.compile(s))
|
||||
}
|
||||
|
||||
// TODO Accept a separator: Array[Byte] argument in these
|
||||
// (to separate content with a line return in particular)
|
||||
final case class Append(path: String) extends PathRule
|
||||
final case class AppendPattern(path: Pattern) extends Rule
|
||||
|
||||
object AppendPattern {
|
||||
def apply(s: String): AppendPattern =
|
||||
AppendPattern(Pattern.compile(s))
|
||||
}
|
||||
}
|
||||
|
||||
def make(jars: Seq[File], output: OutputStream, attributes: Seq[(Attributes.Name, String)], rules: Seq[Rule]): Unit = {
|
||||
|
||||
val rulesMap = rules.collect { case r: Rule.PathRule => r.path -> r }.toMap
|
||||
val excludePatterns = rules.collect { case Rule.ExcludePattern(p) => p }
|
||||
val appendPatterns = rules.collect { case Rule.AppendPattern(p) => p }
|
||||
|
||||
val manifest = new Manifest
|
||||
manifest.getMainAttributes.put(Attributes.Name.MANIFEST_VERSION, "1.0")
|
||||
for ((k, v) <- attributes)
|
||||
manifest.getMainAttributes.put(k, v)
|
||||
|
||||
var zos: ZipOutputStream = null
|
||||
|
||||
try {
|
||||
zos = new JarOutputStream(output, manifest)
|
||||
|
||||
val concatenedEntries = new mutable.HashMap[String, ::[(ZipEntry, Array[Byte])]]
|
||||
|
||||
var ignore = Set.empty[String]
|
||||
|
||||
for (jar <- jars) {
|
||||
var fis: FileInputStream = null
|
||||
var zis: ZipInputStream = null
|
||||
|
||||
try {
|
||||
fis = new FileInputStream(jar)
|
||||
zis = new ZipInputStream(fis)
|
||||
|
||||
for ((ent, content) <- Zip.zipEntries(zis)) {
|
||||
|
||||
def append() =
|
||||
concatenedEntries += ent.getName -> ::((ent, content), concatenedEntries.getOrElse(ent.getName, Nil))
|
||||
|
||||
rulesMap.get(ent.getName) match {
|
||||
case Some(Rule.Exclude(_)) =>
|
||||
// ignored
|
||||
|
||||
case Some(Rule.Append(_)) =>
|
||||
append()
|
||||
|
||||
case None =>
|
||||
if (!excludePatterns.exists(_.matcher(ent.getName).matches())) {
|
||||
if (appendPatterns.exists(_.matcher(ent.getName).matches()))
|
||||
append()
|
||||
else if (!ignore(ent.getName)) {
|
||||
ent.setCompressedSize(-1L)
|
||||
zos.putNextEntry(ent)
|
||||
zos.write(content)
|
||||
zos.closeEntry()
|
||||
|
||||
ignore += ent.getName
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} finally {
|
||||
if (zis != null)
|
||||
zis.close()
|
||||
if (fis != null)
|
||||
fis.close()
|
||||
}
|
||||
}
|
||||
|
||||
for ((_, entries) <- concatenedEntries) {
|
||||
val (ent, _) = entries.head
|
||||
|
||||
ent.setCompressedSize(-1L)
|
||||
|
||||
if (entries.tail.nonEmpty)
|
||||
ent.setSize(entries.map(_._2.length).sum)
|
||||
|
||||
zos.putNextEntry(ent)
|
||||
// for ((_, b) <- entries.reverse)
|
||||
// zos.write(b)
|
||||
zos.write(entries.reverse.toArray.flatMap(_._2))
|
||||
zos.closeEntry()
|
||||
}
|
||||
} finally {
|
||||
if (zos != null)
|
||||
zos.close()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -111,8 +111,10 @@ publish() {
|
|||
testBootstrap() {
|
||||
if [ "$SCALA_VERSION" = 2.12 ]; then
|
||||
sbt scalaFromEnv "project cli" pack
|
||||
cli/target/pack/bin/coursier bootstrap -o cs-echo io.get-coursier:echo:1.0.0
|
||||
if [ "$(./cs-echo foo)" != foo ]; then
|
||||
|
||||
cli/target/pack/bin/coursier bootstrap -o cs-echo io.get-coursier:echo:1.0.1
|
||||
local OUT="$(./cs-echo foo)"
|
||||
if [ "$OUT" != foo ]; then
|
||||
echo "Error: unexpected output from bootstrapped echo command." 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
|
@ -133,6 +135,33 @@ testBootstrap() {
|
|||
echo "Error: unexpected output from bootstrapped echo command (generated by proguarded launcher)." 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# run via the launcher rather than via the sbt-pack scripts, because the latter interprets -Dfoo=baz itself
|
||||
# rather than passing it to coursier since https://github.com/xerial/sbt-pack/pull/118
|
||||
./coursier-test bootstrap -o cs-props -D other=thing -J -Dfoo=baz io.get-coursier:props:1.0.2
|
||||
local OUT="$(./cs-props foo)"
|
||||
if [ "$OUT" != baz ]; then
|
||||
echo -e "Error: unexpected output from bootstrapped props command.\n$OUT" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
local OUT="$(./cs-props other)"
|
||||
if [ "$OUT" != thing ]; then
|
||||
echo -e "Error: unexpected output from bootstrapped props command.\n$OUT" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# assembly tests
|
||||
./coursier-test bootstrap -a -o cs-props-assembly -D other=thing -J -Dfoo=baz io.get-coursier:props:1.0.2
|
||||
local OUT="$(./cs-props-assembly foo)"
|
||||
if [ "$OUT" != baz ]; then
|
||||
echo -e "Error: unexpected output from assembly props command.\n$OUT" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
local OUT="$(./cs-props-assembly other)"
|
||||
if [ "$OUT" != thing ]; then
|
||||
echo -e "Error: unexpected output from assembly props command.\n$OUT" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue