diff --git a/cli/src/main/scala-2.12/coursier/cli/Bootstrap.scala b/cli/src/main/scala-2.12/coursier/cli/Bootstrap.scala index a5eba41f5..0de4519c3 100644 --- a/cli/src/main/scala-2.12/coursier/cli/Bootstrap.scala +++ b/cli/src/main/scala-2.12/coursier/cli/Bootstrap.scala @@ -1,22 +1,276 @@ package coursier package cli -import java.io.{ByteArrayInputStream, ByteArrayOutputStream, File, FileInputStream, IOException} +import java.io.{ByteArrayInputStream, ByteArrayOutputStream, File, FileInputStream, FileOutputStream, IOException} import java.nio.charset.StandardCharsets.UTF_8 import java.nio.file.Files import java.nio.file.attribute.PosixFilePermission import java.util.Properties +import java.util.jar.{JarFile, Attributes => JarAttributes} import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream} import caseapp._ import coursier.cli.options.BootstrapOptions -import coursier.cli.util.Zip +import coursier.cli.util.{Assembly, Zip} import coursier.internal.FileUtil import scala.collection.JavaConverters._ object Bootstrap extends CaseApp[BootstrapOptions] { + private def createNativeBootstrap( + options: BootstrapOptions, + helper: Helper, + mainClass: String + ): Unit = { + + val files = helper.fetch( + sources = false, + javadoc = false, + artifactTypes = options.artifactOptions.artifactTypes(sources = false, javadoc = false) + ) + + val log: String => Unit = + if (options.options.common.verbosityLevel >= 0) + s => Console.err.println(s) + else + _ => () + + val tmpDir = new File(options.options.target) + + try { + coursier.extra.Native.create( + mainClass, + files, + new File(options.options.output), + tmpDir, + log, + verbosity = options.options.common.verbosityLevel + ) + } finally { + if (!options.options.keepTarget) + coursier.extra.Native.deleteRecursive(tmpDir) + } + } + + private def createJarBootstrap(javaOpts: Seq[String], output: File, content: Array[Byte]): Unit = { + + val javaCmd = Seq("java") ++ + javaOpts + // escaping possibly a bit loose :-| + .map(s => "'" + s.replace("'", "\\'") + "'") ++ + Seq( + "-jar", + "\"$0\"", + "\"$@\"" + ) + + val shellPreamble = Seq( + "#!/usr/bin/env sh", + "exec " + javaCmd.mkString(" ") + ).mkString("", "\n", "\n") + + try Files.write(output.toPath, shellPreamble.getBytes(UTF_8) ++ content) + catch { case e: IOException => + Console.err.println(s"Error while writing $output${Option(e.getMessage).fold("")(" (" + _ + ")")}") + sys.exit(1) + } + + try { + val perms = Files.getPosixFilePermissions(output.toPath).asScala.toSet + + var newPerms = perms + if (perms(PosixFilePermission.OWNER_READ)) + newPerms += PosixFilePermission.OWNER_EXECUTE + if (perms(PosixFilePermission.GROUP_READ)) + newPerms += PosixFilePermission.GROUP_EXECUTE + if (perms(PosixFilePermission.OTHERS_READ)) + newPerms += PosixFilePermission.OTHERS_EXECUTE + + if (newPerms != perms) + Files.setPosixFilePermissions( + output.toPath, + newPerms.asJava + ) + } catch { + case e: UnsupportedOperationException => + // Ignored + case e: IOException => + Console.err.println( + s"Error while making $output executable" + + Option(e.getMessage).fold("")(" (" + _ + ")") + ) + sys.exit(1) + } + } + + private def createOneJarLikeJarBootstrap( + options: BootstrapOptions, + helper: Helper, + mainClass: String, + javaOpts: Seq[String], + urls: Seq[String], + files: Seq[File], + output: File + ): Unit = { + + val bootstrapJar = + Option(Thread.currentThread().getContextClassLoader.getResourceAsStream("bootstrap.jar")) match { + case Some(is) => FileUtil.readFully(is) + case None => + Console.err.println(s"Error: bootstrap JAR not found") + sys.exit(1) + } + + val isolatedDeps = options.options.isolated.isolatedDeps(options.options.common.scalaVersion) + + val (_, isolatedArtifactFiles) = + options.options.isolated.targets.foldLeft((Vector.empty[String], Map.empty[String, (Seq[String], Seq[File])])) { + case ((done, acc), target) => + + // TODO Add non regression test checking that optional artifacts indeed land in the isolated loader URLs + + val m = helper.fetchMap( + sources = false, + javadoc = false, + artifactTypes = options.artifactOptions.artifactTypes(sources = false, javadoc = false), + subset = isolatedDeps.getOrElse(target, Seq.empty).toSet + ) + + val (done0, subUrls, subFiles) = + if (options.options.standalone) { + val subFiles0 = m.values.toSeq + (done, Nil, subFiles0) + } else { + val filteredSubArtifacts = m.keys.toSeq.diff(done) + (done ++ filteredSubArtifacts, filteredSubArtifacts, Nil) + } + + val updatedAcc = acc + (target -> (subUrls, subFiles)) + + (done0, updatedAcc) + } + + val isolatedUrls = isolatedArtifactFiles.map { case (k, (v, _)) => k -> v } + val isolatedFiles = isolatedArtifactFiles.map { case (k, (_, v)) => k -> v } + + val buffer = new ByteArrayOutputStream + + val bootstrapZip = new ZipInputStream(new ByteArrayInputStream(bootstrapJar)) + val outputZip = new ZipOutputStream(buffer) + + for ((ent, data) <- Zip.zipEntries(bootstrapZip)) { + outputZip.putNextEntry(ent) + outputZip.write(data) + outputZip.closeEntry() + } + + + val time = System.currentTimeMillis() + + def putStringEntry(name: String, content: String): Unit = { + val entry = new ZipEntry(name) + entry.setTime(time) + + outputZip.putNextEntry(entry) + outputZip.write(content.getBytes(UTF_8)) + outputZip.closeEntry() + } + + def putEntryFromFile(name: String, f: File): Unit = { + val entry = new ZipEntry(name) + entry.setTime(f.lastModified()) + + outputZip.putNextEntry(entry) + outputZip.write(FileUtil.readFully(new FileInputStream(f))) + outputZip.closeEntry() + } + + putStringEntry("bootstrap-jar-urls", urls.mkString("\n")) + + if (options.options.isolated.anyIsolatedDep) { + putStringEntry("bootstrap-isolation-ids", options.options.isolated.targets.mkString("\n")) + + for (target <- options.options.isolated.targets) { + val urls = isolatedUrls.getOrElse(target, Nil) + val files = isolatedFiles.getOrElse(target, Nil) + putStringEntry(s"bootstrap-isolation-$target-jar-urls", urls.mkString("\n")) + putStringEntry(s"bootstrap-isolation-$target-jar-resources", files.map(pathFor).mkString("\n")) + } + } + + def pathFor(f: File) = s"jars/${f.getName}" + + for (f <- files) + putEntryFromFile(pathFor(f), f) + + putStringEntry("bootstrap-jar-resources", files.map(pathFor).mkString("\n")) + + val propsEntry = new ZipEntry("bootstrap.properties") + propsEntry.setTime(time) + + val properties = new Properties + properties.setProperty("bootstrap.mainClass", mainClass) + + outputZip.putNextEntry(propsEntry) + properties.store(outputZip, "") + outputZip.closeEntry() + + outputZip.close() + + createJarBootstrap( + javaOpts, + output, + buffer.toByteArray + ) + } + + private def defaultRules = Seq( + Assembly.Rule.Append("reference.conf"), + Assembly.Rule.AppendPattern("META-INF/services/.*"), + Assembly.Rule.Exclude("log4j.properties"), + Assembly.Rule.Exclude(JarFile.MANIFEST_NAME), + Assembly.Rule.ExcludePattern("META-INF/.*\\.[sS][fF]"), + Assembly.Rule.ExcludePattern("META-INF/.*\\.[dD][sS][aA]"), + Assembly.Rule.ExcludePattern("META-INF/.*\\.[rR][sS][aA]") + ) + + private def createAssemblyJar( + options: BootstrapOptions, + files: Seq[File], + javaOpts: Seq[String], + mainClass: String, + output: File + ): Unit = { + + val parsedRules = options.options.rule.map { s => + s.split(":", 2) match { + case Array("append", v) => Assembly.Rule.Append(v) + case Array("append-pattern", v) => Assembly.Rule.AppendPattern(v) + case Array("exclude", v) => Assembly.Rule.Exclude(v) + case Array("exclude-pattern", v) => Assembly.Rule.ExcludePattern(v) + case _ => + sys.error(s"Malformed assembly rule: $s") + } + } + + val rules = + (if (options.options.defaultRules) defaultRules else Nil) ++ parsedRules + + val attrs = Seq( + JarAttributes.Name.MAIN_CLASS -> mainClass + ) + + val baos = new ByteArrayOutputStream + Assembly.make(files, baos, attrs, rules) + + createJarBootstrap( + javaOpts, + output, + baos.toByteArray + ) + } + def run(options: BootstrapOptions, args: RemainingArgs): Unit = { val helper = new Helper( @@ -38,36 +292,9 @@ object Bootstrap extends CaseApp[BootstrapOptions] { else options.options.mainClass - if (options.options.native) { - - val files = helper.fetch( - sources = false, - javadoc = false, - artifactTypes = options.artifactOptions.artifactTypes(sources = false, javadoc = false) - ) - - val log: String => Unit = - if (options.options.common.verbosityLevel >= 0) - s => Console.err.println(s) - else - _ => () - - val tmpDir = new File(options.options.target) - - try { - coursier.extra.Native.create( - mainClass, - files, - output0, - tmpDir, - log, - verbosity = options.options.common.verbosityLevel - ) - } finally { - if (!options.options.keepTarget) - coursier.extra.Native.deleteRecursive(tmpDir) - } - } else { + if (options.options.native) + createNativeBootstrap(options, helper, mainClass) + else { val (validProperties, wrongProperties) = options.options.property.partition(_.contains("=")) if (wrongProperties.nonEmpty) { @@ -76,165 +303,39 @@ object Bootstrap extends CaseApp[BootstrapOptions] { } val properties0 = validProperties.map { s => - val idx = s.indexOf('=') - assert(idx >= 0) - (s.take(idx), s.drop(idx + 1)) + s.split("=", 2) match { + case Array(k, v) => k -> v + case _ => sys.error("Cannot possibly happen") + } } - val bootstrapJar = - Option(Thread.currentThread().getContextClassLoader.getResourceAsStream("bootstrap.jar")) match { - case Some(is) => FileUtil.readFully(is) - case None => - Console.err.println(s"Error: bootstrap JAR not found") - sys.exit(1) - } - - val isolatedDeps = options.options.isolated.isolatedDeps(options.options.common.scalaVersion) - - val (_, isolatedArtifactFiles) = - options.options.isolated.targets.foldLeft((Vector.empty[String], Map.empty[String, (Seq[String], Seq[File])])) { - case ((done, acc), target) => - - // TODO Add non regression test checking that optional artifacts indeed land in the isolated loader URLs - - val m = helper.fetchMap( - sources = false, - javadoc = false, - artifactTypes = options.artifactOptions.artifactTypes(sources = false, javadoc = false), - subset = isolatedDeps.getOrElse(target, Seq.empty).toSet - ) - - val (done0, subUrls, subFiles) = - if (options.options.standalone) { - val subFiles0 = m.values.toSeq - (done, Nil, subFiles0) - } else { - val filteredSubArtifacts = m.keys.toSeq.diff(done) - (done ++ filteredSubArtifacts, filteredSubArtifacts, Nil) - } - - val updatedAcc = acc + (target -> (subUrls, subFiles)) - - (done0, updatedAcc) - } + val javaOpts = options.options.javaOpt ++ + properties0.map { case (k, v) => s"-D$k=$v" } val (urls, files) = helper.fetchMap( - sources = false, - javadoc = false, - artifactTypes = options.artifactOptions.artifactTypes(sources = false, javadoc = false) - ).toList.foldLeft((List.empty[String], List.empty[File])){ - case ((urls, files), (url, file)) => - if (options.options.standalone) (urls, file :: files) - else if (url.startsWith("file:/")) (urls, file :: files) - else (url :: urls, files) - } - - val isolatedUrls = isolatedArtifactFiles.map { case (k, (v, _)) => k -> v } - val isolatedFiles = isolatedArtifactFiles.map { case (k, (_, v)) => k -> v } - - val buffer = new ByteArrayOutputStream - - val bootstrapZip = new ZipInputStream(new ByteArrayInputStream(bootstrapJar)) - val outputZip = new ZipOutputStream(buffer) - - for ((ent, data) <- Zip.zipEntries(bootstrapZip)) { - outputZip.putNextEntry(ent) - outputZip.write(data) - outputZip.closeEntry() - } - - - val time = System.currentTimeMillis() - - def putStringEntry(name: String, content: String): Unit = { - val entry = new ZipEntry(name) - entry.setTime(time) - - outputZip.putNextEntry(entry) - outputZip.write(content.getBytes(UTF_8)) - outputZip.closeEntry() - } - - def putEntryFromFile(name: String, f: File): Unit = { - val entry = new ZipEntry(name) - entry.setTime(f.lastModified()) - - outputZip.putNextEntry(entry) - outputZip.write(FileUtil.readFully(new FileInputStream(f))) - outputZip.closeEntry() - } - - putStringEntry("bootstrap-jar-urls", urls.mkString("\n")) - - if (options.options.isolated.anyIsolatedDep) { - putStringEntry("bootstrap-isolation-ids", options.options.isolated.targets.mkString("\n")) - - for (target <- options.options.isolated.targets) { - val urls = isolatedUrls.getOrElse(target, Nil) - val files = isolatedFiles.getOrElse(target, Nil) - putStringEntry(s"bootstrap-isolation-$target-jar-urls", urls.mkString("\n")) - putStringEntry(s"bootstrap-isolation-$target-jar-resources", files.map(pathFor).mkString("\n")) + sources = false, + javadoc = false, + artifactTypes = options.artifactOptions.artifactTypes(sources = false, javadoc = false) + ).toList.foldLeft((List.empty[String], List.empty[File])){ + case ((urls, files), (url, file)) => + if (options.options.assembly || options.options.standalone) (urls, file :: files) + else if (url.startsWith("file:/")) (urls, file :: files) + else (url :: urls, files) } - } - def pathFor(f: File) = s"jars/${f.getName}" - - for (f <- files) - putEntryFromFile(pathFor(f), f) - - putStringEntry("bootstrap-jar-resources", files.map(pathFor).mkString("\n")) - - val propsEntry = new ZipEntry("bootstrap.properties") - propsEntry.setTime(time) - - val properties = new Properties - properties.setProperty("bootstrap.mainClass", mainClass) - - outputZip.putNextEntry(propsEntry) - properties.store(outputZip, "") - outputZip.closeEntry() - - outputZip.close() - - // escaping of javaOpt possibly a bit loose :-| - val shellPreamble = Seq( - "#!/usr/bin/env sh", - "exec java -jar " + options.options.javaOpt.map(s => "'" + s.replace("'", "\\'") + "'").mkString(" ") + " \"$0\" \"$@\"" - ).mkString("", "\n", "\n") - - try Files.write(output0.toPath, shellPreamble.getBytes(UTF_8) ++ buffer.toByteArray) - catch { case e: IOException => - Console.err.println(s"Error while writing $output0${Option(e.getMessage).fold("")(" (" + _ + ")")}") - sys.exit(1) - } - - try { - val perms = Files.getPosixFilePermissions(output0.toPath).asScala.toSet - - var newPerms = perms - if (perms(PosixFilePermission.OWNER_READ)) - newPerms += PosixFilePermission.OWNER_EXECUTE - if (perms(PosixFilePermission.GROUP_READ)) - newPerms += PosixFilePermission.GROUP_EXECUTE - if (perms(PosixFilePermission.OTHERS_READ)) - newPerms += PosixFilePermission.OTHERS_EXECUTE - - if (newPerms != perms) - Files.setPosixFilePermissions( - output0.toPath, - newPerms.asJava - ) - } catch { - case e: UnsupportedOperationException => - // Ignored - case e: IOException => - Console.err.println( - s"Error while making $output0 executable" + - Option(e.getMessage).fold("")(" (" + _ + ")") - ) - sys.exit(1) - } + if (options.options.assembly) + createAssemblyJar(options, files, javaOpts, mainClass, output0) + else + createOneJarLikeJarBootstrap( + options, + helper, + mainClass, + javaOpts, + urls, + files, + output0 + ) } } diff --git a/cli/src/main/scala-2.12/coursier/cli/SparkSubmit.scala b/cli/src/main/scala-2.12/coursier/cli/SparkSubmit.scala index 5a4dd9939..731b7d4ea 100644 --- a/cli/src/main/scala-2.12/coursier/cli/SparkSubmit.scala +++ b/cli/src/main/scala-2.12/coursier/cli/SparkSubmit.scala @@ -6,7 +6,7 @@ import java.net.URLClassLoader import caseapp._ import coursier.Dependency import coursier.cli.options.SparkSubmitOptions -import coursier.cli.spark.{Assembly, Submit} +import coursier.cli.spark.{SparkAssembly, Submit} /** @@ -85,7 +85,7 @@ object SparkSubmit extends CaseApp[SparkSubmitOptions] { val (sparkYarnExtraConf, sparkBaseJars) = if (!options.autoAssembly || sparkVersion.startsWith("2.")) { - val assemblyJars = Assembly.sparkJars( + val assemblyJars = SparkAssembly.sparkJars( scalaVersion, sparkVersion, options.yarnVersion, @@ -107,7 +107,7 @@ object SparkSubmit extends CaseApp[SparkSubmitOptions] { (extraConf, assemblyJars) } else { - val assemblyAndJarsOrError = Assembly.spark( + val assemblyAndJarsOrError = SparkAssembly.spark( scalaVersion, sparkVersion, options.yarnVersion, diff --git a/cli/src/main/scala-2.12/coursier/cli/options/BootstrapSpecificOptions.scala b/cli/src/main/scala-2.12/coursier/cli/options/BootstrapSpecificOptions.scala index ba6241bc6..4ee172c77 100644 --- a/cli/src/main/scala-2.12/coursier/cli/options/BootstrapSpecificOptions.scala +++ b/cli/src/main/scala-2.12/coursier/cli/options/BootstrapSpecificOptions.scala @@ -29,6 +29,15 @@ final case class BootstrapSpecificOptions( target: String = "native-target", @Help("Don't wipe native compilation target directory (for debug purposes)") keepTarget: Boolean = false, + @Help("Generate an assembly rather than a bootstrap jar") + @Short("a") + assembly: Boolean = false, + @Help("Add assembly rule") + @Value("append:$path|append-pattern:$pattern|exclude:$path|exclude-pattern:$pattern") + @Short("R") + rule: List[String] = Nil, + @Help("Add default rules to assembly rule list") + defaultRules: Boolean = true, @Recurse isolated: IsolatedLoaderOptions = IsolatedLoaderOptions(), @Recurse diff --git a/cli/src/main/scala-2.12/coursier/cli/spark/Assembly.scala b/cli/src/main/scala-2.12/coursier/cli/spark/SparkAssembly.scala similarity index 54% rename from cli/src/main/scala-2.12/coursier/cli/spark/Assembly.scala rename to cli/src/main/scala-2.12/coursier/cli/spark/SparkAssembly.scala index 477cbb919..1fd8ab549 100644 --- a/cli/src/main/scala-2.12/coursier/cli/spark/Assembly.scala +++ b/cli/src/main/scala-2.12/coursier/cli/spark/SparkAssembly.scala @@ -1,144 +1,28 @@ package coursier.cli.spark -import java.io.{File, FileInputStream, FileOutputStream} +import java.io.{File, FileOutputStream} import java.math.BigInteger import java.nio.charset.StandardCharsets.UTF_8 import java.nio.file.{Files, StandardCopyOption} import java.security.MessageDigest -import java.util.jar.{Attributes, JarFile, JarOutputStream, Manifest} -import java.util.regex.Pattern -import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream} +import java.util.jar.JarFile import coursier.Cache import coursier.cli.Helper import coursier.cli.options.CommonOptions -import coursier.cli.util.Zip +import coursier.cli.util.Assembly -import scala.collection.mutable +object SparkAssembly { -object Assembly { - - sealed abstract class Rule extends Product with Serializable - - object Rule { - sealed abstract class PathRule extends Rule { - def path: String - } - - final case class Exclude(path: String) extends PathRule - final case class ExcludePattern(path: Pattern) extends Rule - - object ExcludePattern { - def apply(s: String): ExcludePattern = - ExcludePattern(Pattern.compile(s)) - } - - // TODO Accept a separator: Array[Byte] argument in these - // (to separate content with a line return in particular) - final case class Append(path: String) extends PathRule - final case class AppendPattern(path: Pattern) extends Rule - - object AppendPattern { - def apply(s: String): AppendPattern = - AppendPattern(Pattern.compile(s)) - } - } - - def make(jars: Seq[File], output: File, rules: Seq[Rule]): Unit = { - - val rulesMap = rules.collect { case r: Rule.PathRule => r.path -> r }.toMap - val excludePatterns = rules.collect { case Rule.ExcludePattern(p) => p } - val appendPatterns = rules.collect { case Rule.AppendPattern(p) => p } - - val manifest = new Manifest - manifest.getMainAttributes.put(Attributes.Name.MANIFEST_VERSION, "1.0") - - var fos: FileOutputStream = null - var zos: ZipOutputStream = null - - try { - fos = new FileOutputStream(output) - zos = new JarOutputStream(fos, manifest) - - val concatenedEntries = new mutable.HashMap[String, ::[(ZipEntry, Array[Byte])]] - - var ignore = Set.empty[String] - - for (jar <- jars) { - var fis: FileInputStream = null - var zis: ZipInputStream = null - - try { - fis = new FileInputStream(jar) - zis = new ZipInputStream(fis) - - for ((ent, content) <- Zip.zipEntries(zis)) { - - def append() = - concatenedEntries += ent.getName -> ::((ent, content), concatenedEntries.getOrElse(ent.getName, Nil)) - - rulesMap.get(ent.getName) match { - case Some(Rule.Exclude(_)) => - // ignored - - case Some(Rule.Append(_)) => - append() - - case None => - if (!excludePatterns.exists(_.matcher(ent.getName).matches())) { - if (appendPatterns.exists(_.matcher(ent.getName).matches())) - append() - else if (!ignore(ent.getName)) { - ent.setCompressedSize(-1L) - zos.putNextEntry(ent) - zos.write(content) - zos.closeEntry() - - ignore += ent.getName - } - } - } - } - - } finally { - if (zis != null) - zis.close() - if (fis != null) - fis.close() - } - } - - for ((_, entries) <- concatenedEntries) { - val (ent, _) = entries.head - - ent.setCompressedSize(-1L) - - if (entries.tail.nonEmpty) - ent.setSize(entries.map(_._2.length).sum) - - zos.putNextEntry(ent) - // for ((_, b) <- entries.reverse) - // zos.write(b) - zos.write(entries.reverse.toArray.flatMap(_._2)) - zos.closeEntry() - } - } finally { - if (zos != null) - zos.close() - if (fos != null) - fos.close() - } - } - - val assemblyRules = Seq[Rule]( - Rule.Append("META-INF/services/org.apache.hadoop.fs.FileSystem"), - Rule.Append("reference.conf"), - Rule.AppendPattern("META-INF/services/.*"), - Rule.Exclude("log4j.properties"), - Rule.Exclude(JarFile.MANIFEST_NAME), - Rule.ExcludePattern("META-INF/.*\\.[sS][fF]"), - Rule.ExcludePattern("META-INF/.*\\.[dD][sS][aA]"), - Rule.ExcludePattern("META-INF/.*\\.[rR][sS][aA]") + val assemblyRules = Seq[Assembly.Rule]( + Assembly.Rule.Append("META-INF/services/org.apache.hadoop.fs.FileSystem"), + Assembly.Rule.Append("reference.conf"), + Assembly.Rule.AppendPattern("META-INF/services/.*"), + Assembly.Rule.Exclude("log4j.properties"), + Assembly.Rule.Exclude(JarFile.MANIFEST_NAME), + Assembly.Rule.ExcludePattern("META-INF/.*\\.[sS][fF]"), + Assembly.Rule.ExcludePattern("META-INF/.*\\.[dD][sS][aA]"), + Assembly.Rule.ExcludePattern("META-INF/.*\\.[rR][sS][aA]") ) def sparkBaseDependencies( @@ -272,7 +156,14 @@ object Assembly { dest.getParentFile.mkdirs() val tmpDest = new File(dest.getParentFile, s".${dest.getName}.part") // FIXME Acquire lock on tmpDest - Assembly.make(jars, tmpDest, assemblyRules) + var fos: FileOutputStream = null + try { + fos = new FileOutputStream(tmpDest) + Assembly.make(jars, fos, Nil, assemblyRules) + } finally { + if (fos != null) + fos.close() + } Files.move(tmpDest.toPath, dest.toPath, StandardCopyOption.ATOMIC_MOVE) Right((dest, jars)) }.left.map(_.describe) diff --git a/cli/src/main/scala-2.12/coursier/cli/util/Assembly.scala b/cli/src/main/scala-2.12/coursier/cli/util/Assembly.scala new file mode 100644 index 000000000..62b834d5a --- /dev/null +++ b/cli/src/main/scala-2.12/coursier/cli/util/Assembly.scala @@ -0,0 +1,122 @@ +package coursier.cli.util + +import java.io.{File, FileInputStream, OutputStream} +import java.util.jar.{Attributes, JarOutputStream, Manifest} +import java.util.regex.Pattern +import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream} + +import scala.collection.mutable + +object Assembly { + + sealed abstract class Rule extends Product with Serializable + + object Rule { + sealed abstract class PathRule extends Rule { + def path: String + } + + final case class Exclude(path: String) extends PathRule + final case class ExcludePattern(path: Pattern) extends Rule + + object ExcludePattern { + def apply(s: String): ExcludePattern = + ExcludePattern(Pattern.compile(s)) + } + + // TODO Accept a separator: Array[Byte] argument in these + // (to separate content with a line return in particular) + final case class Append(path: String) extends PathRule + final case class AppendPattern(path: Pattern) extends Rule + + object AppendPattern { + def apply(s: String): AppendPattern = + AppendPattern(Pattern.compile(s)) + } + } + + def make(jars: Seq[File], output: OutputStream, attributes: Seq[(Attributes.Name, String)], rules: Seq[Rule]): Unit = { + + val rulesMap = rules.collect { case r: Rule.PathRule => r.path -> r }.toMap + val excludePatterns = rules.collect { case Rule.ExcludePattern(p) => p } + val appendPatterns = rules.collect { case Rule.AppendPattern(p) => p } + + val manifest = new Manifest + manifest.getMainAttributes.put(Attributes.Name.MANIFEST_VERSION, "1.0") + for ((k, v) <- attributes) + manifest.getMainAttributes.put(k, v) + + var zos: ZipOutputStream = null + + try { + zos = new JarOutputStream(output, manifest) + + val concatenedEntries = new mutable.HashMap[String, ::[(ZipEntry, Array[Byte])]] + + var ignore = Set.empty[String] + + for (jar <- jars) { + var fis: FileInputStream = null + var zis: ZipInputStream = null + + try { + fis = new FileInputStream(jar) + zis = new ZipInputStream(fis) + + for ((ent, content) <- Zip.zipEntries(zis)) { + + def append() = + concatenedEntries += ent.getName -> ::((ent, content), concatenedEntries.getOrElse(ent.getName, Nil)) + + rulesMap.get(ent.getName) match { + case Some(Rule.Exclude(_)) => + // ignored + + case Some(Rule.Append(_)) => + append() + + case None => + if (!excludePatterns.exists(_.matcher(ent.getName).matches())) { + if (appendPatterns.exists(_.matcher(ent.getName).matches())) + append() + else if (!ignore(ent.getName)) { + ent.setCompressedSize(-1L) + zos.putNextEntry(ent) + zos.write(content) + zos.closeEntry() + + ignore += ent.getName + } + } + } + } + + } finally { + if (zis != null) + zis.close() + if (fis != null) + fis.close() + } + } + + for ((_, entries) <- concatenedEntries) { + val (ent, _) = entries.head + + ent.setCompressedSize(-1L) + + if (entries.tail.nonEmpty) + ent.setSize(entries.map(_._2.length).sum) + + zos.putNextEntry(ent) + // for ((_, b) <- entries.reverse) + // zos.write(b) + zos.write(entries.reverse.toArray.flatMap(_._2)) + zos.closeEntry() + } + } finally { + if (zos != null) + zos.close() + } + } + +} diff --git a/scripts/travis.sh b/scripts/travis.sh index 3e247e940..2a8ee09ce 100755 --- a/scripts/travis.sh +++ b/scripts/travis.sh @@ -111,8 +111,10 @@ publish() { testBootstrap() { if [ "$SCALA_VERSION" = 2.12 ]; then sbt scalaFromEnv "project cli" pack - cli/target/pack/bin/coursier bootstrap -o cs-echo io.get-coursier:echo:1.0.0 - if [ "$(./cs-echo foo)" != foo ]; then + + cli/target/pack/bin/coursier bootstrap -o cs-echo io.get-coursier:echo:1.0.1 + local OUT="$(./cs-echo foo)" + if [ "$OUT" != foo ]; then echo "Error: unexpected output from bootstrapped echo command." 1>&2 exit 1 fi @@ -133,6 +135,33 @@ testBootstrap() { echo "Error: unexpected output from bootstrapped echo command (generated by proguarded launcher)." 1>&2 exit 1 fi + + # run via the launcher rather than via the sbt-pack scripts, because the latter interprets -Dfoo=baz itself + # rather than passing it to coursier since https://github.com/xerial/sbt-pack/pull/118 + ./coursier-test bootstrap -o cs-props -D other=thing -J -Dfoo=baz io.get-coursier:props:1.0.2 + local OUT="$(./cs-props foo)" + if [ "$OUT" != baz ]; then + echo -e "Error: unexpected output from bootstrapped props command.\n$OUT" 1>&2 + exit 1 + fi + local OUT="$(./cs-props other)" + if [ "$OUT" != thing ]; then + echo -e "Error: unexpected output from bootstrapped props command.\n$OUT" 1>&2 + exit 1 + fi + + # assembly tests + ./coursier-test bootstrap -a -o cs-props-assembly -D other=thing -J -Dfoo=baz io.get-coursier:props:1.0.2 + local OUT="$(./cs-props-assembly foo)" + if [ "$OUT" != baz ]; then + echo -e "Error: unexpected output from assembly props command.\n$OUT" 1>&2 + exit 1 + fi + local OUT="$(./cs-props-assembly other)" + if [ "$OUT" != thing ]; then + echo -e "Error: unexpected output from assembly props command.\n$OUT" 1>&2 + exit 1 + fi fi }