From 7d69815f642a99bf31c7f0894fe4acb10b853739 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sat, 8 Jul 2017 14:18:03 +0200 Subject: [PATCH] Allow to generate native bootstraps --- .travis.yml | 2 + appveyor.yml | 10 + build.sbt | 26 +- .../scala-2.11/coursier/cli/Bootstrap.scala | 379 +++++++------- .../scala-2.11/coursier/cli/Options.scala | 10 +- .../scala-2.11/coursier/extra/Native.scala | 466 ++++++++++++++++++ scripts/travis.sh | 57 ++- 7 files changed, 751 insertions(+), 199 deletions(-) create mode 100644 extra/src/main/scala-2.11/coursier/extra/Native.scala diff --git a/.travis.yml b/.travis.yml index 86f14bfac..794ffab2d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,8 @@ matrix: os: linux jdk: oraclejdk8 sudo: required + before_install: + - curl https://raw.githubusercontent.com/scala-native/scala-native/v0.3.1/bin/travis_setup.sh | bash -x services: - docker - env: SCALA_VERSION=2.10.6 PUBLISH=1 diff --git a/appveyor.yml b/appveyor.yml index 0cd412fb4..e43212432 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,6 +14,16 @@ install: - cmd: SET PATH=C:\sbt\sbt\bin;%JAVA_HOME%\bin;%PATH% - cmd: SET SBT_OPTS=-XX:MaxPermSize=2g -Xmx4g - git submodule update --init --recursive + - ps: | + if (!(Test-Path 'C:\Users\appveyor\.ivy2\local\sandbox\sandbox_native0.3_2.11\0.1-SNAPSHOT')) { + iex 'git clone https://github.com/coursier/scala-native' + Set-Location -Path scala-native + iex 'git checkout 550bf6e37d27' + iex 'sbt ++2.11.8 sandbox/publishLocal' + iex 'git checkout f8088aef6981' + iex 'sbt ++2.11.8 nscplugin/publishLocal util/publishLocal nir/publishLocal tools/publishLocal' + Set-Location -Path .. + } - ps: | if (!(Test-Path 'C:\Users\appveyor\.m2\repository\org\anarres\jarjar\jarjar-core\1.0.1-coursier-SNAPSHOT')) { iex 'git clone https://github.com/alexarchambault/jarjar' diff --git a/build.sbt b/build.sbt index 86d0eea5c..a1abe9532 100644 --- a/build.sbt +++ b/build.sbt @@ -110,10 +110,34 @@ lazy val bootstrap = project ) lazy val extra = project + .enablePlugins(ShadingPlugin) .dependsOn(coreJvm) .settings( shared, - coursierPrefix + coursierPrefix, + shading, + libs ++= { + val ver = "0.3.0-coursier-1" + if (scalaBinaryVersion.value == "2.11") + Seq( + "org.scala-native" %% "tools" % ver % "shaded", + // brought by tools, but issues in ShadingPlugin (with things published locally?) makes these not be shaded... + "org.scala-native" %% "nir" % ver % "shaded", + "org.scala-native" %% "util" % ver % "shaded", + Deps.fastParse % "shaded" + ) + else + Nil + }, + shadeNamespaces ++= + Set( + "fastparse", + "sourcecode" + ) ++ + // not blindly shading the whole scala.scalanative here, for some constant strings starting with + // "scala.scalanative.native." in scalanative not to get prefixed with "coursier.shaded." + Seq("codegen", "io", "linker", "nir", "optimizer", "tools", "util") + .map("scala.scalanative." + _) ) lazy val cli = project diff --git a/cli/src/main/scala-2.11/coursier/cli/Bootstrap.scala b/cli/src/main/scala-2.11/coursier/cli/Bootstrap.scala index 1174dd156..ea41b9ac8 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Bootstrap.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Bootstrap.scala @@ -27,209 +27,226 @@ final case class Bootstrap( warnBaseLoaderNotFound = false ) - lazy val downloadDir = - if (options.downloadDir.isEmpty) - helper.baseDependencies.headOption match { - case Some(dep) => - s"$${user.home}/.coursier/bootstrap/${dep.module.organization}/${dep.module.name}" - case None => - Console.err.println("Error: no dependencies specified.") - sys.exit(255) - } - else - options.downloadDir - - val (validProperties, wrongProperties) = options.property.partition(_.contains("=")) - if (wrongProperties.nonEmpty) { - Console.err.println(s"Wrong -P / --property option(s):\n${wrongProperties.mkString("\n")}") - sys.exit(255) - } - - val properties0 = validProperties.map { s => - val idx = s.indexOf('=') - assert(idx >= 0) - (s.take(idx), s.drop(idx + 1)) - } - - val bootstrapJar = - Option(Thread.currentThread().getContextClassLoader.getResourceAsStream("bootstrap.jar")) match { - case Some(is) => Cache.readFullySync(is) - case None => - Console.err.println(s"Error: bootstrap JAR not found") - sys.exit(1) - } - val output0 = new File(options.output) if (!options.force && output0.exists()) { Console.err.println(s"Error: ${options.output} already exists, use -f option to force erasing it.") sys.exit(1) } - val isolatedDeps = options.isolated.isolatedDeps(options.common.scalaVersion) - - val (_, isolatedArtifactFiles) = - options.isolated.targets.foldLeft((Vector.empty[String], Map.empty[String, (Seq[String], Seq[File])])) { - case ((done, acc), target) => - val subRes = helper.res.subset(isolatedDeps.getOrElse(target, Nil).toSet) - val subArtifacts = subRes.artifacts.map(_.url) - - val filteredSubArtifacts = subArtifacts.diff(done) - - def subFiles0 = helper.fetch( - sources = false, - javadoc = false, - artifactTypes = artifactOptions.artifactTypes(sources = false, javadoc = false), - subset = isolatedDeps.getOrElse(target, Seq.empty).toSet - ) - - val (subUrls, subFiles) = - if (options.standalone) - (Nil, subFiles0) - else - (filteredSubArtifacts, Nil) - - val updatedAcc = acc + (target -> (subUrls, subFiles)) - - (done ++ filteredSubArtifacts, updatedAcc) - } - - val (urls, files) = - if (options.standalone) - ( - Seq.empty[String], - helper.fetch( - sources = false, - javadoc = false, - artifactTypes = artifactOptions.artifactTypes(sources = false, javadoc = false) - ) - ) - else - ( - helper.artifacts( - sources = false, - javadoc = false, - artifactTypes = artifactOptions.artifactTypes(sources = false, javadoc = false) - ).map(_.url), - Seq.empty[File] - ) - - val isolatedUrls = isolatedArtifactFiles.map { case (k, (v, _)) => k -> v } - val isolatedFiles = isolatedArtifactFiles.map { case (k, (_, v)) => k -> v } - - val nonHttpUrls = urls.filter(s => !s.startsWith("http://") && !s.startsWith("https://")) - if (nonHttpUrls.nonEmpty) - Console.err.println(s"Warning: non HTTP URLs:\n${nonHttpUrls.mkString("\n")}") - val mainClass = if (options.mainClass.isEmpty) helper.retainedMainClass else options.mainClass - val buffer = new ByteArrayOutputStream + if (options.native) { - val bootstrapZip = new ZipInputStream(new ByteArrayInputStream(bootstrapJar)) - val outputZip = new ZipOutputStream(buffer) + val files = helper.fetch( + sources = false, + javadoc = false, + artifactTypes = artifactOptions.artifactTypes(sources = false, javadoc = false) + ) - for ((ent, data) <- Zip.zipEntries(bootstrapZip)) { - outputZip.putNextEntry(ent) - outputZip.write(data) + val log: String => Unit = + if (options.common.verbosityLevel >= 0) + s => Console.err.println(s) + else + _ => () + + val tmpDir = new File(options.target) + + try { + coursier.extra.Native.create( + mainClass, + files, + output0, + tmpDir, + log, + verbosity = options.common.verbosityLevel + ) + } finally { + if (!options.keepTarget) + coursier.extra.Native.deleteRecursive(tmpDir) + } + } else { + + val (validProperties, wrongProperties) = options.property.partition(_.contains("=")) + if (wrongProperties.nonEmpty) { + Console.err.println(s"Wrong -P / --property option(s):\n${wrongProperties.mkString("\n")}") + sys.exit(255) + } + + val properties0 = validProperties.map { s => + val idx = s.indexOf('=') + assert(idx >= 0) + (s.take(idx), s.drop(idx + 1)) + } + + val bootstrapJar = + Option(Thread.currentThread().getContextClassLoader.getResourceAsStream("bootstrap.jar")) match { + case Some(is) => Cache.readFullySync(is) + case None => + Console.err.println(s"Error: bootstrap JAR not found") + sys.exit(1) + } + + val isolatedDeps = options.isolated.isolatedDeps(options.common.scalaVersion) + + val (_, isolatedArtifactFiles) = + options.isolated.targets.foldLeft((Vector.empty[String], Map.empty[String, (Seq[String], Seq[File])])) { + case ((done, acc), target) => + val subRes = helper.res.subset(isolatedDeps.getOrElse(target, Nil).toSet) + val subArtifacts = subRes.artifacts.map(_.url) + + val filteredSubArtifacts = subArtifacts.diff(done) + + def subFiles0 = helper.fetch( + sources = false, + javadoc = false, + artifactTypes = artifactOptions.artifactTypes(sources = false, javadoc = false), + subset = isolatedDeps.getOrElse(target, Seq.empty).toSet + ) + + val (subUrls, subFiles) = + if (options.standalone) + (Nil, subFiles0) + else + (filteredSubArtifacts, Nil) + + val updatedAcc = acc + (target -> (subUrls, subFiles)) + + (done ++ filteredSubArtifacts, updatedAcc) + } + + val (urls, files) = + if (options.standalone) + ( + Seq.empty[String], + helper.fetch( + sources = false, + javadoc = false, + artifactTypes = artifactOptions.artifactTypes(sources = false, javadoc = false) + ) + ) + else + ( + helper.artifacts( + sources = false, + javadoc = false, + artifactTypes = artifactOptions.artifactTypes(sources = false, javadoc = false) + ).map(_.url), + Seq.empty[File] + ) + + val isolatedUrls = isolatedArtifactFiles.map { case (k, (v, _)) => k -> v } + val isolatedFiles = isolatedArtifactFiles.map { case (k, (_, v)) => k -> v } + + val nonHttpUrls = urls.filter(s => !s.startsWith("http://") && !s.startsWith("https://")) + if (nonHttpUrls.nonEmpty) + Console.err.println(s"Warning: non HTTP URLs:\n${nonHttpUrls.mkString("\n")}") + + val buffer = new ByteArrayOutputStream + + val bootstrapZip = new ZipInputStream(new ByteArrayInputStream(bootstrapJar)) + val outputZip = new ZipOutputStream(buffer) + + for ((ent, data) <- Zip.zipEntries(bootstrapZip)) { + outputZip.putNextEntry(ent) + outputZip.write(data) + outputZip.closeEntry() + } + + + val time = System.currentTimeMillis() + + def putStringEntry(name: String, content: String): Unit = { + val entry = new ZipEntry(name) + entry.setTime(time) + + outputZip.putNextEntry(entry) + outputZip.write(content.getBytes("UTF-8")) + outputZip.closeEntry() + } + + def putEntryFromFile(name: String, f: File): Unit = { + val entry = new ZipEntry(name) + entry.setTime(f.lastModified()) + + outputZip.putNextEntry(entry) + outputZip.write(Cache.readFullySync(new FileInputStream(f))) + outputZip.closeEntry() + } + + putStringEntry("bootstrap-jar-urls", urls.mkString("\n")) + + if (options.isolated.anyIsolatedDep) { + putStringEntry("bootstrap-isolation-ids", options.isolated.targets.mkString("\n")) + + for (target <- options.isolated.targets) { + val urls = isolatedUrls.getOrElse(target, Nil) + val files = isolatedFiles.getOrElse(target, Nil) + putStringEntry(s"bootstrap-isolation-$target-jar-urls", urls.mkString("\n")) + putStringEntry(s"bootstrap-isolation-$target-jar-resources", files.map(pathFor).mkString("\n")) + } + } + + def pathFor(f: File) = s"jars/${f.getName}" + + for (f <- files) + putEntryFromFile(pathFor(f), f) + + putStringEntry("bootstrap-jar-resources", files.map(pathFor).mkString("\n")) + + val propsEntry = new ZipEntry("bootstrap.properties") + propsEntry.setTime(time) + + val properties = new Properties + properties.setProperty("bootstrap.mainClass", mainClass) + + outputZip.putNextEntry(propsEntry) + properties.store(outputZip, "") outputZip.closeEntry() - } + outputZip.close() - val time = System.currentTimeMillis() + // escaping of javaOpt possibly a bit loose :-| + val shellPreamble = Seq( + "#!/usr/bin/env sh", + "exec java -jar " + options.javaOpt.map(s => "'" + s.replace("'", "\\'") + "'").mkString(" ") + " \"$0\" \"$@\"" + ).mkString("", "\n", "\n") - def putStringEntry(name: String, content: String): Unit = { - val entry = new ZipEntry(name) - entry.setTime(time) + try FileUtil.write(output0, shellPreamble.getBytes("UTF-8") ++ buffer.toByteArray) + catch { case e: IOException => + Console.err.println(s"Error while writing $output0${Option(e.getMessage).fold("")(" (" + _ + ")")}") + sys.exit(1) + } - outputZip.putNextEntry(entry) - outputZip.write(content.getBytes("UTF-8")) - outputZip.closeEntry() - } + try { + val perms = Files.getPosixFilePermissions(output0.toPath).asScala.toSet - def putEntryFromFile(name: String, f: File): Unit = { - val entry = new ZipEntry(name) - entry.setTime(f.lastModified()) + var newPerms = perms + if (perms(PosixFilePermission.OWNER_READ)) + newPerms += PosixFilePermission.OWNER_EXECUTE + if (perms(PosixFilePermission.GROUP_READ)) + newPerms += PosixFilePermission.GROUP_EXECUTE + if (perms(PosixFilePermission.OTHERS_READ)) + newPerms += PosixFilePermission.OTHERS_EXECUTE - outputZip.putNextEntry(entry) - outputZip.write(Cache.readFullySync(new FileInputStream(f))) - outputZip.closeEntry() - } - - putStringEntry("bootstrap-jar-urls", urls.mkString("\n")) - - if (options.isolated.anyIsolatedDep) { - putStringEntry("bootstrap-isolation-ids", options.isolated.targets.mkString("\n")) - - for (target <- options.isolated.targets) { - val urls = isolatedUrls.getOrElse(target, Nil) - val files = isolatedFiles.getOrElse(target, Nil) - putStringEntry(s"bootstrap-isolation-$target-jar-urls", urls.mkString("\n")) - putStringEntry(s"bootstrap-isolation-$target-jar-resources", files.map(pathFor).mkString("\n")) + if (newPerms != perms) + Files.setPosixFilePermissions( + output0.toPath, + newPerms.asJava + ) + } catch { + case e: UnsupportedOperationException => + // Ignored + case e: IOException => + Console.err.println( + s"Error while making $output0 executable" + + Option(e.getMessage).fold("")(" (" + _ + ")") + ) + sys.exit(1) } } - def pathFor(f: File) = s"jars/${f.getName}" - - for (f <- files) - putEntryFromFile(pathFor(f), f) - - putStringEntry("bootstrap-jar-resources", files.map(pathFor).mkString("\n")) - - val propsEntry = new ZipEntry("bootstrap.properties") - propsEntry.setTime(time) - - val properties = new Properties - properties.setProperty("bootstrap.mainClass", mainClass) - if (!options.standalone) - properties.setProperty("bootstrap.jarDir", downloadDir) - - outputZip.putNextEntry(propsEntry) - properties.store(outputZip, "") - outputZip.closeEntry() - - outputZip.close() - - // escaping of javaOpt possibly a bit loose :-| - val shellPreamble = Seq( - "#!/usr/bin/env sh", - "exec java -jar " + options.javaOpt.map(s => "'" + s.replace("'", "\\'") + "'").mkString(" ") + " \"$0\" \"$@\"" - ).mkString("", "\n", "\n") - - try FileUtil.write(output0, shellPreamble.getBytes("UTF-8") ++ buffer.toByteArray) - catch { case e: IOException => - Console.err.println(s"Error while writing $output0${Option(e.getMessage).fold("")(" (" + _ + ")")}") - sys.exit(1) - } - - - try { - val perms = Files.getPosixFilePermissions(output0.toPath).asScala.toSet - - var newPerms = perms - if (perms(PosixFilePermission.OWNER_READ)) - newPerms += PosixFilePermission.OWNER_EXECUTE - if (perms(PosixFilePermission.GROUP_READ)) - newPerms += PosixFilePermission.GROUP_EXECUTE - if (perms(PosixFilePermission.OTHERS_READ)) - newPerms += PosixFilePermission.OTHERS_EXECUTE - - if (newPerms != perms) - Files.setPosixFilePermissions( - output0.toPath, - newPerms.asJava - ) - } catch { - case e: UnsupportedOperationException => - // Ignored - case e: IOException => - Console.err.println( - s"Error while making $output0 executable" + - Option(e.getMessage).fold("")(" (" + _ + ")") - ) - sys.exit(1) - } - } diff --git a/cli/src/main/scala-2.11/coursier/cli/Options.scala b/cli/src/main/scala-2.11/coursier/cli/Options.scala index 0649423e1..06ea5847d 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Options.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Options.scala @@ -232,8 +232,6 @@ final case class BootstrapOptions( mainClass: String = "", @Short("o") output: String = "bootstrap", - @Short("d") - downloadDir: String = "", @Short("f") force: Boolean = false, @Help("Generate a standalone launcher, with all JARs included, instead of one downloading its dependencies on startup.") @@ -247,6 +245,14 @@ final case class BootstrapOptions( @Value("option") @Short("J") javaOpt: List[String] = Nil, + @Help("Generate native launcher") + @Short("S") + native: Boolean = false, + @Help("Native compilation target directory") + @Short("d") + target: String = "native-target", + @Help("Don't wipe native compilation target directory (for debug purposes)") + keepTarget: Boolean = false, @Recurse isolated: IsolatedLoaderOptions = IsolatedLoaderOptions(), @Recurse diff --git a/extra/src/main/scala-2.11/coursier/extra/Native.scala b/extra/src/main/scala-2.11/coursier/extra/Native.scala new file mode 100644 index 000000000..bc4bd9334 --- /dev/null +++ b/extra/src/main/scala-2.11/coursier/extra/Native.scala @@ -0,0 +1,466 @@ +package coursier.extra + +import java.io.{File, FileInputStream, FileOutputStream, InputStream} +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.security.MessageDigest +import java.util.regex.Pattern +import java.util.zip.ZipInputStream + +import scala.collection.mutable +import scala.scalanative.{nir, tools} +import scala.sys.process._ +import scala.util.Try + +object Native { + + def discover(binaryName: String, + binaryVersions: Seq[(String, String)]): File = { + val docSetup = + "http://www.scala-native.org/en/latest/user/setup.html" + + val envName = + if (binaryName == "clang") "CLANG" + else if (binaryName == "clang++") "CLANGPP" + else binaryName + + sys.env.get(s"${envName}_PATH") match { + case Some(path) => new File(path) + case None => + val binaryNames = binaryVersions.flatMap { + case (major, minor) => + Seq(s"$binaryName$major$minor", s"$binaryName-$major.$minor") + } :+ binaryName + + Process("which" +: binaryNames).lines_! + .map(new File(_)) + .headOption + .getOrElse { + sys.error( + s"no ${binaryNames.mkString(", ")} found in $$PATH. Install clang ($docSetup)") + } + } + } + + val clangVersions = + Seq(("4", "0"), ("3", "9"), ("3", "8"), ("3", "7")) + + def checkThatClangIsRecentEnough(pathToClangBinary: File): Unit = { + def maybeFile(f: File) = f match { + case file if file.exists => Some(file.getAbsolutePath) + case none => None + } + + def definesBuiltIn( + pathToClangBinary: Option[String]): Option[Seq[String]] = { + def commandLineToListBuiltInDefines(clang: String) = + Seq("echo", "") #| Seq(clang, "-dM", "-E", "-") + def splitIntoLines(s: String) = s.split(f"%n") + def removeLeadingDefine(s: String) = s.substring(s.indexOf(' ') + 1) + + for { + clang <- pathToClangBinary + output = commandLineToListBuiltInDefines(clang).!! + lines = splitIntoLines(output) + } yield lines map removeLeadingDefine + } + + val clang = maybeFile(pathToClangBinary) + val defines: Seq[String] = definesBuiltIn(clang).to[Seq].flatten + val clangIsRecentEnough = + defines.contains("__DECIMAL_DIG__ __LDBL_DECIMAL_DIG__") + + if (!clangIsRecentEnough) { + sys.error( + s"No recent installation of clang found " + + s"at $pathToClangBinary.\nSee http://scala-native.readthedocs.io" + + s"/en/latest/user/setup.html for details.") + } + } + + implicit class FileOps(val f: File) extends AnyVal { + def **(expression: String): Seq[File] = { + def quote(s: String) = if (s.isEmpty) "" else Pattern.quote(s.replaceAll("\n", """\n""")) + val pattern = Pattern.compile(expression.split("\\*", -1).map(quote).mkString(".*")) + **(pattern) + } + + def **(pattern: Pattern): Seq[File] = + if (f.isDirectory) + f.listFiles().flatMap(_.**(pattern)) + else if (pattern.matcher(f.getName).matches()) + Seq(f) + else + Nil + } + + def include(linkerResult: tools.LinkerResult, gc: String, path: String) = { + val sep = java.io.File.separator + + if (path.contains(sep + "optional" + sep)) { + val name = new File(path).getName.split("\\.").head + linkerResult.links.map(_.name).contains(name) + } else if (path.contains(sep + "gc" + sep)) { + path.contains("gc" + sep + gc) + } else { + true + } + } + + sealed abstract class GarbageCollector(val name: String, + val links: Seq[String] = Nil) + object GarbageCollector { + object None extends GarbageCollector("none") + object Boehm extends GarbageCollector("boehm", Seq("gc")) + object Immix extends GarbageCollector("immix") + } + + def garbageCollector(gc: String) = gc match { + case "none" => GarbageCollector.None + case "boehm" => GarbageCollector.Boehm + case "immix" => GarbageCollector.Immix + case value => + sys.error("nativeGC can be either \"none\", \"boehm\" or \"immix\", not: " + value) + } + + def deleteRecursive(f: File): Unit = { + if (f.isDirectory) + f.listFiles().foreach(deleteRecursive) + + f.delete() + } + + def hash(f: File): Array[Byte] = { + + val md = MessageDigest.getInstance("SHA-1") + + val is = new FileInputStream(f) + try withContent(is, md.update(_, 0, _)) + finally is.close() + + md.digest() + } + + def unzip(from: File, toDirectory: File): Set[File] = { + + toDirectory.mkdirs() + + def extract(from: ZipInputStream, toDirectory: File) = { + val set = new mutable.HashSet[File] + def next(): Unit = { + val entry = from.getNextEntry + if (entry != null) { + val name = entry.getName + val target = new File(toDirectory, name) + + if (entry.isDirectory) + target.mkdirs() + else { + set += target + target.getParentFile.mkdirs() + + var fos: FileOutputStream = null + + try { + fos = new FileOutputStream(target) + withContent( + from, + (b, n) => fos.write(b, 0, n) + ) + } finally { + if (fos != null) fos.close() + } + } + + target.setLastModified(entry.getTime) + from.closeEntry() + next() + } + } + next() + Set() ++ set + } + + var fis: InputStream = null + var zis: ZipInputStream = null + + try { + fis = new FileInputStream(from) + zis = new ZipInputStream(fis) + extract(zis, toDirectory) + } finally { + if (fis != null) fis.close() + if (zis != null) zis.close() + } + } + + private def withContent(is: InputStream, f: (Array[Byte], Int) => Unit): Unit = { + val data = Array.ofDim[Byte](16384) + + var nRead = is.read(data, 0, data.length) + while (nRead != -1) { + f(data, nRead) + nRead = is.read(data, 0, data.length) + } + } + + + def create( + mainClass: String, + files: Seq[File], + output0: File, + wd: File, + log: String => Unit = s => Console.err.println(s), + verbosity: Int = 0 + ) = { + + val entry = nir.Global.Top(mainClass + "$") + + val clang = { + val clang = Native.discover("clang", Native.clangVersions) + Native.checkThatClangIsRecentEnough(clang) + clang + } + + val clangpp = { + val clang = Native.discover("clang++", Native.clangVersions) + Native.checkThatClangIsRecentEnough(clang) + clang + } + + val nativeTarget = { + // Use non-standard extension to not include the ll file when linking (#639) + val targetc = new File(wd, "target/c.probe") + val targetll = new File(wd, "target/ll.probe") + val compilec = + Seq( + clang.getAbsolutePath, + "-S", + "-xc", + "-emit-llvm", + "-o", + targetll.getAbsolutePath, + targetc.getAbsolutePath + ) + + def fail = + sys.error("Failed to detect native target.") + + targetc.getParentFile.mkdirs() + Files.write(targetc.toPath, "int probe;".getBytes(StandardCharsets.UTF_8)) + Console.err.println(compilec) + val exit = sys.process.Process(compilec, wd).! + if (exit == 0) + scala.io.Source.fromFile(targetll)(scala.io.Codec.UTF8) + .getLines() + .collectFirst { + case line if line.startsWith("target triple") => + line.split("\"").apply(1) + } + .getOrElse(fail) + else + fail + } + + def running(command: Seq[String]): Unit = + if (verbosity >= 2) + log("running" + System.lineSeparator() + command.mkString(System.lineSeparator() + "\t")) + + log(s"${files.length} files in classpath:") + if (verbosity >= 1) + for (f <- files) + log(f.toString) + + val nativeMode: tools.Mode = tools.Mode.Debug + + val config = + tools.Config.empty + .withEntry(entry) + .withPaths(files) + .withWorkdir(wd) + .withTarget(nativeTarget) + .withMode(nativeMode) + + val driver = + tools.OptimizerDriver(config) + + val linkingReporter = + tools.LinkerReporter.empty + + + log("Linking") + val linkerResult = tools.link(config, driver, linkingReporter) + + + if (linkerResult.unresolved.isEmpty) { + val classCount = linkerResult.defns.count { + case _: nir.Defn.Class | _: nir.Defn.Module | _: nir.Defn.Trait => true + case _ => false + } + + val methodCount = linkerResult.defns.count(_.isInstanceOf[nir.Defn.Define]) + + log(s"Discovered $classCount classes and $methodCount methods") + } else { + for (signature <- linkerResult.unresolved.map(_.show).sorted) + log(s"cannot link: $signature") + + sys.error("unable to link") + } + + + val optimizeReporter = tools.OptimizerReporter.empty + + log("Optimizing") + val optimized = tools.optimize(config, driver, linkerResult.defns, linkerResult.dyns, optimizeReporter) + + log("Generating intermediate code") + tools.codegen(config, optimized) + val generated = wd ** "*.ll" + + log(s"Produced ${generated.length} files") + + log("Compiling to native code") + + val compileOpts = { + val includes = { + val includedir = + Try(Process("llvm-config --includedir").lines_!) + .getOrElse(Seq.empty) + ("/usr/local/include" +: includedir).map(s => s"-I$s") + } + includes :+ "-Qunused-arguments" :+ + (nativeMode match { + case tools.Mode.Debug => "-O0" + case tools.Mode.Release => "-O2" + }) + } + + val apppaths = generated + .par + .map { ll => + val apppath = ll.getAbsolutePath + val outpath = apppath + ".o" + val compile = Seq(clangpp.getAbsolutePath, "-c", apppath, "-o", outpath) ++ compileOpts + running(compile) + Process(compile, wd).! + new File(outpath) + } + .seq + + + // this unpacks extra source files + val nativelib = { + + val lib = new File(wd, "lib") + val jar = + files + .map(entry => entry.getAbsolutePath) + .collectFirst { + case p if p.contains("scala-native") && p.contains("nativelib") => + new File(p) + } + .get + val jarhash = Native.hash(jar).toSeq + val jarhashfile = new File(lib, "jarhash") + val unpacked = + lib.exists && + jarhashfile.exists() && + jarhash == Files.readAllBytes(jarhashfile.toPath).toSeq + + if (!unpacked) { + Native.deleteRecursive(lib) + Native.unzip(jar, lib) + Files.write(jarhashfile.toPath, Native.hash(jar)) + } + + lib + } + + val cpaths = (wd ** "*.c").map(_.getAbsolutePath) + val cpppaths = (wd ** "*.cpp").map(_.getAbsolutePath) + val paths = cpaths ++ cpppaths + + val gc = "boehm" + + for (path <- paths if !Native.include(linkerResult, gc, path)) { + val ofile = new File(path + ".o") + if (ofile.exists()) + ofile.delete() + } + + val nativeCompileOptions = { + val includes = { + val includedir = + Try(Process("llvm-config --includedir").lines_!) + .getOrElse(Seq.empty) + ("/usr/local/include" +: includedir).map(s => s"-I$s") + } + includes :+ "-Qunused-arguments" :+ + (nativeMode match { + case tools.Mode.Debug => "-O0" + case tools.Mode.Release => "-O2" + }) + } + + val opts = nativeCompileOptions ++ Seq("-O2") + + paths + .par + .map { path => + val opath = path + ".o" + if (Native.include(linkerResult, gc, path) && !new File(opath).exists()) { + val isCpp = path.endsWith(".cpp") + val compiler = (if (isCpp) clangpp else clang).getAbsolutePath + val flags = (if (isCpp) Seq("-std=c++11") else Seq()) ++ opts + val compilec = Seq(compiler) ++ flags ++ Seq("-c", path, "-o", opath) + + running(compilec) + val result = Process(compilec, wd).! + if (result != 0) + sys.error("Failed to compile native library runtime code.") + result + } + } + .seq + + + val links = { + val os = Option(sys.props("os.name")).getOrElse("") + val arch = nativeTarget.split("-").head + // we need re2 to link the re2 c wrapper (cre2.h) + val librt = os match { + case "Linux" => Seq("rt") + case _ => Seq.empty + } + val libunwind = os match { + case "Mac OS X" => Seq.empty + case _ => Seq("unwind", "unwind-" + arch) + } + librt ++ libunwind ++ linkerResult.links + .map(_.name) ++ Native.garbageCollector(gc).links + } + + val nativeLinkingOptions = { + val libs = { + val libdir = + Try(Process("llvm-config --libdir").lines_!) + .getOrElse(Seq.empty) + ("/usr/local/lib" +: libdir).map(s => s"-L$s") + } + libs + } + + val linkopts = links.map("-l" + _) ++ nativeLinkingOptions + val targetopt = Seq("-target", nativeTarget) + val flags = Seq("-o", output0.getAbsolutePath) ++ linkopts ++ targetopt + val opaths = (nativelib ** "*.o").map(_.getAbsolutePath) + val paths0 = apppaths.map(_.getAbsolutePath) ++ opaths + val compile = clangpp.getAbsolutePath +: (flags ++ paths0) + + log("Linking native code") + running(compile) + Process(compile, wd).! + } + +} diff --git a/scripts/travis.sh b/scripts/travis.sh index f882249bd..184f2f469 100755 --- a/scripts/travis.sh +++ b/scripts/travis.sh @@ -9,6 +9,18 @@ SCALA_JS="${SCALA_JS:-0}" JARJAR_VERSION="${JARJAR_VERSION:-1.0.1-coursier-SNAPSHOT}" +is210() { + echo "$SCALA_VERSION" | grep -q "^2\.10" +} + +is211() { + echo "$SCALA_VERSION" | grep -q "^2\.11" +} + +is212() { + echo "$SCALA_VERSION" | grep -q "^2\.12" +} + setupCoursierBinDir() { mkdir -p bin cp coursier bin/ @@ -16,6 +28,7 @@ setupCoursierBinDir() { } downloadInstallSbtExtras() { + mkdir -p bin curl -L -o bin/sbt https://github.com/paulp/sbt-extras/raw/9ade5fa54914ca8aded44105bf4b9a60966f3ccd/sbt chmod +x bin/sbt } @@ -43,6 +56,18 @@ integrationTestsRequirements() { launchTestRepo --port 8081 } +setupCustomScalaNative() { + if [ ! -d "$HOME/.ivy2/local/org.scala-native/tools_2.11/0.3.0-coursier-1" ]; then + git clone https://github.com/coursier/scala-native.git + cd scala-native + git checkout 550bf6e37d27 + sbt ++2.11.8 sandbox/publishLocal + git checkout f8088aef6981 + sbt ++2.11.8 nscplugin/publishLocal util/publishLocal nir/publishLocal tools/publishLocal + cd .. + fi +} + setupCustomJarjar() { if [ ! -d "$HOME/.m2/repository/org/anarres/jarjar/jarjar-core/$JARJAR_VERSION" ]; then git clone https://github.com/alexarchambault/jarjar.git @@ -70,18 +95,6 @@ sbtShading() { [ "$SBT_SHADING" = 1 ] } -is210() { - echo "$SCALA_VERSION" | grep -q "^2\.10" -} - -is211() { - echo "$SCALA_VERSION" | grep -q "^2\.11" -} - -is212() { - echo "$SCALA_VERSION" | grep -q "^2\.12" -} - runSbtCoursierTests() { addPgpKeys sbt ++$SCALA_VERSION sbt-plugins/publishLocal @@ -216,6 +229,17 @@ testBootstrap() { fi } +testNativeBootstrap() { + if is211; then + sbt ++${SCALA_VERSION} cli/pack + cli/target/pack/bin/coursier bootstrap -S -o native-test sandbox::sandbox_native0.3:0.1-SNAPSHOT + if [ "$(./native-test)" != "Hello, World!" ]; then + echo "Error: unexpected output from native test bootstrap." 1>&2 + exit 1 + fi + fi +} + addPgpKeys() { for key in b41f2bce 9fa47a44 ae548ced b4493b94 53a97466 36ee59d9 dc426429 3b80305d 69e0a56c fdd5c0cd 35543c27 70173ee5 111557de 39c263a9; do gpg --keyserver keyserver.ubuntu.com --recv "$key" @@ -225,15 +249,18 @@ addPgpKeys() { # TODO Add coverage once https://github.com/scoverage/sbt-scoverage/issues/111 is fixed -setupCustomJarjar - -setupCoursierBinDir downloadInstallSbtExtras +setupCoursierBinDir + +setupCustomJarjar +setupCustomScalaNative if isScalaJs; then jsCompile runJsTests else + testNativeBootstrap + integrationTestsRequirements jvmCompile