Merge pull request #103 from alexarchambault/topic/bootstrap-isolation

Add support for ClassLoader isolation in generated bootstrap launchers

bootstrap/src/main/java/coursier/Bootstrap.java

@@ -11,6 +11,8 @@ import java.net.URL;
 import java.net.URLClassLoader;
 import java.net.URLConnection;
 import java.nio.file.Files;
+import java.security.CodeSource;
+import java.security.ProtectionDomain;
 import java.util.*;
 import java.util.concurrent.*;
 import java.util.regex.Matcher;
@@ -37,14 +39,41 @@ public class Bootstrap {
         return buffer.toByteArray();
     }
 
-    static String[] readJarUrls() throws IOException {
+    final static String defaultURLResource = "bootstrap-jar-urls";
+    final static String isolationIDsResource = "bootstrap-isolation-ids";
+
+    static String[] readStringSequence(String resource) throws IOException {
         ClassLoader loader = Thread.currentThread().getContextClassLoader();
-        InputStream is = loader.getResourceAsStream("bootstrap-jar-urls");
+        InputStream is = loader.getResourceAsStream(resource);
+        if (is == null)
+            return new String[] {};
         byte[] rawContent = readFullySync(is);
         String content = new String(rawContent, "UTF-8");
+        if (content.length() == 0)
+            return new String[] {};
         return content.split("\n");
     }
 
+    static Map<String, URL[]> readIsolationContexts(File jarDir, String[] isolationIDs) throws IOException {
+        final Map<String, URL[]> perContextURLs = new LinkedHashMap<>();
+
+        for (String isolationID: isolationIDs) {
+            String[] contextURLs = readStringSequence("bootstrap-isolation-" + isolationID + "-jar-urls");
+            List<URL> urls = new ArrayList<>();
+            for (String strURL : contextURLs) {
+                URL url = new URL(strURL);
+                File local = localFile(jarDir, url);
+                if (local.exists())
+                    urls.add(local.toURI().toURL());
+                else
+                    System.err.println("Warning: " + local + " not found.");
+            }
+            perContextURLs.put(isolationID, urls.toArray(new URL[urls.size()]));
+        }
+
+        return perContextURLs;
+    }
+
     final static int concurrentDownloadCount = 6;
 
     // http://stackoverflow.com/questions/872272/how-to-reference-another-property-in-java-util-properties/27724276#27724276
@@ -86,6 +115,33 @@ public class Bootstrap {
         return resolved;
     }
 
+    static String mainJarPath() {
+        ProtectionDomain protectionDomain = Bootstrap.class.getProtectionDomain();
+        if (protectionDomain != null) {
+            CodeSource source = protectionDomain.getCodeSource();
+            if (source != null) {
+                URL location = source.getLocation();
+                if (location != null && location.getProtocol().equals("file")) {
+                    return location.getPath();
+                }
+            }
+        }
+
+        return "";
+    }
+
+    static File localFile(File jarDir, URL url) {
+        if (url.getProtocol().equals("file"))
+            return new File(url.getPath());
+
+        String path = url.getPath();
+        int idx = path.lastIndexOf('/');
+        // FIXME Add other components in path to prevent conflicts?
+        String fileName = path.substring(idx + 1);
+        return new File(jarDir, fileName);
+    }
+
+
     public static void main(String[] args) throws Throwable {
 
         ThreadFactory threadFactory = new ThreadFactory() {
@@ -100,6 +156,12 @@ public class Bootstrap {
 
         ExecutorService pool = Executors.newFixedThreadPool(concurrentDownloadCount, threadFactory);
 
+        System.setProperty("coursier.mainJar", mainJarPath());
+
+        for (int i = 0; i < args.length; i++) {
+            System.setProperty("coursier.main.arg-" + i, args[i]);
+        }
+
         Map<String,String> properties = loadPropertiesMap(Thread.currentThread().getContextClassLoader().getResourceAsStream("bootstrap.properties"));
         for (Map.Entry<String, String> ent : properties.entrySet()) {
             System.setProperty(ent.getKey(), ent.getValue());
@@ -118,7 +180,7 @@ public class Bootstrap {
         } else if (!jarDir.mkdirs())
             System.err.println("Warning: cannot create " + jarDir0 + ", continuing anyway.");
 
-        String[] jarStrUrls = readJarUrls();
+        String[] jarStrUrls = readStringSequence(defaultURLResource);
 
         List<String> errors = new ArrayList<>();
         List<URL> urls = new ArrayList<>();
@@ -154,11 +216,7 @@ public class Bootstrap {
                 completionService.submit(new Callable<URL>() {
                     @Override
                     public URL call() throws Exception {
-                        String path = url0.getPath();
+                        File dest = localFile(jarDir, url0);
-                        int idx = path.lastIndexOf('/');
-                        // FIXME Add other components in path to prevent conflicts?
-                        String fileName = path.substring(idx + 1);
-                        File dest = new File(jarDir, fileName);
 
                         if (!dest.exists()) {
                             System.err.println("Downloading " + url0);
@@ -198,9 +256,17 @@ public class Bootstrap {
             exit("Interrupted");
         }
 
+        final String[] isolationIDs = readStringSequence(isolationIDsResource);
+        final Map<String, URL[]> perIsolationContextURLs = readIsolationContexts(jarDir, isolationIDs);
+
         Thread thread = Thread.currentThread();
         ClassLoader parentClassLoader = thread.getContextClassLoader();
 
+        for (String isolationID: isolationIDs) {
+            URL[] contextURLs = perIsolationContextURLs.get(isolationID);
+            parentClassLoader = new IsolatedClassLoader(contextURLs, parentClassLoader, new String[]{ isolationID });
+        }
+
         URLClassLoader classLoader = new URLClassLoader(localURLs.toArray(new URL[localURLs.size()]), parentClassLoader);
 
         Class<?> mainClass = null;

bootstrap/src/main/java/coursier/IsolatedClassLoader.java

@@ -0,0 +1,29 @@
+package coursier;
+
+import java.net.URL;
+import java.net.URLClassLoader;
+
+public class IsolatedClassLoader extends URLClassLoader {
+
+    private String[] isolationTargets;
+
+    public IsolatedClassLoader(
+            URL[] urls,
+            ClassLoader parent,
+            String[] isolationTargets
+    ) {
+        super(urls, parent);
+        this.isolationTargets = isolationTargets;
+    }
+
+    /**
+     * Applications wanting to access an isolated `ClassLoader` should inspect the hierarchy of
+     * loaders, and look into each of them for this method, by reflection. Then they should
+     * call it (still by reflection), and look for an agreed in advance target in it. If it is found,
+     * then the corresponding `ClassLoader` is the one with isolated dependencies.
+     */
+    public String[] getIsolationTargets() {
+        return isolationTargets;
+    }
+
+}

cache/src/main/scala/coursier/Cache.scala

@@ -301,12 +301,13 @@ object Cache {
 
         val res =
           if (url.startsWith("file:/")) {
-            def filtered(s: String) =
+            // for debug purposes, flaky with URL-encoded chars anyway
-              s.stripPrefix("file:/").stripPrefix("//").stripSuffix("/")
+            // def filtered(s: String) =
-            assert(
+            //   s.stripPrefix("file:/").stripPrefix("//").stripSuffix("/")
-              filtered(url) == filtered(file.toURI.toString),
+            // assert(
-              s"URL: ${filtered(url)}, file: ${filtered(file.toURI.toString)}"
+            //   filtered(url) == filtered(file.toURI.toString),
-            )
+            //   s"URL: ${filtered(url)}, file: ${filtered(file.toURI.toString)}"
+            // )
             checkFileExists(file, url)
           } else
             cachePolicy match {

cli/src/main/scala-2.11/coursier/cli/Coursier.scala

@@ -97,30 +97,18 @@ case class Fetch(
 
 }
 
-case class Launch(
+case class IsolatedLoaderOptions(
-  @Short("M")
-  @Short("main")
-    mainClass: String,
   @Value("target:dependency")
   @Short("I")
     isolated: List[String],
   @Help("Comma-separated isolation targets")
   @Short("i")
-    isolateTarget: List[String],
+    isolateTarget: List[String]
-  @Recurse
+) {
-    common: CommonOptions
-) extends CoursierCommand {
 
-  val (rawDependencies, extraArgs) = {
+  def anyIsolatedDep = isolateTarget.nonEmpty || isolated.nonEmpty
-    val idxOpt = Some(remainingArgs.indexOf("--")).filter(_ >= 0)
-    idxOpt.fold((remainingArgs, Seq.empty[String])) { idx =>
-      val (l, r) = remainingArgs.splitAt(idx)
-      assert(r.nonEmpty)
-      (l, r.tail)
-    }
-  }
 
-  val isolateTargets = {
+  lazy val targets = {
     val l = isolateTarget.flatMap(_.split(',')).filter(_.nonEmpty)
     val (invalid, valid) = l.partition(_.contains(":"))
     if (invalid.nonEmpty) {
@@ -135,21 +123,23 @@ case class Launch(
       valid.toArray
   }
 
-  val (validIsolated, unrecognizedIsolated) = isolated.partition(s => isolateTargets.exists(t => s.startsWith(t + ":")))
+  lazy val (validIsolated, unrecognizedIsolated) = isolated.partition(s => targets.exists(t => s.startsWith(t + ":")))
 
-  if (unrecognizedIsolated.nonEmpty) {
+  def check() = {
-    Console.err.println(s"Unrecognized isolation targets in:")
+    if (unrecognizedIsolated.nonEmpty) {
-    for (i <- unrecognizedIsolated)
+      Console.err.println(s"Unrecognized isolation targets in:")
-      Console.err.println(s"  $i")
+      for (i <- unrecognizedIsolated)
-    sys.exit(255)
+        Console.err.println(s"  $i")
+      sys.exit(255)
+    }
   }
 
-  val rawIsolated = validIsolated.map { s =>
+  lazy val rawIsolated = validIsolated.map { s =>
     val Array(target, dep) = s.split(":", 2)
     target -> dep
   }
 
-  val isolatedModuleVersions = rawIsolated.groupBy { case (t, _) => t }.map {
+  lazy val isolatedModuleVersions = rawIsolated.groupBy { case (t, _) => t }.map {
     case (t, l) =>
       val (errors, modVers) = Parse.moduleVersions(l.map { case (_, d) => d })
 
@@ -161,7 +151,7 @@ case class Launch(
       t -> modVers
   }
 
-  val isolatedDeps = isolatedModuleVersions.map {
+  lazy val isolatedDeps = isolatedModuleVersions.map {
     case (t, l) =>
       t -> l.map {
         case (mod, ver) =>
@@ -169,9 +159,30 @@ case class Launch(
       }
   }
 
+}
+
+case class Launch(
+  @Short("M")
+  @Short("main")
+    mainClass: String,
+  @Recurse
+    isolated: IsolatedLoaderOptions,
+  @Recurse
+    common: CommonOptions
+) extends CoursierCommand {
+
+  val (rawDependencies, extraArgs) = {
+    val idxOpt = Some(remainingArgs.indexOf("--")).filter(_ >= 0)
+    idxOpt.fold((remainingArgs, Seq.empty[String])) { idx =>
+      val (l, r) = remainingArgs.splitAt(idx)
+      assert(r.nonEmpty)
+      (l, r.tail)
+    }
+  }
+
   val helper = new Helper(
     common.copy(forceVersion = common.forceVersion),
-    rawDependencies ++ rawIsolated.map { case (_, dep) => dep }
+    rawDependencies ++ isolated.rawIsolated.map { case (_, dep) => dep }
   )
 
 
@@ -185,17 +196,17 @@ case class Launch(
   )
 
   val (parentLoader, filteredFiles) =
-    if (isolated.isEmpty)
+    if (isolated.isolated.isEmpty)
       (parentLoader0, files0)
     else {
-      val (isolatedLoader, filteredFiles0) = isolateTargets.foldLeft((parentLoader0, files0)) {
+      val (isolatedLoader, filteredFiles0) = isolated.targets.foldLeft((parentLoader0, files0)) {
         case ((parent, files0), target) =>
 
           // FIXME These were already fetched above
           val isolatedFiles = helper.fetch(
             sources = false,
             javadoc = false,
-            subset = isolatedDeps.getOrElse(target, Seq.empty).toSet
+            subset = isolated.isolatedDeps.getOrElse(target, Seq.empty).toSet
           )
 
           if (common.verbose0 >= 1) {
@@ -303,6 +314,8 @@ case class Bootstrap(
   @Value("key=value")
   @Short("P")
     property: List[String],
+  @Recurse
+    isolated: IsolatedLoaderOptions,
   @Recurse
     common: CommonOptions
 ) extends CoursierCommand {
@@ -368,9 +381,20 @@ case class Bootstrap(
 
   val helper = new Helper(common, remainingArgs)
 
-  val artifacts = helper.res.artifacts
+  val urls = helper.res.artifacts.map(_.url)
 
-  val urls = artifacts.map(_.url)
+  val (_, isolatedUrls) =
+    isolated.targets.foldLeft((Vector.empty[String], Map.empty[String, Seq[String]])) {
+      case ((done, acc), target) =>
+        val subRes = helper.res.subset(isolated.isolatedDeps.getOrElse(target, Nil).toSet)
+        val subUrls = subRes.artifacts.map(_.url)
+
+        val filteredSubUrls = subUrls.diff(done)
+
+        val updatedAcc = acc + (target -> filteredSubUrls)
+
+        (done ++ filteredSubUrls, updatedAcc)
+    }
 
   val unrecognized = urls.filter(s => !s.startsWith("http://") && !s.startsWith("https://"))
   if (unrecognized.nonEmpty)
@@ -390,12 +414,25 @@ case class Bootstrap(
 
   val time = System.currentTimeMillis()
 
-  val jarListEntry = new ZipEntry("bootstrap-jar-urls")
+  def putStringEntry(name: String, content: String): Unit = {
-  jarListEntry.setTime(time)
+    val entry = new ZipEntry(name)
+    entry.setTime(time)
 
-  outputZip.putNextEntry(jarListEntry)
+    outputZip.putNextEntry(entry)
-  outputZip.write(urls.mkString("\n").getBytes("UTF-8"))
+    outputZip.write(content.getBytes("UTF-8"))
-  outputZip.closeEntry()
+    outputZip.closeEntry()
+  }
+
+  putStringEntry("bootstrap-jar-urls", urls.mkString("\n"))
+
+  if (isolated.anyIsolatedDep) {
+    putStringEntry("bootstrap-isolation-ids", isolated.targets.mkString("\n"))
+
+    for (target <- isolated.targets) {
+      val urls = isolatedUrls.getOrElse(target, Nil)
+      putStringEntry(s"bootstrap-isolation-$target-jar-urls", urls.mkString("\n"))
+    }
+  }
 
   val propsEntry = new ZipEntry("bootstrap.properties")
   propsEntry.setTime(time)