Merge pull request #526 from coursier/topic/shade-whole-namespaces

Allow to manually shade whole namespaces to speed up shading

sbt-shading/src/main/scala-2.10/coursier/Shading.scala

@@ -54,23 +54,16 @@ object Shading {
     }
   }
 
-  def createPackage(
-    baseJar: File,
+  def toShadeJars(
     currentProject: Project,
     res: Resolution,
     configs: Map[String, Set[String]],
     artifactFilesOrErrors: Map[Artifact, FileError \/ File],
     classpathTypes: Set[String],
-    shadingNamespace: String,
     baseConfig: String,
     shadedConf: String,
     log: sbt.Logger
-  ) = {
-
-    val outputJar = new File(
-      baseJar.getParentFile,
-      baseJar.getName.stripSuffix(".jar") + "-shading.jar"
-    )
+  ): Seq[File] = {
 
     def configDependencies(config: String) = {
 
@@ -98,11 +91,6 @@ object Shading {
       )
     }
 
-    // Things could be split into intermediate tasks here, like
-    //   shadingDependencies: Seq[coursier.Dependency], dependencies whose JARs are to be shaded
-    //   shadingJars: Seq[java.io.File], JARs about to be shaded
-    // Note that shadingDependencies is not explicitly calculated below.
-
     val dependencyArtifacts = res.dependencyArtifacts
       .filter { case (_, a) => classpathTypes(a.`type`) }
       .groupBy(_._1)
@@ -154,19 +142,55 @@ object Shading {
         allShadedConfJars.map("  " + _).sorted.mkString("\n")
     )
 
-    val toShadeJars = allShadedConfJars.filterNot(noShadeJars.toSet)
+    allShadedConfJars.filterNot(noShadeJars.toSet)
+  }
+
+  def toShadeClasses(
+    shadeNamespaces: Set[String],
+    toShadeJars: Seq[File],
+    log: sbt.Logger
+  ): Seq[String] = {
 
     log.info(
       s"Shading ${toShadeJars.length} JAR(s):\n" +
         toShadeJars.map("  " + _).sorted.mkString("\n")
     )
 
-    val toShadeClasses = toShadeJars.flatMap(jarClassNames)
+    val toShadeClasses0 = toShadeJars.flatMap(jarClassNames)
 
-    log.info(s"Found ${toShadeClasses.length} class(es) in JAR(s) to be shaded")
-    log.debug(toShadeClasses.map("  " + _).sorted.mkString("\n"))
+    log.info(s"Found ${toShadeClasses0.length} class(es) in JAR(s) to be shaded")
+    log.debug(toShadeClasses0.map("  " + _).sorted.mkString("\n"))
+
+    shadeNamespaces.toVector.sorted.foldLeft(toShadeClasses0) {
+      (toShade, namespace) =>
+        val prefix = namespace + "."
+        val (filteredOut, remaining) = toShade.partition(_.startsWith(prefix))
+
+        log.info(s"${filteredOut.length} classes already filtered out by shaded namespace $namespace")
+        log.debug(filteredOut.map("  " + _).sorted.mkString("\n"))
+
+        remaining
+    }
+  }
+
+  def createPackage(
+    baseJar: File,
+    shadingNamespace: String,
+    shadeNamespaces: Set[String],
+    toShadeClasses: Seq[String],
+    toShadeJars: Seq[File]
+  ) = {
+
+    val outputJar = new File(
+      baseJar.getParentFile,
+      baseJar.getName.stripSuffix(".jar") + "-shading.jar"
+    )
 
     val processor = new DefaultJarProcessor
+
+    for (namespace <- shadeNamespaces)
+      processor.addClassRename(new ClassRename(namespace + ".**", shadingNamespace + ".@0"))
+
     for (cls <- toShadeClasses)
       processor.addClassRename(new ClassRename(cls, shadingNamespace + ".@0"))
 

sbt-shading/src/main/scala-2.10/coursier/ShadingPlugin.scala

@@ -1,6 +1,8 @@
 package coursier
 
-import coursier.ivy.IvyXml.{ mappings => ivyXmlMappings }
+import java.io.File
+
+import coursier.ivy.IvyXml.{mappings => ivyXmlMappings}
 import sbt.Keys._
 import sbt.{AutoPlugin, Compile, Configuration, TaskKey, inConfig}
 
@@ -16,8 +18,15 @@ object ShadingPlugin extends AutoPlugin {
   private val baseDependencyConfiguration = "compile"
   val Shaded = Configuration("shaded", "", isPublic = true, List(), transitive = true)
 
+  // make that a setting?
   val shadingNamespace = TaskKey[String]("shading-namespace")
 
+  // make that a setting?
+  val shadeNamespaces = TaskKey[Set[String]]("shade-namespaces")
+
+  val toShadeJars = TaskKey[Seq[File]]("to-shade-jars")
+  val toShadeClasses = TaskKey[Seq[String]]("to-shade-classes")
+
   object autoImport {
 
     /** Scope for shading related tasks */
@@ -26,7 +35,18 @@ object ShadingPlugin extends AutoPlugin {
     /** Ivy configuration for shaded dependencies */
     val Shaded = ShadingPlugin.Shaded
 
+    /** Namespace under which shaded things will be moved */
     val shadingNamespace = ShadingPlugin.shadingNamespace
+
+    /**
+      * Assume everything under these namespaces is to be shaded.
+      *
+      * Allows to speed the shading phase, if everything under some namespaces is to be shaded.
+      */
+    val shadeNamespaces = ShadingPlugin.shadeNamespaces
+
+    val toShadeJars = ShadingPlugin.toShadeJars
+    val toShadeClasses = ShadingPlugin.toShadeClasses
   }
 
   // same as similar things under sbt.Classpaths, tweaking a bit the configuration scope
@@ -50,7 +70,8 @@ object ShadingPlugin extends AutoPlugin {
             conf.extend(Shaded)
           else
             conf
-      }
+      },
+      shadeNamespaces := Set()
     ) ++
     inConfig(Shading)(
       sbt.Defaults.configSettings ++
@@ -76,21 +97,35 @@ object ShadingPlugin extends AutoPlugin {
           },
           // required for cross-projects in particular
           unmanagedSourceDirectories := (unmanagedSourceDirectories in Compile).value,
-          packageBin := {
-            coursier.Shading.createPackage(
-              packageBin.in(baseSbtConfiguration).value,
+          toShadeJars := {
+            coursier.Shading.toShadeJars(
               coursierProject.in(baseSbtConfiguration).value,
               coursierResolution.in(baseSbtConfiguration).value,
               coursierConfigurations.in(baseSbtConfiguration).value,
               Keys.coursierArtifacts.in(baseSbtConfiguration).value,
               classpathTypes.value,
-              shadingNamespace.?.value.getOrElse {
-                throw new NoSuchElementException("shadingNamespace key not set")
-              },
               baseDependencyConfiguration,
               Shaded.name,
               streams.value.log
             )
+          },
+          toShadeClasses := {
+            coursier.Shading.toShadeClasses(
+              shadeNamespaces.value,
+              toShadeJars.value,
+              streams.value.log
+            )
+          },
+          packageBin := {
+            coursier.Shading.createPackage(
+              packageBin.in(baseSbtConfiguration).value,
+              shadingNamespace.?.value.getOrElse {
+                throw new NoSuchElementException("shadingNamespace key not set")
+              },
+              shadeNamespaces.value,
+              toShadeClasses.value,
+              toShadeJars.value
+            )
           }
         )
     )

sbt-shading/src/sbt-test/sbt-shading/shade-namespace/build.sbt

@@ -0,0 +1,31 @@
+
+enablePlugins(coursier.ShadingPlugin)
+shadingNamespace := "test.shaded"
+
+shadeNamespaces += "argonaut"
+
+libraryDependencies ++= Seq(
+  "io.argonaut" %% "argonaut" % "6.2-RC2" % "shaded",
+  // directly depending on that one for it not to be shaded
+  "org.scala-lang" % "scala-reflect" % scalaVersion.value
+)
+
+scalaVersion := "2.11.8"
+organization := "io.get-coursier.test"
+name := "shading-base-test"
+version := "0.1.0-SNAPSHOT"
+
+lazy val checkToShadeClasses = TaskKey[Unit]("check-to-shade-classes")
+
+checkToShadeClasses := {
+  val toShadeClasses0 = toShadeClasses.in(Shading).value
+
+  if (toShadeClasses0.nonEmpty) {
+    val log = streams.value.log
+    log.error(s"Found ${toShadeClasses0.length} classes to be explicitly shaded")
+    for (name <- toShadeClasses0)
+      log.error("  " + name)
+  }
+
+  assert(toShadeClasses0.isEmpty)
+}

sbt-shading/src/sbt-test/sbt-shading/shade-namespace/project/plugins.sbt

@@ -0,0 +1,31 @@
+{
+  val pluginVersion = sys.props.getOrElse(
+    "plugin.version",
+    throw new RuntimeException(
+      """|The system property 'plugin.version' is not defined.
+         |Specify this property using the scriptedLaunchOpts -D.""".stripMargin
+    )
+  )
+
+  addSbtPlugin("io.get-coursier" % "sbt-shading" % pluginVersion)
+}
+
+// for the locally publish jarjar
+resolvers += Resolver.mavenLocal
+
+val coursierJarjarVersion = "1.0.1-coursier-SNAPSHOT"
+
+def coursierJarjarFoundInM2 =
+  (file(sys.props("user.home")) / s".m2/repository/org/anarres/jarjar/jarjar-core/$coursierJarjarVersion").exists()
+
+def jarjarVersion =
+  if (coursierJarjarFoundInM2)
+    coursierJarjarVersion
+  else
+    sys.error(
+      "Ad hoc jarjar version not found. Run\n" +
+      "  git clone https://github.com/alexarchambault/jarjar.git && cd jarjar && git checkout 249c8dbb970f8 && ./gradlew install\n" +
+      "to run this test"
+    )
+
+libraryDependencies += "org.anarres.jarjar" % "jarjar-core" % jarjarVersion

sbt-shading/src/sbt-test/sbt-shading/shade-namespace/src/main/scala/Main.scala

@@ -0,0 +1,27 @@
+import java.io.File
+import java.nio.file.Files
+
+import argonaut._
+
+object Main extends App {
+
+  val expectedClassName =
+    if (args.headOption == Some("--shaded"))
+      "test.shaded.argonaut.Json"
+    else
+      // Don't use the literal "argonaut.Json", that seems to get
+      // changed to "test.shaded.argonaut.Json" by shading
+      "argonaut" + '.' + "Json"
+
+  val className = classOf[Json].getName
+
+  Console.err.println(s"Expected class name: $expectedClassName")
+  Console.err.println(s"Class name: $className")
+
+  if (className != expectedClassName)
+    sys.error(s"Expected class name $expectedClassName, got $className")
+
+  val msg = Json.obj().nospaces
+
+  Files.write(new File("output").toPath, msg.getBytes("UTF-8"))
+}

sbt-shading/src/sbt-test/sbt-shading/shade-namespace/test

@@ -0,0 +1,10 @@
+$ delete output
+> checkToShadeClasses
+> run
+$ exists output
+> publishLocal
+$ exec java -jar coursier launch io.get-coursier.test:shading-base-test_2.11:0.1.0-SNAPSHOT
+-$ exec java -jar coursier launch io.get-coursier.test:shading-base-test_2.11:0.1.0-SNAPSHOT -- --shaded
+> shading:publishLocal
+-$ exec java -jar coursier launch io.get-coursier.test:shading-base-test_2.11:0.1.0-SNAPSHOT
+$ exec java -jar coursier launch io.get-coursier.test:shading-base-test_2.11:0.1.0-SNAPSHOT -- --shaded