diff --git a/.ci/travis.sh b/.ci/travis.sh index 544e3214b..9a1c66ac8 100755 --- a/.ci/travis.sh +++ b/.ci/travis.sh @@ -43,9 +43,33 @@ function isMasterOrDevelop() { SBT_COMMANDS="compile test it:test" +RUN_SHADING_TESTS=0 + if echo "$TRAVIS_SCALA_VERSION" | grep -q "^2\.10"; then SBT_COMMANDS="$SBT_COMMANDS publishLocal" # to make the scripted tests happy SBT_COMMANDS="$SBT_COMMANDS plugin/scripted" + + if [ "$RUN_SHADING_TESTS" = 1 ]; then + # for the shading scripted test + sudo cp coursier /usr/local/bin/ + + JARJAR_VERSION=1.0.1-coursier-SNAPSHOT + + if [ ! -d "$HOME/.m2/repository/org/anarres/jarjar/jarjar-core/$JARJAR_VERSION" ]; then + git clone https://github.com/alexarchambault/jarjar.git + cd jarjar + if ! grep -q "^version=$JARJAR_VERSION\$" gradle.properties; then + echo "Expected jarjar version not found" 1>&2 + exit 1 + fi + git checkout 249c8dbb970f8 + ./gradlew :jarjar-core:install + cd .. + rm -rf jarjar + fi + + SBT_COMMANDS="$SBT_COMMANDS sbt-shading/scripted" + fi fi SBT_COMMANDS="$SBT_COMMANDS tut coreJVM/mimaReportBinaryIssues cache/mimaReportBinaryIssues" diff --git a/.travis.yml b/.travis.yml index 427805666..4b2fda4ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,3 +36,9 @@ env: branches: only: - master +cache: + directories: + - $HOME/.m2 + - $HOME/.ivy2/cache + - $HOME/.sbt +# Not adding $HOME/.coursier, we check that sbt-coursier works fine with an initially empty cache diff --git a/appveyor.yml b/appveyor.yml index 6013b49bc..c9ad467a4 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,6 +14,15 @@ install: - cmd: SET PATH=C:\sbt\sbt\bin;%JAVA_HOME%\bin;%PATH% - cmd: SET SBT_OPTS=-XX:MaxPermSize=2g -Xmx4g - cmd: SET COURSIER_NO_TERM=1 + # required by the sbt-shading scripted tests, disabled for now (see below) + # ps: | + # if (!(Test-Path 'C:\Users\appveyor\.m2\repository\org\anarres\jarjar\jarjar-core\1.0.1-coursier-SNAPSHOT')) { + # iex 'git clone https://github.com/alexarchambault/jarjar' + # Set-Location -Path jarjar + # iex 'git checkout 249c8dbb970f8' + # iex './gradlew.bat :jarjar-core:install' + # Set-Location -Path .. + # } build_script: - sbt ++2.11.8 clean compile coreJVM/publishLocal http-server/publishLocal - sbt ++2.10.6 clean compile @@ -23,6 +32,9 @@ test_script: - sbt ++2.12.0 testsJVM/test testsJVM/it:test # Would node be around for testsJS/test? - sbt ++2.11.8 testsJVM/test testsJVM/it:test - sbt ++2.10.6 testsJVM/test testsJVM/it:test plugin/scripted + # not running the scripted tests of sbt-shading here, as these seem to fail randomly on single core machines + # (which is kind of worrying) cache: - C:\Users\appveyor\.ivy2 + - C:\Users\appveyor\.m2 - C:\Users\appveyor\.sbt diff --git a/build.sbt b/build.sbt index b55aa8e2f..0ff75dd5a 100644 --- a/build.sbt +++ b/build.sbt @@ -107,6 +107,28 @@ lazy val commonSettings = scalaVersionAgnosticCommonSettings ++ Seq( } ) +lazy val pluginSettings = + scalaVersionAgnosticCommonSettings ++ + noPublishForScalaVersionSettings("2.11", "2.12") ++ + ScriptedPlugin.scriptedSettings ++ + Seq( + scriptedLaunchOpts ++= Seq( + "-Xmx1024M", + "-XX:MaxPermSize=256M", + "-Dplugin.version=" + version.value, + "-Dsbttest.base=" + (sourceDirectory.value / "sbt-test").getAbsolutePath + ), + scriptedBufferLog := false, + sbtPlugin := (scalaBinaryVersion.value == "2.10"), + resolvers ++= Seq( + // added so that 2.10 artifacts of the other modules can be found by + // the too-naive-for-now inter-project resolver of the coursier SBT plugin + Resolver.sonatypeRepo("snapshots"), + // added for sbt-scripted to be fine even with ++2.11.x + Resolver.typesafeIvyRepo("releases") + ) + ) + val scalazVersion = "7.2.7" lazy val core = crossProject @@ -550,28 +572,20 @@ lazy val doc = project // Don't try to compile that if you're not in 2.10 lazy val plugin = project .dependsOn(coreJvm, cache) - .settings(scalaVersionAgnosticCommonSettings) - .settings(noPublishForScalaVersionSettings("2.11", "2.12")) + .settings(pluginSettings) .settings( - name := "sbt-coursier", - sbtPlugin := (scalaBinaryVersion.value == "2.10"), - resolvers ++= Seq( - // added so that 2.10 artifacts of the other modules can be found by - // the too-naive-for-now inter-project resolver of the coursier SBT plugin - Resolver.sonatypeRepo("snapshots"), - // added for sbt-scripted to be fine even with ++2.11.x - Resolver.typesafeIvyRepo("releases") - ) + name := "sbt-coursier" ) - .settings(ScriptedPlugin.scriptedSettings) + +// Don't try to compile that if you're not in 2.10 +lazy val `sbt-shading` = project + .dependsOn(plugin) + .settings(pluginSettings) .settings( - scriptedLaunchOpts ++= Seq( - "-Xmx1024M", - "-XX:MaxPermSize=256M", - "-Dplugin.version=" + version.value, - "-Dsbttest.base=" + (sourceDirectory.value / "sbt-test").getAbsolutePath - ), - scriptedBufferLog := false + // Warning: this version doesn't handle well class names with '$'s + // (so basically any Scala library) + // See https://github.com/shevek/jarjar/pull/4 + libraryDependencies += "org.anarres.jarjar" % "jarjar-core" % "1.0.0" ) val http4sVersion = "0.8.6" @@ -606,7 +620,7 @@ lazy val okhttp = project ) lazy val `coursier` = project.in(file(".")) - .aggregate(coreJvm, coreJs, `fetch-js`, testsJvm, testsJs, cache, bootstrap, cli, plugin, web, doc, `http-server`, okhttp) + .aggregate(coreJvm, coreJs, `fetch-js`, testsJvm, testsJs, cache, bootstrap, cli, plugin, `sbt-shading`, web, doc, `http-server`, okhttp) .settings(commonSettings) .settings(noPublishSettings) .settings(releaseSettings) diff --git a/plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala b/plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala index dcf7378c2..0778c35f9 100644 --- a/plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala +++ b/plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala @@ -38,6 +38,10 @@ object CoursierPlugin extends AutoPlugin { val coursierDependencyTree = Keys.coursierDependencyTree val coursierDependencyInverseTree = Keys.coursierDependencyInverseTree + + val coursierArtifacts = Keys.coursierArtifacts + val coursierClassifiersArtifacts = Keys.coursierClassifiersArtifacts + val coursierSbtClassifiersArtifacts = Keys.coursierSbtClassifiersArtifacts } import autoImport._ @@ -51,7 +55,10 @@ object CoursierPlugin extends AutoPlugin { ) ) - override lazy val projectSettings = Seq( + def coursierSettings( + shadedConfigOpt: Option[(String, String)], + packageConfigs: Seq[(Configuration, String)] + ) = Seq( coursierParallelDownloads := 6, coursierMaxIterations := 50, coursierDefaultArtifactType := "", @@ -68,27 +75,42 @@ object CoursierPlugin extends AutoPlugin { coursierCredentials := Map.empty, coursierFallbackDependencies <<= Tasks.coursierFallbackDependenciesTask, coursierCache := Cache.default, - update <<= Tasks.updateTask(withClassifiers = false), + coursierArtifacts <<= Tasks.artifactFilesOrErrors(withClassifiers = false), + coursierClassifiersArtifacts <<= Tasks.artifactFilesOrErrors( + withClassifiers = true + ), + coursierSbtClassifiersArtifacts <<= Tasks.artifactFilesOrErrors( + withClassifiers = true, + sbtClassifiers = true + ), + update <<= Tasks.updateTask( + shadedConfigOpt, + withClassifiers = false + ), updateClassifiers <<= Tasks.updateTask( + shadedConfigOpt, withClassifiers = true, ignoreArtifactErrors = true ), updateSbtClassifiers in Defaults.TaskGlobal <<= Tasks.updateTask( + shadedConfigOpt, withClassifiers = true, sbtClassifiers = true, ignoreArtifactErrors = true ), coursierProject <<= Tasks.coursierProjectTask, coursierInterProjectDependencies <<= Tasks.coursierInterProjectDependenciesTask, - coursierPublications <<= Tasks.coursierPublicationsTask, + coursierPublications <<= Tasks.coursierPublicationsTask(packageConfigs: _*), coursierSbtClassifiersModule <<= classifiersModule in updateSbtClassifiers, - coursierConfigurations <<= Tasks.coursierConfigurationsTask, + coursierConfigurations <<= Tasks.coursierConfigurationsTask(None), coursierResolution <<= Tasks.resolutionTask(), coursierSbtClassifiersResolution <<= Tasks.resolutionTask( sbtClassifiers = true ) - ) ++ - inConfig(Compile)(treeSettings) ++ - inConfig(Test)(treeSettings) + ) -} + override lazy val projectSettings = coursierSettings(None, Seq(Compile, Test).map(c => c -> c.name)) ++ + inConfig(Compile)(treeSettings) ++ + inConfig(Test)(treeSettings) + +} \ No newline at end of file diff --git a/plugin/src/main/scala-2.10/coursier/Keys.scala b/plugin/src/main/scala-2.10/coursier/Keys.scala index e8b5c5bc7..f2c894163 100644 --- a/plugin/src/main/scala-2.10/coursier/Keys.scala +++ b/plugin/src/main/scala-2.10/coursier/Keys.scala @@ -8,6 +8,7 @@ import coursier.core.Publication import sbt.{ GetClassifiersModule, Resolver, SettingKey, TaskKey } import scala.concurrent.duration.Duration +import scalaz.\/ object Keys { val coursierParallelDownloads = SettingKey[Int]("coursier-parallel-downloads") @@ -51,4 +52,8 @@ object Keys { "coursier-dependency-inverse-tree", "Prints dependencies and transitive dependencies as an inverted tree (dependees as children)" ) + + val coursierArtifacts = TaskKey[Map[Artifact, FileError \/ File]]("coursier-artifacts") + val coursierClassifiersArtifacts = TaskKey[Map[Artifact, FileError \/ File]]("coursier-classifiers-artifacts") + val coursierSbtClassifiersArtifacts = TaskKey[Map[Artifact, FileError \/ File]]("coursier-sbt-classifiers-artifacts") } diff --git a/plugin/src/main/scala-2.10/coursier/MakeIvyXml.scala b/plugin/src/main/scala-2.10/coursier/MakeIvyXml.scala index 45ef60556..eb81ed8c7 100644 --- a/plugin/src/main/scala-2.10/coursier/MakeIvyXml.scala +++ b/plugin/src/main/scala-2.10/coursier/MakeIvyXml.scala @@ -4,7 +4,20 @@ import scala.xml.{ Node, PrefixedAttribute } object MakeIvyXml { - def apply(project: Project): Node = { + def apply(project0: Project, shadedConfigOpt: Option[String]): Node = { + + val filterOutDependencies = + shadedConfigOpt.toSet[String].flatMap { shadedConfig => + project0 + .dependencies + .collect { case (`shadedConfig`, dep) => dep } + } + + val project: Project = project0.copy( + dependencies = project0.dependencies.collect { + case p @ (_, dep) if !filterOutDependencies(dep) => p + } + ) val infoAttrs = project.module.attributes.foldLeft[xml.MetaData](xml.Null) { case (acc, (k, v)) => @@ -31,11 +44,12 @@ object MakeIvyXml { } % infoAttrs - val confElems = project.configurations.toVector.map { - case (name, extends0) => + val confElems = project.configurations.toVector.collect { + case (name, extends0) if shadedConfigOpt != Some(name) => + val extends1 = shadedConfigOpt.fold(extends0)(c => extends0.filter(_ != c)) val n = - if (extends0.nonEmpty) - n % .attributes + if (extends1.nonEmpty) + n % .attributes else n } diff --git a/plugin/src/main/scala-2.10/coursier/Tasks.scala b/plugin/src/main/scala-2.10/coursier/Tasks.scala index 8f681ff8e..33bb136e2 100644 --- a/plugin/src/main/scala-2.10/coursier/Tasks.scala +++ b/plugin/src/main/scala-2.10/coursier/Tasks.scala @@ -13,7 +13,6 @@ import coursier.util.{ Config, Print } import org.apache.ivy.core.module.id.ModuleRevisionId import sbt.{ UpdateReport, Classpaths, Resolver, Def } -import sbt.Configurations.{ Compile, Test } import sbt.Keys._ import scala.collection.mutable @@ -147,7 +146,7 @@ object Tasks { coursierProject.forAllProjects(state, projects).map(_.values.toVector) } - def coursierPublicationsTask: Def.Initialize[sbt.Task[Seq[(String, Publication)]]] = + def coursierPublicationsTask(configsMap: (sbt.Configuration, String)*): Def.Initialize[sbt.Task[Seq[(String, Publication)]]] = ( sbt.Keys.state, sbt.Keys.thisProjectRef, @@ -158,17 +157,16 @@ object Tasks { ).map { (state, projectRef, projId, sv, sbv, ivyConfs) => val packageTasks = Seq(packageBin, packageSrc, packageDoc) - val configs = Seq(Compile, Test) val sbtArtifacts = for { pkgTask <- packageTasks - config <- configs + (config, targetConfig) <- configsMap } yield { val publish = publishArtifact.in(projectRef).in(pkgTask).in(config).getOrElse(state, false) if (publish) Option(artifact.in(projectRef).in(pkgTask).in(config).getOrElse(state, null)) - .map(config.name -> _) + .map(targetConfig -> _) else None } @@ -219,7 +217,7 @@ object Tasks { sbtArtifactsPublication ++ extraSbtArtifactsPublication } - def coursierConfigurationsTask = Def.task { + def coursierConfigurationsTask(shadedConfig: Option[(String, String)]) = Def.task { val configs0 = ivyConfigurations.value.map { config => config.name -> config.extendsConfigs.map(_.name) @@ -238,10 +236,18 @@ object Tasks { helper(Set(c)) } - configs0.map { + val map = configs0.map { case (config, _) => config -> allExtends(config) } + + map ++ shadedConfig.toSeq.flatMap { + case (baseConfig, shadedConfig) => + Seq( + baseConfig -> (map.getOrElse(baseConfig, Set(baseConfig)) + shadedConfig), + shadedConfig -> map.getOrElse(shadedConfig, Set(shadedConfig)) + ) + } } private case class ResolutionCacheKey( @@ -647,14 +653,155 @@ object Tasks { } } - def updateTask( + def artifactFilesOrErrors( withClassifiers: Boolean, sbtClassifiers: Boolean = false, ignoreArtifactErrors: Boolean = false ) = Def.task { - def grouped[K, V](map: Seq[(K, V)]): Map[K, Seq[V]] = - map.groupBy { case (k, _) => k }.map { + // let's update only one module at once, for a better output + // Downloads are already parallel, no need to parallelize further anyway + synchronized { + + lazy val cm = coursierSbtClassifiersModule.value + + lazy val projectName = thisProjectRef.value.project + + val parallelDownloads = coursierParallelDownloads.value + val artifactsChecksums = coursierArtifactsChecksums.value + val cachePolicies = coursierCachePolicies.value + val ttl = coursierTtl.value + val cache = coursierCache.value + + val log = streams.value.log + + val verbosityLevel = coursierVerbosity.value + + val res = { + if (withClassifiers && sbtClassifiers) + coursierSbtClassifiersResolution + else + coursierResolution + }.value + + val classifiers = + if (withClassifiers) + Some { + if (sbtClassifiers) + cm.classifiers + else + transitiveClassifiers.value + } + else + None + + val allArtifacts = + classifiers match { + case None => res.artifacts + case Some(cl) => res.classifiersArtifacts(cl) + } + + var pool: ExecutorService = null + var artifactsLogger: TermDisplay = null + + val printOptionalMessage = verbosityLevel >= 0 && verbosityLevel <= 1 + + val artifactFilesOrErrors = try { + pool = Executors.newFixedThreadPool(parallelDownloads, Strategy.DefaultDaemonThreadFactory) + artifactsLogger = createLogger() + + val artifactFileOrErrorTasks = allArtifacts.toVector.map { a => + def f(p: CachePolicy) = + Cache.file( + a, + cache, + p, + checksums = artifactsChecksums, + logger = Some(artifactsLogger), + pool = pool, + ttl = ttl + ) + + cachePolicies.tail + .foldLeft(f(cachePolicies.head))(_ orElse f(_)) + .run + .map((a, _)) + } + + val artifactInitialMessage = + if (verbosityLevel >= 0) + s"Fetching artifacts of $projectName" + + (if (sbtClassifiers) " (sbt classifiers)" else "") + else + "" + + if (verbosityLevel >= 2) + log.info(artifactInitialMessage) + + artifactsLogger.init(if (printOptionalMessage) log.info(artifactInitialMessage)) + + Task.gatherUnordered(artifactFileOrErrorTasks).attemptRun match { + case -\/(ex) => + ResolutionError.UnknownDownloadException(ex) + .throwException() + case \/-(l) => + l.toMap + } + } finally { + if (pool != null) + pool.shutdown() + if (artifactsLogger != null) + if ((artifactsLogger.stopDidPrintSomething() && printOptionalMessage) || verbosityLevel >= 2) + log.info( + s"Fetched artifacts of $projectName" + + (if (sbtClassifiers) " (sbt classifiers)" else "") + ) + } + + artifactFilesOrErrors + } + } + + private def artifactFileOpt( + sbtBootJarOverrides: Map[(Module, String), File], + artifactFiles: Map[Artifact, File], + erroredArtifacts: Set[Artifact], + log: sbt.Logger, + module: Module, + version: String, + artifact: Artifact + ) = { + + val artifact0 = artifact + .copy(attributes = Attributes()) // temporary hack :-( + + // Under some conditions, SBT puts the scala JARs of its own classpath + // in the application classpath. Ensuring we return SBT's jars rather than + // JARs from the coursier cache, so that a same JAR doesn't land twice in the + // application classpath (once via SBT jars, once via coursier cache). + val fromBootJars = + if (artifact.classifier.isEmpty && artifact.`type` == "jar") + sbtBootJarOverrides.get((module, version)) + else + None + + val res = fromBootJars.orElse(artifactFiles.get(artifact0)) + + if (res.isEmpty && !erroredArtifacts(artifact0)) + log.error(s"${artifact.url} not downloaded (should not happen)") + + res + } + + def updateTask( + shadedConfigOpt: Option[(String, String)], + withClassifiers: Boolean, + sbtClassifiers: Boolean = false, + ignoreArtifactErrors: Boolean = false + ) = Def.task { + + def grouped[K, V](map: Seq[(K, V)])(mapKey: K => K): Map[K, Seq[V]] = + map.groupBy { case (k, _) => mapKey(k) }.map { case (k, l) => k -> l.map { case (_, v) => v } } @@ -677,8 +824,6 @@ object Tasks { lazy val cm = coursierSbtClassifiersModule.value - lazy val projectName = thisProjectRef.value.project - val currentProject = if (sbtClassifiers) { val sv = scalaVersion.value @@ -713,12 +858,6 @@ object Tasks { val cacheIvyFile = ivyCacheManager.getResolvedIvyFileInCache(ivyModule) val cacheIvyPropertiesFile = ivyCacheManager.getResolvedIvyPropertiesInCache(ivyModule) - val parallelDownloads = coursierParallelDownloads.value - val artifactsChecksums = coursierArtifactsChecksums.value - val cachePolicies = coursierCachePolicies.value - val ttl = coursierTtl.value - val cache = coursierCache.value - val log = streams.value.log val verbosityLevel = coursierVerbosity.value @@ -730,7 +869,7 @@ object Tasks { val b = new StringBuilder b ++= """""" b += '\n' - b ++= printer.format(MakeIvyXml(currentProject)) + b ++= printer.format(MakeIvyXml(currentProject, shadedConfigOpt.map(_._2))) cacheIvyFile.getParentFile.mkdirs() FileUtil.write(cacheIvyFile, b.result().getBytes("UTF-8")) @@ -739,6 +878,8 @@ object Tasks { FileUtil.write(cacheIvyPropertiesFile, "".getBytes("UTF-8")) } + writeIvyFiles() + val res = { if (withClassifiers && sbtClassifiers) coursierSbtClassifiersResolution @@ -748,9 +889,25 @@ object Tasks { def report = { - val depsByConfig = grouped(currentProject.dependencies) + val depsByConfig = grouped(currentProject.dependencies)( + config => + shadedConfigOpt match { + case Some((baseConfig, `config`)) => + baseConfig + case _ => + config + } + ) - val configs = coursierConfigurations.value + val configs = { + val m = coursierConfigurations.value + shadedConfigOpt.fold(m) { + case (baseConfig, shadedConfig) => + (m - shadedConfig) + ( + baseConfig -> (m.getOrElse(baseConfig, Set()) - shadedConfig) + ) + } + } if (verbosityLevel >= 2) { val finalDeps = Config.dependenciesWithConfig( @@ -775,75 +932,22 @@ object Tasks { else None - val allArtifacts = - classifiers match { - case None => res.artifacts - case Some(cl) => res.classifiersArtifacts(cl) - } - - var pool: ExecutorService = null - var artifactsLogger: TermDisplay = null - - val printOptionalMessage = verbosityLevel >= 0 && verbosityLevel <= 1 - - val artifactFilesOrErrors = try { - pool = Executors.newFixedThreadPool(parallelDownloads, Strategy.DefaultDaemonThreadFactory) - artifactsLogger = createLogger() - - val artifactFileOrErrorTasks = allArtifacts.toVector.map { a => - def f(p: CachePolicy) = - Cache.file( - a, - cache, - p, - checksums = artifactsChecksums, - logger = Some(artifactsLogger), - pool = pool, - ttl = ttl - ) - - cachePolicies.tail - .foldLeft(f(cachePolicies.head))(_ orElse f(_)) - .run - .map((a, _)) - } - - val artifactInitialMessage = - if (verbosityLevel >= 0) - s"Fetching artifacts of $projectName" + - (if (sbtClassifiers) " (sbt classifiers)" else "") + val artifactFilesOrErrors0 = ( + if (withClassifiers) { + if (sbtClassifiers) + Keys.coursierSbtClassifiersArtifacts else - "" + Keys.coursierClassifiersArtifacts + } else + Keys.coursierArtifacts + ).value - if (verbosityLevel >= 2) - log.info(artifactInitialMessage) - - artifactsLogger.init(if (printOptionalMessage) log.info(artifactInitialMessage)) - - Task.gatherUnordered(artifactFileOrErrorTasks).attemptRun match { - case -\/(ex) => - ResolutionError.UnknownDownloadException(ex) - .throwException() - case \/-(l) => - l.toMap - } - } finally { - if (pool != null) - pool.shutdown() - if (artifactsLogger != null) - if ((artifactsLogger.stopDidPrintSomething() && printOptionalMessage) || verbosityLevel >= 2) - log.info( - s"Fetched artifacts of $projectName" + - (if (sbtClassifiers) " (sbt classifiers)" else "") - ) - } - - val artifactFiles = artifactFilesOrErrors.collect { + val artifactFiles = artifactFilesOrErrors0.collect { case (artifact, \/-(file)) => artifact -> file } - val artifactErrors = artifactFilesOrErrors.toVector.collect { + val artifactErrors = artifactFilesOrErrors0.toVector.collect { case (_, -\/(err)) => err } @@ -858,42 +962,25 @@ object Tasks { } // can be non empty only if ignoreArtifactErrors is true - val erroredArtifacts = artifactFilesOrErrors.collect { + val erroredArtifacts = artifactFilesOrErrors0.collect { case (artifact, -\/(_)) => artifact }.toSet - def artifactFileOpt(module: Module, version: String, artifact: Artifact) = { - - val artifact0 = artifact - .copy(attributes = Attributes()) // temporary hack :-( - - // Under some conditions, SBT puts the scala JARs of its own classpath - // in the application classpath. Ensuring we return SBT's jars rather than - // JARs from the coursier cache, so that a same JAR doesn't land twice in the - // application classpath (once via SBT jars, once via coursier cache). - val fromBootJars = - if (artifact.classifier.isEmpty && artifact.`type` == "jar") - sbtBootJarOverrides.get((module, version)) - else - None - - val res = fromBootJars.orElse(artifactFiles.get(artifact0)) - - if (res.isEmpty && !erroredArtifacts(artifact0)) - log.error(s"${artifact.url} not downloaded (should not happen)") - - res - } - - writeIvyFiles() - ToSbt.updateReport( depsByConfig, res, configs, classifiers, - artifactFileOpt + artifactFileOpt( + sbtBootJarOverrides, + artifactFiles, + erroredArtifacts, + log, + _, + _, + _ + ) ) } diff --git a/sbt-shading/src/main/scala-2.10/coursier/Shading.scala b/sbt-shading/src/main/scala-2.10/coursier/Shading.scala new file mode 100644 index 000000000..510c479fa --- /dev/null +++ b/sbt-shading/src/main/scala-2.10/coursier/Shading.scala @@ -0,0 +1,152 @@ +package coursier + +import java.io.{File, FileInputStream} +import java.util.jar.JarInputStream +import java.util.zip.{ZipEntry, ZipInputStream} + +import com.tonicsystems.jarjar.classpath.ClassPath +import com.tonicsystems.jarjar.transform.JarTransformer +import com.tonicsystems.jarjar.transform.config.ClassRename +import com.tonicsystems.jarjar.transform.jar.DefaultJarProcessor +import coursier.core.Orders +import sbt.file + +import scalaz.{\/, \/-} + +object Shading { + + // FIXME Also vaguely in cli + def zipEntries(zipStream: ZipInputStream): Iterator[ZipEntry] = + new Iterator[ZipEntry] { + var nextEntry = Option.empty[ZipEntry] + def update() = + nextEntry = Option(zipStream.getNextEntry) + + update() + + def hasNext = nextEntry.nonEmpty + def next() = { + val ent = nextEntry.get + update() + ent + } + } + + def jarClassNames(jar: File): Seq[String] = { + + var fis: FileInputStream = null + var zis: JarInputStream = null + + try { + fis = new FileInputStream(jar) + zis = new JarInputStream(fis) + + zipEntries(zis) + .map(_.getName) + .filter(_.endsWith(".class")) + .map(_.stripSuffix(".class").replace('/', '.')) + .toVector + } finally { + if (zis != null) + zis.close() + if (fis != null) + fis.close() + } + } + + def createPackage( + baseJar: File, + currentProject: Project, + res: Resolution, + configs: Map[String, Set[String]], + artifactFilesOrErrors: Map[Artifact, FileError \/ File], + shadingNamespace: String, + baseConfig: String, + shadedConf: String, + log: sbt.Logger + ) = { + + val outputJar = new File( + baseJar.getParentFile, + baseJar.getName.stripSuffix(".jar") + "-shading.jar" + ) + + def configDependencies(config: String) = { + + def minDependencies(dependencies: Set[Dependency]): Set[Dependency] = + Orders.minDependencies( + dependencies, + dep => + res + .projectCache + .get(dep) + .map(_._2.configurations) + .getOrElse(Map.empty) + ) + + val includedConfigs = configs.getOrElse(config, Set.empty) + config + + minDependencies( + currentProject + .dependencies + .collect { + case (cfg, dep) if includedConfigs(cfg) => + dep + } + .toSet + ) + } + + val dependencyArtifacts = res.dependencyArtifacts.toMap + + val artifactFilesOrErrors0 = artifactFilesOrErrors + .collect { + case (a, \/-(f)) => a.url -> f + } + + val compileDeps = configDependencies(baseConfig) + val shadedDeps = configDependencies(shadedConf) + + val compileOnlyDeps = compileDeps.filterNot(shadedDeps) + + log.info(s"Found ${compileDeps.size} dependencies in $baseConfig") + log.debug(compileDeps.toVector.map(" " + _).sorted.mkString("\n")) + log.info(s"Found ${compileOnlyDeps.size} dependencies only in $baseConfig") + log.debug(compileOnlyDeps.toVector.map(" " + _).sorted.mkString("\n")) + log.info(s"Found ${shadedDeps.size} dependencies in $shadedConf") + log.debug(shadedDeps.toVector.map(" " + _).sorted.mkString("\n")) + + def files(deps: Set[Dependency]) = res + .subset(deps) + .dependencies + .toSeq + .flatMap(dependencyArtifacts.get) + .map(_.url) + .flatMap(artifactFilesOrErrors0.get) + + val compileOnlyJars = files(compileOnlyDeps) + val shadedJars = files(shadedDeps) + + log.info(s"Found ${compileOnlyJars.length} JAR(s) only in $baseConfig") + log.debug(compileOnlyJars.map(" " + _).sorted.mkString("\n")) + log.info(s"Found ${shadedJars.length} JAR(s) in $shadedConf") + log.debug(shadedJars.map(" " + _).sorted.mkString("\n")) + + val shadeJars = shadedJars.filterNot(compileOnlyJars.toSet) + val shadeClasses = shadeJars.flatMap(Shading.jarClassNames) + + log.info(s"Will shade ${shadeClasses.length} class(es)") + log.debug(shadeClasses.map(" " + _).sorted.mkString("\n")) + + val processor = new DefaultJarProcessor + for (cls <- shadeClasses) + processor.addClassRename(new ClassRename(cls, shadingNamespace + ".@0")) + + val transformer = new JarTransformer(outputJar, processor) + val cp = new ClassPath(file(sys.props("user.dir")), (baseJar +: shadeJars).toArray) + transformer.transform(cp) + + outputJar + } + +} \ No newline at end of file diff --git a/sbt-shading/src/main/scala-2.10/coursier/ShadingPlugin.scala b/sbt-shading/src/main/scala-2.10/coursier/ShadingPlugin.scala new file mode 100644 index 000000000..510fcdaf9 --- /dev/null +++ b/sbt-shading/src/main/scala-2.10/coursier/ShadingPlugin.scala @@ -0,0 +1,97 @@ +package coursier + +import coursier.ivy.IvyXml +import sbt.Keys._ +import sbt.{AutoPlugin, Compile, Configuration, TaskKey, inConfig} + +object ShadingPlugin extends AutoPlugin { + + override def trigger = noTrigger + + override def requires = sbt.plugins.IvyPlugin + + private val baseSbtConfiguration = Compile + val Shading = Configuration("shading", "", isPublic = false, List(baseSbtConfiguration), transitive = true) + + private val baseDependencyConfiguration = "compile" + val Shaded = Configuration("shaded", "", isPublic = true, List(), transitive = true) + + val shadingNamespace = TaskKey[String]("shading-namespace") + + object autoImport { + + /** Scope for shading related tasks */ + val Shading = ShadingPlugin.Shading + + /** Ivy configuration for shaded dependencies */ + val Shaded = ShadingPlugin.Shaded + + val shadingNamespace = ShadingPlugin.shadingNamespace + } + + // same as similar things under sbt.Classpaths, tweaking a bit the configuration scope + lazy val shadingDefaultArtifactTasks = + makePom +: Seq(packageBin, packageSrc, packageDoc).map(_.in(Shading)) + lazy val shadingJvmPublishSettings = Seq( + artifacts <<= sbt.Classpaths.artifactDefs(shadingDefaultArtifactTasks), + packagedArtifacts <<= sbt.Classpaths.packaged(shadingDefaultArtifactTasks) + ) + + import CoursierPlugin.autoImport._ + + override lazy val projectSettings = + Seq( + coursierConfigurations <<= Tasks.coursierConfigurationsTask( + Some(baseDependencyConfiguration -> Shaded.name) + ), + ivyConfigurations := Shaded +: ivyConfigurations.value.map { + conf => + if (conf.name == "compile") + conf.extend(Shaded) + else + conf + } + ) ++ + inConfig(Shading)( + sbt.Defaults.configSettings ++ + sbt.Classpaths.ivyBaseSettings ++ + sbt.Classpaths.ivyPublishSettings ++ + shadingJvmPublishSettings ++ + CoursierPlugin.coursierSettings( + Some(baseDependencyConfiguration -> Shaded.name), + Seq(Shading -> Compile.name) + ) ++ + CoursierPlugin.treeSettings ++ + Seq( + configuration := baseSbtConfiguration, // wuw + ivyConfigurations := ivyConfigurations.in(baseSbtConfiguration).value + .filter(_.name != Shaded.name) + .map(c => c.copy(extendsConfigs = c.extendsConfigs.filter(_.name != Shaded.name))), + libraryDependencies := libraryDependencies.in(baseSbtConfiguration).value.filter { dep => + val isShaded = dep.configurations.exists { mappings => + IvyXml.mappings(mappings).exists(_._1 == Shaded.name) + } + + !isShaded + }, + // required for cross-projects in particular + unmanagedSourceDirectories := (unmanagedSourceDirectories in Compile).value, + packageBin := { + coursier.Shading.createPackage( + packageBin.in(baseSbtConfiguration).value, + coursierProject.in(baseSbtConfiguration).value, + coursierResolution.in(baseSbtConfiguration).value, + coursierConfigurations.in(baseSbtConfiguration).value, + Keys.coursierArtifacts.in(baseSbtConfiguration).value, + shadingNamespace.?.value.getOrElse { + throw new NoSuchElementException("shadingNamespace key not set") + }, + baseDependencyConfiguration, + Shaded.name, + streams.value.log + ) + } + ) + ) + +} \ No newline at end of file diff --git a/sbt-shading/src/sbt-test/sbt-shading/cross-project-shading/build.sbt b/sbt-shading/src/sbt-test/sbt-shading/cross-project-shading/build.sbt new file mode 100644 index 000000000..0144a1ce2 --- /dev/null +++ b/sbt-shading/src/sbt-test/sbt-shading/cross-project-shading/build.sbt @@ -0,0 +1,20 @@ + +lazy val root = crossProject + .in(file(".")) + .jvmConfigure( + _.enablePlugins(coursier.ShadingPlugin) + ) + .jvmSettings( + shadingNamespace := "test.shaded", + libraryDependencies += "io.argonaut" %% "argonaut" % "6.2-RC2" % "shaded" + ) + .settings( + scalaVersion := "2.11.8", + organization := "io.get-coursier.test", + name := "shading-cross-test", + version := "0.1.0-SNAPSHOT", + libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value + ) + +lazy val jvm = root.jvm +lazy val js = root.js diff --git a/sbt-shading/src/sbt-test/sbt-shading/cross-project-shading/coursier b/sbt-shading/src/sbt-test/sbt-shading/cross-project-shading/coursier new file mode 100755 index 000000000..13c8a2b55 Binary files /dev/null and b/sbt-shading/src/sbt-test/sbt-shading/cross-project-shading/coursier differ diff --git a/sbt-shading/src/sbt-test/sbt-shading/cross-project-shading/jvm/src/main/scala/Main.scala b/sbt-shading/src/sbt-test/sbt-shading/cross-project-shading/jvm/src/main/scala/Main.scala new file mode 100644 index 000000000..30502dda1 --- /dev/null +++ b/sbt-shading/src/sbt-test/sbt-shading/cross-project-shading/jvm/src/main/scala/Main.scala @@ -0,0 +1,21 @@ +import java.io.File +import java.nio.file.Files + +import argonaut._ + +import Foo._ + +object Main extends App { + + val expectedClassName0 = expectedClassName(args.headOption == Some("--shaded")) + + Console.err.println(s"Expected class name: $expectedClassName0") + Console.err.println(s"Class name: $className") + + if (className != expectedClassName0) + sys.error(s"Expected class name $expectedClassName0, got $className") + + val msg = Json.obj().nospaces + + Files.write(new File("output").toPath, msg.getBytes("UTF-8")) +} diff --git a/sbt-shading/src/sbt-test/sbt-shading/cross-project-shading/project/plugins.sbt b/sbt-shading/src/sbt-test/sbt-shading/cross-project-shading/project/plugins.sbt new file mode 100644 index 000000000..fc3a48bf6 --- /dev/null +++ b/sbt-shading/src/sbt-test/sbt-shading/cross-project-shading/project/plugins.sbt @@ -0,0 +1,33 @@ +{ + val pluginVersion = sys.props.getOrElse( + "plugin.version", + throw new RuntimeException( + """|The system property 'plugin.version' is not defined. + |Specify this property using the scriptedLaunchOpts -D.""".stripMargin + ) + ) + + addSbtPlugin("io.get-coursier" % "sbt-shading" % pluginVersion) +} + +// for the locally publish jarjar +resolvers += Resolver.mavenLocal + +val coursierJarjarVersion = "1.0.1-coursier-SNAPSHOT" + +def coursierJarjarFoundInM2 = + (file(sys.props("user.home")) / s".m2/repository/org/anarres/jarjar/jarjar-core/$coursierJarjarVersion").exists() + +def jarjarVersion = + if (coursierJarjarFoundInM2) + coursierJarjarVersion + else + sys.error( + "Ad hoc jarjar version not found. Run\n" + + " git clone https://github.com/alexarchambault/jarjar.git && cd jarjar && git checkout 249c8dbb970f8 && ./gradlew install\n" + + "to run this test" + ) + +libraryDependencies += "org.anarres.jarjar" % "jarjar-core" % jarjarVersion + +addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.13") diff --git a/sbt-shading/src/sbt-test/sbt-shading/cross-project-shading/shared/src/main/scala/Foo.scala b/sbt-shading/src/sbt-test/sbt-shading/cross-project-shading/shared/src/main/scala/Foo.scala new file mode 100644 index 000000000..6e510b152 --- /dev/null +++ b/sbt-shading/src/sbt-test/sbt-shading/cross-project-shading/shared/src/main/scala/Foo.scala @@ -0,0 +1,16 @@ + +import argonaut._ + +object Foo { + + def expectedClassName(shaded: Boolean) = + if (shaded) + "test.shaded.argonaut.Json" + else + // Don't use the literal "argonaut.Json", that seems to get + // changed to "test.shaded.argonaut.Json" by shading + "argonaut" + '.' + "Json" + + val className = classOf[Json].getName + +} diff --git a/sbt-shading/src/sbt-test/sbt-shading/cross-project-shading/test b/sbt-shading/src/sbt-test/sbt-shading/cross-project-shading/test new file mode 100644 index 000000000..72419bd1d --- /dev/null +++ b/sbt-shading/src/sbt-test/sbt-shading/cross-project-shading/test @@ -0,0 +1,10 @@ +$ delete output +> rootJVM/run +$ exists output +$ delete output +> rootJVM/publishLocal +$ exec java -jar coursier launch io.get-coursier.test:shading-cross-test_2.11:0.1.0-SNAPSHOT +-$ exec java -jar coursier launch io.get-coursier.test:shading-cross-test_2.11:0.1.0-SNAPSHOT -- --shaded +> rootJVM/shading:publishLocal +-$ exec java -jar coursier launch io.get-coursier.test:shading-cross-test_2.11:0.1.0-SNAPSHOT +$ exec java -jar coursier launch io.get-coursier.test:shading-cross-test_2.11:0.1.0-SNAPSHOT -- --shaded diff --git a/sbt-shading/src/sbt-test/sbt-shading/shading/build.sbt b/sbt-shading/src/sbt-test/sbt-shading/shading/build.sbt new file mode 100644 index 000000000..c125c822d --- /dev/null +++ b/sbt-shading/src/sbt-test/sbt-shading/shading/build.sbt @@ -0,0 +1,10 @@ + +enablePlugins(coursier.ShadingPlugin) +shadingNamespace := "test.shaded" + +libraryDependencies += "io.argonaut" %% "argonaut" % "6.2-RC2" % "shaded" + +scalaVersion := "2.11.8" +organization := "io.get-coursier.test" +name := "shading-base-test" +version := "0.1.0-SNAPSHOT" diff --git a/sbt-shading/src/sbt-test/sbt-shading/shading/coursier b/sbt-shading/src/sbt-test/sbt-shading/shading/coursier new file mode 100755 index 000000000..13c8a2b55 Binary files /dev/null and b/sbt-shading/src/sbt-test/sbt-shading/shading/coursier differ diff --git a/sbt-shading/src/sbt-test/sbt-shading/shading/project/plugins.sbt b/sbt-shading/src/sbt-test/sbt-shading/shading/project/plugins.sbt new file mode 100644 index 000000000..8f83e814d --- /dev/null +++ b/sbt-shading/src/sbt-test/sbt-shading/shading/project/plugins.sbt @@ -0,0 +1,31 @@ +{ + val pluginVersion = sys.props.getOrElse( + "plugin.version", + throw new RuntimeException( + """|The system property 'plugin.version' is not defined. + |Specify this property using the scriptedLaunchOpts -D.""".stripMargin + ) + ) + + addSbtPlugin("io.get-coursier" % "sbt-shading" % pluginVersion) +} + +// for the locally publish jarjar +resolvers += Resolver.mavenLocal + +val coursierJarjarVersion = "1.0.1-coursier-SNAPSHOT" + +def coursierJarjarFoundInM2 = + (file(sys.props("user.home")) / s".m2/repository/org/anarres/jarjar/jarjar-core/$coursierJarjarVersion").exists() + +def jarjarVersion = + if (coursierJarjarFoundInM2) + coursierJarjarVersion + else + sys.error( + "Ad hoc jarjar version not found. Run\n" + + " git clone https://github.com/alexarchambault/jarjar.git && cd jarjar && git checkout 249c8dbb970f8 && ./gradlew install\n" + + "to run this test" + ) + +libraryDependencies += "org.anarres.jarjar" % "jarjar-core" % jarjarVersion diff --git a/sbt-shading/src/sbt-test/sbt-shading/shading/src/main/scala/Main.scala b/sbt-shading/src/sbt-test/sbt-shading/shading/src/main/scala/Main.scala new file mode 100644 index 000000000..b7eee31a8 --- /dev/null +++ b/sbt-shading/src/sbt-test/sbt-shading/shading/src/main/scala/Main.scala @@ -0,0 +1,27 @@ +import java.io.File +import java.nio.file.Files + +import argonaut._ + +object Main extends App { + + val expectedClassName = + if (args.headOption == Some("--shaded")) + "test.shaded.argonaut.Json" + else + // Don't use the literal "argonaut.Json", that seems to get + // changed to "test.shaded.argonaut.Json" by shading + "argonaut" + '.' + "Json" + + val className = classOf[Json].getName + + Console.err.println(s"Expected class name: $expectedClassName") + Console.err.println(s"Class name: $className") + + if (className != expectedClassName) + sys.error(s"Expected class name $expectedClassName, got $className") + + val msg = Json.obj().nospaces + + Files.write(new File("output").toPath, msg.getBytes("UTF-8")) +} diff --git a/sbt-shading/src/sbt-test/sbt-shading/shading/test b/sbt-shading/src/sbt-test/sbt-shading/shading/test new file mode 100644 index 000000000..6f020327e --- /dev/null +++ b/sbt-shading/src/sbt-test/sbt-shading/shading/test @@ -0,0 +1,9 @@ +$ delete output +> run +$ exists output +> publishLocal +$ exec java -jar coursier launch io.get-coursier.test:shading-base-test_2.11:0.1.0-SNAPSHOT +-$ exec java -jar coursier launch io.get-coursier.test:shading-base-test_2.11:0.1.0-SNAPSHOT -- --shaded +> shading:publishLocal +-$ exec java -jar coursier launch io.get-coursier.test:shading-base-test_2.11:0.1.0-SNAPSHOT +$ exec java -jar coursier launch io.get-coursier.test:shading-base-test_2.11:0.1.0-SNAPSHOT -- --shaded diff --git a/sbt-shading/src/sbt-test/sbt-shading/transitive-shading/build.sbt b/sbt-shading/src/sbt-test/sbt-shading/transitive-shading/build.sbt new file mode 100644 index 000000000..3f4280d74 --- /dev/null +++ b/sbt-shading/src/sbt-test/sbt-shading/transitive-shading/build.sbt @@ -0,0 +1,14 @@ + +enablePlugins(coursier.ShadingPlugin) +shadingNamespace := "test.shaded" + +libraryDependencies ++= Seq( + "com.github.alexarchambault" %% "argonaut-shapeless_6.2" % "1.2.0-M4" % "shaded", + "com.chuusai" %% "shapeless" % "2.3.2", + "org.scala-lang" % "scala-reflect" % scalaVersion.value +) + +scalaVersion := "2.11.8" +organization := "io.get-coursier.test" +name := "shading-transitive-test" +version := "0.1.0-SNAPSHOT" diff --git a/sbt-shading/src/sbt-test/sbt-shading/transitive-shading/coursier b/sbt-shading/src/sbt-test/sbt-shading/transitive-shading/coursier new file mode 100755 index 000000000..13c8a2b55 Binary files /dev/null and b/sbt-shading/src/sbt-test/sbt-shading/transitive-shading/coursier differ diff --git a/sbt-shading/src/sbt-test/sbt-shading/transitive-shading/project/plugins.sbt b/sbt-shading/src/sbt-test/sbt-shading/transitive-shading/project/plugins.sbt new file mode 100644 index 000000000..8f83e814d --- /dev/null +++ b/sbt-shading/src/sbt-test/sbt-shading/transitive-shading/project/plugins.sbt @@ -0,0 +1,31 @@ +{ + val pluginVersion = sys.props.getOrElse( + "plugin.version", + throw new RuntimeException( + """|The system property 'plugin.version' is not defined. + |Specify this property using the scriptedLaunchOpts -D.""".stripMargin + ) + ) + + addSbtPlugin("io.get-coursier" % "sbt-shading" % pluginVersion) +} + +// for the locally publish jarjar +resolvers += Resolver.mavenLocal + +val coursierJarjarVersion = "1.0.1-coursier-SNAPSHOT" + +def coursierJarjarFoundInM2 = + (file(sys.props("user.home")) / s".m2/repository/org/anarres/jarjar/jarjar-core/$coursierJarjarVersion").exists() + +def jarjarVersion = + if (coursierJarjarFoundInM2) + coursierJarjarVersion + else + sys.error( + "Ad hoc jarjar version not found. Run\n" + + " git clone https://github.com/alexarchambault/jarjar.git && cd jarjar && git checkout 249c8dbb970f8 && ./gradlew install\n" + + "to run this test" + ) + +libraryDependencies += "org.anarres.jarjar" % "jarjar-core" % jarjarVersion diff --git a/sbt-shading/src/sbt-test/sbt-shading/transitive-shading/src/main/scala/Main.scala b/sbt-shading/src/sbt-test/sbt-shading/transitive-shading/src/main/scala/Main.scala new file mode 100644 index 000000000..b7eee31a8 --- /dev/null +++ b/sbt-shading/src/sbt-test/sbt-shading/transitive-shading/src/main/scala/Main.scala @@ -0,0 +1,27 @@ +import java.io.File +import java.nio.file.Files + +import argonaut._ + +object Main extends App { + + val expectedClassName = + if (args.headOption == Some("--shaded")) + "test.shaded.argonaut.Json" + else + // Don't use the literal "argonaut.Json", that seems to get + // changed to "test.shaded.argonaut.Json" by shading + "argonaut" + '.' + "Json" + + val className = classOf[Json].getName + + Console.err.println(s"Expected class name: $expectedClassName") + Console.err.println(s"Class name: $className") + + if (className != expectedClassName) + sys.error(s"Expected class name $expectedClassName, got $className") + + val msg = Json.obj().nospaces + + Files.write(new File("output").toPath, msg.getBytes("UTF-8")) +} diff --git a/sbt-shading/src/sbt-test/sbt-shading/transitive-shading/test b/sbt-shading/src/sbt-test/sbt-shading/transitive-shading/test new file mode 100644 index 000000000..84b72a875 --- /dev/null +++ b/sbt-shading/src/sbt-test/sbt-shading/transitive-shading/test @@ -0,0 +1,9 @@ +$ delete output +> run +$ exists output +> publishLocal +$ exec java -jar coursier launch io.get-coursier.test:shading-transitive-test_2.11:0.1.0-SNAPSHOT +-$ exec java -jar coursier launch io.get-coursier.test:shading-transitive-test_2.11:0.1.0-SNAPSHOT -- --shaded +> shading:publishLocal +-$ exec java -jar coursier launch io.get-coursier.test:shading-transitive-test_2.11:0.1.0-SNAPSHOT +$ exec java -jar coursier launch io.get-coursier.test:shading-transitive-test_2.11:0.1.0-SNAPSHOT -- --shaded