Implement static launcher for sbt

This is a combination of 13 commits. I squashed these 13 commits to make
forward porting those changes easier, since some commit undo the changes
of other commits.

The PRs that include these changes can be found at
https://github.com/sbt/sbt/pull/2564 and
https://github.com/sbt/sbt/pull/2576.

Static launcher, get bridge sources from resources

This commit introduces a new "static" launcher that does not use Ivy to
gather all the artifacts that it requires, but rather expect them to be
immediately available.

To be able to use sbt without Internet access, we add a new
`ComponentCompiler` that is able to retrieve the bridge sources from the
resources on classpath and compile it.

Fix classpath issues in static launcher

The launcher defines a top classloader that willbe used by all
`ScalaInstance`s. Previously, this top classloader had a parent that
contained the scala library 2.10, which prevented the correct
compilation of the compiler bridge for scala 2.11.

Also, we no longer need the scala-reflect JAR.

Tests for FakeResolver

Add `scala-reflect.jar` to JARs of `StaticScalaProvider`

It turns out we need to have `scala-reflect.jar` on classpath to compile
the compiler bridge for the static scala instance of the launcher.

Comply to Ivy's specification in `FakeResolver`

Remove `CompilerBridgeProvider` and `ResourceBridgeProvider`

It turns out that we can leverage the`FakeResolver` that has been
implemented to use with the static launcher, and resolve a "fake
compiler bridge" using it, rather than copying it from the resources.

This also has the advantage of not requiring to change the build
definition.

Fix NPE in FakeResolver

Add compiler bridge sources to fake resolver

This allows sbt to resolve the compiler bridge sources when using the
static launcher

Don't hardcode sbt version in static launcher

Add scala compiler and library to fake resolver

This allows us to still resolve them if we have no other resolver
configured.

Add `RepositoriesParser`

This parser is used by the static launcher to parse the definition of
resolvers that override the build resolvers.

Support repositories override in static launcher

The static launcher will now parse user-defined repositories like the
usual launcher does.

Specifically, the static launcher now uses the following configuration:

 - `sbt.boot.directory`: specifies the boot directory that sbt will use.
   Defaults to `~/.sbt/boot`.
 - `sbt.override.build.repos`: indicate whether we want to override the
   build resolvers. Defaults to false.
 - `sbt.repository.config`: specifies the path to the files that
   contains repositories definition. Defaults to
   `${sbt.boot.directory}/repositories`.

Notes for sbt/sbt#2564 & sbt/sbt#2576
This commit is contained in:
Martin Duhem 2016-04-06 11:52:51 +02:00
parent 596354beae
commit 65adc869d4
9 changed files with 514 additions and 1 deletions

View File

@ -62,7 +62,7 @@ lazy val lm = (project in file("librarymanagement")).
utilLogging, (utilLogging % Test).classifier("tests"),
sbtIO, (sbtIO % Test).classifier("tests"),
utilTesting % Test,
utilCollection, ivy, jsch, sbtSerialization, scalaReflect.value, launcherInterface),
utilCollection, utilCompletion, ivy, jsch, sbtSerialization, scalaReflect.value, launcherInterface),
resourceGenerators in Compile <+= (version, resourceManaged, streams, compile in Compile) map Util.generateVersionFile,
name := "librarymanagement",
binaryIssueFilters ++= Seq(

View File

@ -0,0 +1,194 @@
package sbt
import java.io.File
import java.net.URL
import org.apache.ivy.core.cache.ArtifactOrigin
import org.apache.ivy.core.cache.{ DefaultRepositoryCacheManager, RepositoryCacheManager }
import org.apache.ivy.core.module.descriptor.{ Artifact => IvyArtifact, DefaultArtifact, DefaultDependencyArtifactDescriptor, DefaultModuleDescriptor, DependencyArtifactDescriptor, DependencyDescriptor }
import org.apache.ivy.core.module.id.ModuleRevisionId
import org.apache.ivy.core.report.ArtifactDownloadReport
import org.apache.ivy.core.report.{ DownloadReport, DownloadStatus }
import org.apache.ivy.core.report.MetadataArtifactDownloadReport
import org.apache.ivy.core.resolve.{ DownloadOptions, ResolveData, ResolvedModuleRevision }
import org.apache.ivy.core.search.{ ModuleEntry, OrganisationEntry, RevisionEntry }
import org.apache.ivy.core.settings.IvySettings
import org.apache.ivy.plugins.namespace.Namespace
import org.apache.ivy.plugins.repository.url.URLResource
import org.apache.ivy.plugins.resolver.{ DependencyResolver, ResolverSettings }
import org.apache.ivy.plugins.resolver.util.ResolvedResource
import FakeResolver._
/**
* A fake `DependencyResolver` that statically serves predefined artifacts.
*/
private[sbt] class FakeResolver(private var name: String, cacheDir: File, modules: ModulesMap) extends DependencyResolver {
private object Artifact {
def unapply(art: IvyArtifact): Some[(String, String, String)] = {
val revisionID = art.getModuleRevisionId()
val organisation = revisionID.getOrganisation
val name = revisionID.getName
val revision = revisionID.getRevision
Some((organisation, name, revision))
}
def unapply(dd: DependencyDescriptor): Some[(String, String, String)] = {
val module = dd.getDependencyId()
val organisation = module.getOrganisation
val name = module.getName
val mrid = dd.getDependencyRevisionId()
val revision = mrid.getRevision()
Some((organisation, name, revision))
}
}
override def publish(artifact: IvyArtifact, src: File, overwrite: Boolean): Unit =
throw new UnsupportedOperationException("This resolver doesn't support publishing.")
override def abortPublishTransaction(): Unit =
throw new UnsupportedOperationException("This resolver doesn't support publishing.")
override def beginPublishTransaction(module: ModuleRevisionId, overwrite: Boolean): Unit =
throw new UnsupportedOperationException("This resolver doesn't support publishing.")
override def commitPublishTransaction(): Unit =
throw new UnsupportedOperationException("This resolver doesn't support publishing.")
override def download(artifact: ArtifactOrigin, options: DownloadOptions): ArtifactDownloadReport = {
val report = new ArtifactDownloadReport(artifact.getArtifact)
val path = new URL(artifact.getLocation).toURI.getPath
val localFile = new File(path)
if (path.nonEmpty && localFile.exists) {
report.setLocalFile(localFile)
report.setDownloadStatus(DownloadStatus.SUCCESSFUL)
report.setSize(localFile.length)
} else {
report.setDownloadStatus(DownloadStatus.FAILED)
}
report
}
override def download(artifacts: Array[IvyArtifact], options: DownloadOptions): DownloadReport = {
val report = new DownloadReport
artifacts foreach { art =>
val artifactOrigin = locate(art)
Option(locate(art)) foreach (o => report.addArtifactReport(download(o, options)))
}
report
}
override def dumpSettings(): Unit = ()
override def exists(artifact: IvyArtifact): Boolean = {
val Artifact(organisation, name, revision) = artifact
modules.get((organisation, name, revision)).isDefined
}
// This is a fake resolver and we don't have Ivy files. Ivy's spec says we can return `null` if
// we can't find the module descriptor.
override def findIvyFileRef(dd: DependencyDescriptor, data: ResolveData): ResolvedResource = null
override def getDependency(dd: DependencyDescriptor, data: ResolveData): ResolvedModuleRevision = {
val Artifact(organisation, name, revision) = dd
val mrid = dd.getDependencyRevisionId()
val artifact = modules get ((organisation, name, revision)) map { arts =>
val artifacts: Array[DependencyArtifactDescriptor] = arts.toArray map (_ artifactOf dd)
val moduleDescriptor = DefaultModuleDescriptor.newDefaultInstance(mrid, artifacts)
val defaultArtifact = arts.headOption match {
case Some(FakeArtifact(name, tpe, ext, _)) => new DefaultArtifact(mrid, new java.util.Date, name, tpe, ext)
case None => null
}
val metadataReport = new MetadataArtifactDownloadReport(defaultArtifact)
metadataReport.setDownloadStatus(DownloadStatus.SUCCESSFUL)
new ResolvedModuleRevision(this, this, moduleDescriptor, metadataReport)
}
artifact.orNull
}
override def getName(): String = name
override val getNamespace: Namespace = {
val ns = new Namespace()
ns.setName(name)
ns
}
override val getRepositoryCacheManager: RepositoryCacheManager = {
val cacheName = name + "-cache"
val ivySettings = new IvySettings()
val baseDir = cacheDir
new DefaultRepositoryCacheManager(cacheName, ivySettings, baseDir)
}
override def listModules(organisation: OrganisationEntry): Array[ModuleEntry] =
modules.keys.collect {
case (o, m, _) if o == organisation.getOrganisation =>
val organisationEntry = new OrganisationEntry(this, o)
new ModuleEntry(organisationEntry, m)
}.toArray
override def listOrganisations(): Array[OrganisationEntry] =
modules.keys.map { case (o, _, _) => new OrganisationEntry(this, o) }.toArray
override def listRevisions(module: ModuleEntry): Array[RevisionEntry] =
modules.keys.collect {
case (o, m, v) if o == module.getOrganisation && m == module.getModule =>
new RevisionEntry(module, v)
}.toArray
override def listTokenValues(tokens: Array[String], criteria: java.util.Map[_, _]): Array[java.util.Map[_, _]] =
Array.empty
override def listTokenValues(token: String, otherTokenValues: java.util.Map[_, _]): Array[String] =
Array.empty
override def locate(art: IvyArtifact): ArtifactOrigin = {
val Artifact(moduleOrganisation, moduleName, moduleRevision) = art
val artifact =
for {
artifacts <- modules get ((moduleOrganisation, moduleName, moduleRevision))
artifact <- artifacts find (a => a.name == art.getName && a.tpe == art.getType && a.ext == art.getExt)
} yield new ArtifactOrigin(art, /* isLocal = */ true, artifact.file.toURI.toURL.toString)
artifact.orNull
}
override def reportFailure(art: IvyArtifact): Unit = ()
override def reportFailure(): Unit = ()
override def setName(name: String): Unit = {
this.name = name
getNamespace.setName(name)
}
override def setSettings(settings: ResolverSettings): Unit = ()
private class LocalURLResource(jar: File) extends URLResource(jar.toURI.toURL) {
override def isLocal(): Boolean = true
}
}
private[sbt] object FakeResolver {
type ModulesMap = Map[(String, String, String), Seq[FakeArtifact]]
final case class FakeArtifact(name: String, tpe: String, ext: String, file: File) {
def artifactOf(dd: DependencyDescriptor): DependencyArtifactDescriptor =
new DefaultDependencyArtifactDescriptor(dd, name, tpe, ext, file.toURI.toURL, new java.util.HashMap)
}
}

View File

@ -0,0 +1,91 @@
package sbt
package internal
package librarymanagement
import java.io.File
import java.net.URL
import scala.io.Source
import sbt.internal.util.complete.Parser
import sbt.internal.util.complete.DefaultParsers._
private[sbt] object RepositoriesParser {
private case class AfterPattern(artifactPattern: Option[String], flags: Int)
final case class PredefinedRepository(override val id: xsbti.Predefined) extends xsbti.PredefinedRepository
final case class MavenRepository(override val id: String, override val url: URL) extends xsbti.MavenRepository
final case class IvyRepository(override val id: String, override val url: URL, override val ivyPattern: String,
override val artifactPattern: String, override val mavenCompatible: Boolean, override val skipConsistencyCheck: Boolean,
override val descriptorOptional: Boolean, val bootOnly: Boolean) extends xsbti.IvyRepository
// Predefined repositories
def local: Parser[xsbti.Repository] = "local" ^^^ new PredefinedRepository(xsbti.Predefined.Local)
def mavenLocal: Parser[xsbti.Repository] = "maven-local" ^^^ new PredefinedRepository(xsbti.Predefined.MavenLocal)
def mavenCentral: Parser[xsbti.Repository] = "maven-central" ^^^ new PredefinedRepository(xsbti.Predefined.MavenCentral)
def predefinedResolver: Parser[xsbti.Repository] = local | mavenLocal | mavenCentral
// Options
def descriptorOptional: Parser[Int] = "descriptorOptional" ^^^ Flags.descriptorOptionalFlag
def skipConsistencyCheck: Parser[Int] = "skipConsistencyCheck" ^^^ Flags.skipConsistencyCheckFlag
def bootOnly: Parser[Int] = "bootOnly" ^^^ Flags.bootOnlyFlag
def mavenCompatible: Parser[Int] = "mavenCompatible" ^^^ Flags.mavenCompatibleFlag
def option: Parser[Int] = descriptorOptional | skipConsistencyCheck | bootOnly | mavenCompatible
def options: Parser[Int] = rep1sep(option, separator) map (_ reduce (_ | _))
def name: Parser[String] = ID
def separator: Parser[String] = "," ~> charClass(c => c == ' ' || c == '\t').*.string
def nonComma: Parser[String] = charClass(_ != ',').*.string
def ivyPattern: Parser[String] = nonComma
def artifactPattern: Parser[String] = nonComma
private def afterPattern: Parser[AfterPattern] = {
def onlyOptions = options map (AfterPattern(None, _))
def both = artifactPattern ~ (separator ~> options).? map {
case ap ~ opts => AfterPattern(Some(ap), opts getOrElse 0)
}
onlyOptions | both
}
def customResolver: Parser[xsbti.Repository] =
name ~ ": " ~ basicUri ~ (separator ~> ivyPattern).? ~ (separator ~> afterPattern).? map {
case name ~ ": " ~ uri ~ None ~ _ =>
new MavenRepository(name, uri.toURL)
case name ~ ": " ~ uri ~ Some(ivy) ~ None =>
new IvyRepository(name, uri.toURL, ivy, ivy, false, false, false, false)
case name ~ ": " ~ uri ~ Some(ivy) ~ Some(AfterPattern(artifactPattern, Flags(dOpt, sc, bo, mc))) =>
new IvyRepository(name, uri.toURL, ivy, artifactPattern getOrElse ivy, mc, sc, dOpt, bo)
}
def resolver: Parser[xsbti.Repository] =
predefinedResolver | customResolver
def getResolver[T](in: String)(parser: Parser[T]): Option[T] =
Parser.parse(in.trim, parser).right.toOption
def apply(lines: Iterator[String]): Seq[xsbti.Repository] =
if (lines.isEmpty) Nil
else {
if (lines.next != "[repositories]") throw new Exception("Repositories file must start with '[repositories]'")
lines.flatMap(getResolver(_)(resolver)).toList
}
def apply(str: String): Seq[xsbti.Repository] = apply(str.lines)
def apply(file: File): Seq[xsbti.Repository] = {
if (!file.exists) Nil
else apply(Source.fromFile(file).getLines)
}
object Flags {
val descriptorOptionalFlag = 1 << 0
val skipConsistencyCheckFlag = 1 << 1
val bootOnlyFlag = 1 << 2
val mavenCompatibleFlag = 1 << 3
def unapply(flags: Int): Some[(Boolean, Boolean, Boolean, Boolean)] = {
val dOpt = (flags & descriptorOptionalFlag) != 0
val sc = (flags & skipConsistencyCheckFlag) != 0
val bo = (flags & bootOnlyFlag) != 0
val mc = (flags & mavenCompatibleFlag) != 0
Some((dOpt, sc, bo, mc))
}
}
}

View File

@ -159,6 +159,10 @@ final case class SftpRepository(name: String, connection: SshConnection, pattern
protected def copy(patterns: Patterns): SftpRepository = SftpRepository(name, connection, patterns)
protected def copy(connection: SshConnection): SftpRepository = SftpRepository(name, connection, patterns)
}
/** A repository that conforms to sbt launcher's interface */
private[sbt] class FakeRepository(resolver: DependencyResolver) extends xsbti.Repository {
def rawRepository = new RawRepository(resolver)
}
import Resolver._

Binary file not shown.

View File

@ -0,0 +1,78 @@
package sbt
package internal
package librarymanagement
import java.io.File
import sbt.librarymanagement.{ ModuleID, RawRepository, Resolver, UpdateReport }
class FakeResolverSpecification extends BaseIvySpecification {
import FakeResolver._
val myModule = ModuleID("org.example", "my-module", "0.0.1-SNAPSHOT", Some("compile"))
val example = ModuleID("com.example", "example", "1.0.0", Some("compile"))
val anotherExample = ModuleID("com.example", "another-example", "1.0.0", Some("compile"))
val nonExisting = ModuleID("com.example", "does-not-exist", "1.2.3", Some("compile"))
"The FakeResolver" should "find modules with only one artifact" in {
val m = getModule(myModule)
val report = ivyUpdate(m)
val allFiles = getAllFiles(report)
report.allModules.length shouldBe 1
report.configurations.length shouldBe 3
allFiles.toSet.size shouldBe 1
allFiles(1).getName shouldBe "artifact1-0.0.1-SNAPSHOT.jar"
}
it should "find modules with more than one artifact" in {
val m = getModule(example)
val report = ivyUpdate(m)
val allFiles = getAllFiles(report).toSet
report.allModules.length shouldBe 1
report.configurations.length shouldBe 3
allFiles.toSet.size shouldBe 2
allFiles map (_.getName) shouldBe Set("artifact1-1.0.0.jar", "artifact2-1.0.0.txt")
}
it should "fail gracefully when asked for unknown modules" in {
val m = getModule(nonExisting)
a[ResolveException] should be thrownBy ivyUpdate(m)
}
it should "fail gracefully when some artifacts cannot be found" in {
val m = getModule(anotherExample)
the[ResolveException] thrownBy ivyUpdate(m) should have message "download failed: com.example#another-example;1.0.0!non-existing.txt"
}
private def artifact1 = new File(getClass.getResource("/artifact1.jar").toURI.getPath)
private def artifact2 = new File(getClass.getResource("/artifact2.txt").toURI.getPath)
private def modules = Map(
("org.example", "my-module", "0.0.1-SNAPSHOT") -> List(
FakeArtifact("artifact1", "jar", "jar", artifact1)
),
("com.example", "example", "1.0.0") -> List(
FakeArtifact("artifact1", "jar", "jar", artifact1),
FakeArtifact("artifact2", "txt", "txt", artifact2)
),
("com.example", "another-example", "1.0.0") -> List(
FakeArtifact("artifact1", "jar", "jar", artifact1),
FakeArtifact("non-existing", "txt", "txt", new File("non-existing-file"))
)
)
private def fakeResolver = new FakeResolver("FakeResolver", new File("tmp"), modules)
override def resolvers: Seq[Resolver] = Seq(new RawRepository(fakeResolver))
private def getModule(myModule: ModuleID): IvySbt#Module = module(defaultModuleId, Seq(myModule), None)
private def getAllFiles(report: UpdateReport) =
for {
conf <- report.configurations
m <- conf.modules
(_, f) <- m.artifacts
} yield f
}

View File

@ -0,0 +1,145 @@
package sbt
package internal
package librarymanagement
import sbt.internal.util.UnitSpec
import java.net.URL
/**
* Tests that we can correctly parse repositories definitions.
*/
class RepositoriesParserSpecification extends UnitSpec {
import RepositoriesParser._
"The RepositoriesParser" should "check that repositories file starts with [repositories]" in {
val file = """local
|maven-central""".stripMargin
a[Exception] should be thrownBy RepositoriesParser(file)
}
it should "parse the local repository" in {
val file = """[repositories]
| local""".stripMargin
val repos = RepositoriesParser(file)
repos.size shouldBe 1
repos(0) shouldBe PredefinedRepository(xsbti.Predefined.Local)
}
it should "parse the local maven repository" in {
val file = """[repositories]
| maven-local""".stripMargin
val repos = RepositoriesParser(file)
repos.size shouldBe 1
repos(0) shouldBe PredefinedRepository(xsbti.Predefined.MavenLocal)
}
it should "parse Maven Central repository" in {
val file = """[repositories]
| maven-central""".stripMargin
val repos = RepositoriesParser(file)
repos.size shouldBe 1
repos(0) shouldBe PredefinedRepository(xsbti.Predefined.MavenCentral)
}
it should "parse simple Maven repository" in {
val file = """[repositories]
| mavenRepo: https://repo1.maven.org""".stripMargin
val repos = RepositoriesParser(file)
repos.size shouldBe 1
repos(0) shouldBe MavenRepository("mavenRepo", new URL("https://repo1.maven.org"))
}
it should "parse `bootOnly` option" in {
val file = """[repositories]
| ivyRepo: https://repo1.maven.org, [orgPath], bootOnly""".stripMargin
val repos = RepositoriesParser(file)
val expected =
IvyRepository("ivyRepo", new URL("https://repo1.maven.org"), "[orgPath]", "[orgPath]",
mavenCompatible = false,
skipConsistencyCheck = false,
descriptorOptional = false,
bootOnly = true)
repos.size shouldBe 1
repos(0) shouldBe expected
}
it should "parse `mavenCompatible` option" in {
val file = """[repositories]
| ivyRepo: https://repo1.maven.org, [orgPath], mavenCompatible""".stripMargin
val repos = RepositoriesParser(file)
val expected =
IvyRepository("ivyRepo", new URL("https://repo1.maven.org"), "[orgPath]", "[orgPath]",
mavenCompatible = true,
skipConsistencyCheck = false,
descriptorOptional = false,
bootOnly = false)
repos.size shouldBe 1
repos(0) shouldBe expected
}
it should "parse `skipConsistencyCheck` option" in {
val file = """[repositories]
| ivyRepo: https://repo1.maven.org, [orgPath], skipConsistencyCheck""".stripMargin
val repos = RepositoriesParser(file)
val expected =
IvyRepository("ivyRepo", new URL("https://repo1.maven.org"), "[orgPath]", "[orgPath]",
mavenCompatible = false,
skipConsistencyCheck = true,
descriptorOptional = false,
bootOnly = false)
repos.size shouldBe 1
repos(0) shouldBe expected
}
it should "parse `descriptorOptional` option" in {
val file = """[repositories]
| ivyRepo: https://repo1.maven.org, [orgPath], descriptorOptional""".stripMargin
val repos = RepositoriesParser(file)
val expected =
IvyRepository("ivyRepo", new URL("https://repo1.maven.org"), "[orgPath]", "[orgPath]",
mavenCompatible = false,
skipConsistencyCheck = false,
descriptorOptional = true,
bootOnly = false)
repos.size shouldBe 1
repos(0) shouldBe expected
}
it should "parse complex ivy repository definition" in {
val file = """[repositories]
| ivyRepo: https://repo1.maven.org, [orgPath], [artPath], descriptorOptional, skipConsistencyCheck""".stripMargin
val repos = RepositoriesParser(file)
val expected =
IvyRepository("ivyRepo", new URL("https://repo1.maven.org"), "[orgPath]", "[artPath]",
mavenCompatible = false,
skipConsistencyCheck = true,
descriptorOptional = true,
bootOnly = false)
repos.size shouldBe 1
repos(0) shouldBe expected
}
it should "parse multiple repositories defined together" in {
val file = """[repositories]
| local
| ivyRepo: https://repo1.maven.org, [orgPath], [artPath], descriptorOptional, skipConsistencyCheck
| mavenRepo: https://repo1.maven.org""".stripMargin
val expected0 = PredefinedRepository(xsbti.Predefined.Local)
val expected1 =
IvyRepository("ivyRepo", new URL("https://repo1.maven.org"), "[orgPath]", "[artPath]",
mavenCompatible = false,
skipConsistencyCheck = true,
descriptorOptional = true,
bootOnly = false)
val expected2 = MavenRepository("mavenRepo", new URL("https://repo1.maven.org"))
val repos = RepositoriesParser(file)
repos.size shouldBe 3
repos(0) shouldBe expected0
repos(1) shouldBe expected1
repos(2) shouldBe expected2
}
}

View File

@ -11,6 +11,7 @@ object Dependencies {
lazy val utilCollection = "org.scala-sbt" %% "util-collection" % utilVersion
lazy val utilLogging = "org.scala-sbt" %% "util-logging" % utilVersion
lazy val utilTesting = "org.scala-sbt" %% "util-testing" % utilVersion
lazy val utilCompletion = "org.scala-sbt" %% "util-completion" % utilVersion
lazy val launcherInterface = "org.scala-sbt" % "launcher-interface" % "1.0.0-M1"
lazy val ivy = "org.scala-sbt.ivy" % "ivy" % "2.3.0-sbt-2cc8d2761242b072cedb0a04cb39435c4fa24f9a"