Merge pull request #2564 from Duhemm/wip/fix-2518

Off-the-grid installation of sbt
This commit is contained in:
eugene yokota 2016-04-20 16:53:19 -04:00
commit b10414da73
9 changed files with 517 additions and 2 deletions

View File

@ -28,7 +28,24 @@ def commonSettings: Seq[Setting[_]] = Seq(
incOptions := incOptions.value.withNameHashing(true),
crossScalaVersions := Seq(scala210),
bintrayPackage := (bintrayPackage in ThisBuild).value,
bintrayRepository := (bintrayRepository in ThisBuild).value
bintrayRepository := (bintrayRepository in ThisBuild).value,
test in assembly := {},
assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = true),
assemblyMergeStrategy in assembly := {
case PathList(ps @ _*) if ps.last == "javax.inject.Named" => MergeStrategy.first
case PathList(ps @ _*) if ps.last endsWith ".class" => MergeStrategy.first
case PathList(ps @ _*) if ps.last endsWith "module.properties" => MergeStrategy.first
case PathList(ps @ _*) if ps.last == "MANIFEST.MF" => MergeStrategy.rename
case "LICENSE" => MergeStrategy.first
case "NOTICE" => MergeStrategy.first
// excluded from fat jar because otherwise we may pick it up when determining the `actualVersion`
// of other scala instances.
case "compiler.properties" => MergeStrategy.discard
case x =>
val oldStrategy = (assemblyMergeStrategy in assembly).value
oldStrategy(x)
}
)
def minimalSettings: Seq[Setting[_]] =
@ -373,11 +390,24 @@ lazy val compilerIntegrationProj = (project in (compilePath / "integration")).
name := "Compiler Integration"
)
lazy val packageBridgeSource = settingKey[Boolean]("Whether to package the compiler bridge sources in compiler ivy project's resources.")
lazy val compilerIvyProj = (project in compilePath / "ivy").
dependsOn (ivyProj, compilerProj).
settings(
baseSettings,
name := "Compiler Ivy Integration"
name := "Compiler Ivy Integration",
packageBridgeSource := false,
resourceGenerators in Compile <+= Def.task {
if (packageBridgeSource.value) {
val compilerBridgeSrc = (Keys.packageSrc in (compileInterfaceProj, Compile)).value
val xsbtiJAR = (Keys.packageBin in (interfaceProj, Compile)).value
// They are immediately used by the static launcher.
val included = Set("scala-compiler.jar", "scala-library.jar", "scala-reflect.jar")
val scalaJars = (externalDependencyClasspath in Compile).value.map(_.data).filter(j => included contains j.getName)
Seq(compilerBridgeSrc, xsbtiJAR) ++ scalaJars
}
else Nil
}
)
lazy val scriptedBaseProj = (project in scriptedPath / "base").
@ -646,5 +676,16 @@ def customCommands: Seq[Setting[_]] = Seq(
"publish" ::
"bintrayRelease" ::
state
},
// Produces a fat runnable JAR that contains everything needed to use sbt.
commands += Command.command("install") { state =>
val packageBridgeSourceKey = packageBridgeSource.key.label
val compilerIvy = compilerIvyProj.id
val sbt = sbtProj.id
s"$compilerIvy/clean" ::
s"set $packageBridgeSourceKey in $compilerIvy := true" ::
s"$sbt/assembly" ::
s"set $packageBridgeSourceKey in $compilerIvy := false" ::
state
}
)

View File

@ -0,0 +1,194 @@
package sbt
import java.io.File
import java.net.URL
import org.apache.ivy.core.cache.ArtifactOrigin
import org.apache.ivy.core.cache.{ DefaultRepositoryCacheManager, RepositoryCacheManager }
import org.apache.ivy.core.module.descriptor.{ Artifact => IvyArtifact, DefaultArtifact, DefaultDependencyArtifactDescriptor, DefaultModuleDescriptor, DependencyArtifactDescriptor, DependencyDescriptor }
import org.apache.ivy.core.module.id.ModuleRevisionId
import org.apache.ivy.core.report.ArtifactDownloadReport
import org.apache.ivy.core.report.{ DownloadReport, DownloadStatus }
import org.apache.ivy.core.report.MetadataArtifactDownloadReport
import org.apache.ivy.core.resolve.{ DownloadOptions, ResolveData, ResolvedModuleRevision }
import org.apache.ivy.core.search.{ ModuleEntry, OrganisationEntry, RevisionEntry }
import org.apache.ivy.core.settings.IvySettings
import org.apache.ivy.plugins.namespace.Namespace
import org.apache.ivy.plugins.repository.url.URLResource
import org.apache.ivy.plugins.resolver.{ DependencyResolver, ResolverSettings }
import org.apache.ivy.plugins.resolver.util.ResolvedResource
import FakeResolver._
/**
* A fake `DependencyResolver` that statically serves predefined artifacts.
*/
private[sbt] class FakeResolver(private var name: String, cacheDir: File, modules: ModulesMap) extends DependencyResolver {
private object Artifact {
def unapply(art: IvyArtifact): Some[(String, String, String)] = {
val revisionID = art.getModuleRevisionId()
val organisation = revisionID.getOrganisation
val name = revisionID.getName
val revision = revisionID.getRevision
Some((organisation, name, revision))
}
def unapply(dd: DependencyDescriptor): Some[(String, String, String)] = {
val module = dd.getDependencyId()
val organisation = module.getOrganisation
val name = module.getName
val mrid = dd.getDependencyRevisionId()
val revision = mrid.getRevision()
Some((organisation, name, revision))
}
}
override def publish(artifact: IvyArtifact, src: File, overwrite: Boolean): Unit =
throw new UnsupportedOperationException("This resolver doesn't support publishing.")
override def abortPublishTransaction(): Unit =
throw new UnsupportedOperationException("This resolver doesn't support publishing.")
override def beginPublishTransaction(module: ModuleRevisionId, overwrite: Boolean): Unit =
throw new UnsupportedOperationException("This resolver doesn't support publishing.")
override def commitPublishTransaction(): Unit =
throw new UnsupportedOperationException("This resolver doesn't support publishing.")
override def download(artifact: ArtifactOrigin, options: DownloadOptions): ArtifactDownloadReport = {
val report = new ArtifactDownloadReport(artifact.getArtifact)
val path = new URL(artifact.getLocation).toURI.getPath
val localFile = new File(path)
if (path.nonEmpty && localFile.exists) {
report.setLocalFile(localFile)
report.setDownloadStatus(DownloadStatus.SUCCESSFUL)
report.setSize(localFile.length)
} else {
report.setDownloadStatus(DownloadStatus.FAILED)
}
report
}
override def download(artifacts: Array[IvyArtifact], options: DownloadOptions): DownloadReport = {
val report = new DownloadReport
artifacts foreach { art =>
val artifactOrigin = locate(art)
Option(locate(art)) foreach (o => report.addArtifactReport(download(o, options)))
}
report
}
override def dumpSettings(): Unit = ()
override def exists(artifact: IvyArtifact): Boolean = {
val Artifact(organisation, name, revision) = artifact
modules.get((organisation, name, revision)).isDefined
}
// This is a fake resolver and we don't have Ivy files. Ivy's spec says we can return `null` if
// we can't find the module descriptor.
override def findIvyFileRef(dd: DependencyDescriptor, data: ResolveData): ResolvedResource = null
override def getDependency(dd: DependencyDescriptor, data: ResolveData): ResolvedModuleRevision = {
val Artifact(organisation, name, revision) = dd
val mrid = dd.getDependencyRevisionId()
val artifact = modules get ((organisation, name, revision)) map { arts =>
val artifacts: Array[DependencyArtifactDescriptor] = arts.toArray map (_ artifactOf dd)
val moduleDescriptor = DefaultModuleDescriptor.newDefaultInstance(mrid, artifacts)
val defaultArtifact = arts.headOption match {
case Some(FakeArtifact(name, tpe, ext, _)) => new DefaultArtifact(mrid, new java.util.Date, name, tpe, ext)
case None => null
}
val metadataReport = new MetadataArtifactDownloadReport(defaultArtifact)
metadataReport.setDownloadStatus(DownloadStatus.SUCCESSFUL)
new ResolvedModuleRevision(this, this, moduleDescriptor, metadataReport)
}
artifact.orNull
}
override def getName(): String = name
override val getNamespace: Namespace = {
val ns = new Namespace()
ns.setName(name)
ns
}
override val getRepositoryCacheManager: RepositoryCacheManager = {
val cacheName = name + "-cache"
val ivySettings = new IvySettings()
val baseDir = cacheDir
new DefaultRepositoryCacheManager(cacheName, ivySettings, baseDir)
}
override def listModules(organisation: OrganisationEntry): Array[ModuleEntry] =
modules.keys.collect {
case (o, m, _) if o == organisation.getOrganisation =>
val organisationEntry = new OrganisationEntry(this, o)
new ModuleEntry(organisationEntry, m)
}.toArray
override def listOrganisations(): Array[OrganisationEntry] =
modules.keys.map { case (o, _, _) => new OrganisationEntry(this, o) }.toArray
override def listRevisions(module: ModuleEntry): Array[RevisionEntry] =
modules.keys.collect {
case (o, m, v) if o == module.getOrganisation && m == module.getModule =>
new RevisionEntry(module, v)
}.toArray
override def listTokenValues(tokens: Array[String], criteria: java.util.Map[_, _]): Array[java.util.Map[_, _]] =
Array.empty
override def listTokenValues(token: String, otherTokenValues: java.util.Map[_, _]): Array[String] =
Array.empty
override def locate(art: IvyArtifact): ArtifactOrigin = {
val Artifact(moduleOrganisation, moduleName, moduleRevision) = art
val artifact =
for {
artifacts <- modules get ((moduleOrganisation, moduleName, moduleRevision))
artifact <- artifacts find (a => a.name == art.getName && a.tpe == art.getType && a.ext == art.getExt)
} yield new ArtifactOrigin(art, /* isLocal = */ true, artifact.file.toURI.toURL.toString)
artifact.orNull
}
override def reportFailure(art: IvyArtifact): Unit = ()
override def reportFailure(): Unit = ()
override def setName(name: String): Unit = {
this.name = name
getNamespace.setName(name)
}
override def setSettings(settings: ResolverSettings): Unit = ()
private class LocalURLResource(jar: File) extends URLResource(jar.toURI.toURL) {
override def isLocal(): Boolean = true
}
}
private[sbt] object FakeResolver {
type ModulesMap = Map[(String, String, String), Seq[FakeArtifact]]
final case class FakeArtifact(name: String, tpe: String, ext: String, file: File) {
def artifactOf(dd: DependencyDescriptor): DependencyArtifactDescriptor =
new DefaultDependencyArtifactDescriptor(dd, name, tpe, ext, file.toURI.toURL, new java.util.HashMap)
}
}

View File

@ -163,6 +163,11 @@ final case class SftpRepository(name: String, connection: SshConnection, pattern
protected def copy(patterns: Patterns): SftpRepository = SftpRepository(name, connection, patterns)
protected def copy(connection: SshConnection): SftpRepository = SftpRepository(name, connection, patterns)
}
/** A repository that conforms to sbt launcher's interface */
private[sbt] class FakeRepository(resolver: DependencyResolver) extends xsbti.Repository {
def rawRepository = new RawRepository(resolver)
}
import Resolver._

Binary file not shown.

View File

View File

@ -0,0 +1,86 @@
package sbt
import java.io.File
import org.specs2._
class FakeResolverSpecification extends BaseIvySpecification {
import FakeResolver._
def is = s2"""
This is a specification for the FakeResolver
The FakeResolver should
find modules with only one artifact $singleArtifact
find modules with more than one artifact $multipleArtifacts
fail gracefully when asked for unknown modules $nonExistingModule
fail gracefully when some artifacts cannot be found $existingAndNonExistingArtifacts
"""
val myModule = ModuleID("org.example", "my-module", "0.0.1-SNAPSHOT", Some("compile"))
val example = ModuleID("com.example", "example", "1.0.0", Some("compile"))
val anotherExample = ModuleID("com.example", "another-example", "1.0.0", Some("compile"))
val nonExisting = ModuleID("com.example", "does-not-exist", "1.2.3", Some("compile"))
def singleArtifact = {
val m = getModule(myModule)
val report = ivyUpdate(m)
val allFiles = getAllFiles(report)
report.allModules should haveLength(1)
report.configurations should haveLength(3)
allFiles should haveLength(1)
allFiles(1).getName should beEqualTo("artifact1-0.0.1-SNAPSHOT.jar")
}
def multipleArtifacts = {
val m = getModule(example)
val report = ivyUpdate(m)
val allFiles = getAllFiles(report).toSet
report.allModules should haveLength(1)
report.configurations should haveLength(3)
allFiles should haveLength(2)
allFiles map (_.getName) should beEqualTo(Set("artifact1-1.0.0.jar", "artifact2-1.0.0.txt"))
}
def nonExistingModule = {
val m = getModule(nonExisting)
ivyUpdate(m) should throwA[ResolveException]
}
def existingAndNonExistingArtifacts = {
val m = getModule(anotherExample)
ivyUpdate(m) should throwA[ResolveException]("download failed: com.example#another-example;1.0.0!non-existing.txt")
}
private def artifact1 = new File(getClass.getResource("/artifact1.jar").toURI.getPath)
private def artifact2 = new File(getClass.getResource("/artifact2.txt").toURI.getPath)
private def modules = Map(
("org.example", "my-module", "0.0.1-SNAPSHOT") -> List(
FakeArtifact("artifact1", "jar", "jar", artifact1)
),
("com.example", "example", "1.0.0") -> List(
FakeArtifact("artifact1", "jar", "jar", artifact1),
FakeArtifact("artifact2", "txt", "txt", artifact2)
),
("com.example", "another-example", "1.0.0") -> List(
FakeArtifact("artifact1", "jar", "jar", artifact1),
FakeArtifact("non-existing", "txt", "txt", new File("non-existing-file"))
)
)
private def fakeResolver = new FakeResolver("FakeResolver", new File("tmp"), modules)
override def resolvers: Seq[Resolver] = Seq(new RawRepository(fakeResolver))
private def getModule(myModule: ModuleID): IvySbt#Module = module(defaultModuleId, Seq(myModule), None)
private def getAllFiles(report: UpdateReport) =
for {
conf <- report.configurations
m <- conf.modules
(_, f) <- m.artifacts
} yield f
}

View File

@ -1883,6 +1883,7 @@ object Classpaths {
{
import xsbti.Predefined
repo match {
case f: FakeRepository => f.rawRepository
case m: xsbti.MavenRepository => MavenRepository(m.id, m.url.toString)
case i: xsbti.IvyRepository =>
val patterns = Patterns(i.ivyPattern :: Nil, i.artifactPattern :: Nil, mavenCompatible(i), descriptorOptional(i), skipConsistencyCheck(i))

View File

@ -7,3 +7,4 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-javaversioncheck" % "0.1.0")
addSbtPlugin("com.typesafe.sbt" % "sbt-scalariform" % "1.3.0") // 1.6.0 is out but is a hard upgrade
addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "0.8.2")
addSbtPlugin("me.lessis" % "bintray-sbt" % "0.3.0")
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.2")

View File

@ -0,0 +1,187 @@
package sbt
import java.net.URLClassLoader
import java.util.Properties
/**
* A Main class for running sbt without sbt launcher.
*/
object Main {
def main(args: Array[String]): Unit = {
val appConfiguration = new StaticAppConfiguration(args)
new xMain().run(appConfiguration)
}
}
private object StaticUtils {
val MAIN = "sbt.Main"
val SCALA_ORG = "org.scala-lang"
val COMPILER = "compiler"
val COMPILER_JAR = "scala-compiler.jar"
val LIBRARY = "library"
val LIBRARY_JAR = "scala-library.jar"
val REFLECT = "reflect"
val REFLECT_JAR = "scala-reflect.jar"
val BRIDGE = "compiler-interface"
val BRIDGE_JAR = s"compiler-interface-${sbtApplicationID.version}-sources.jar"
val XSBTI = "xsbti"
val XSBTI_JAR = s"interface-${sbtApplicationID.version}.jar"
val thisJAR: File = new File(getClass.getProtectionDomain().getCodeSource().getLocation().toURI().getPath())
def getProperty(loader: ClassLoader, filename: String, property: String): Option[String] =
for {
stream <- Option(loader.getResourceAsStream(filename))
props = new Properties()
_ = props.load(stream)
o <- Option(props get property)
s = o.asInstanceOf[String]
} yield s
}
private class StaticComponentProvider(bootDirectory: File) extends xsbti.ComponentProvider {
override def addToComponent(componentID: String, components: Array[File]): Boolean = {
components foreach { c =>
IO.copyFile(c, componentLocation(componentID) / c.getName)
}
true
}
override def component(componentID: String): Array[File] =
PathFinder(componentLocation(componentID)).***.get.filter(_.isFile).toArray
override def componentLocation(id: String): File =
bootDirectory / s"static-sbt-${sbtApplicationID.version}" / id
override def defineComponent(componentID: String, components: Array[File]): Unit =
addToComponent(componentID, components)
override def lockFile(): File = null
}
private object sbtApplicationID extends xsbti.ApplicationID {
override val groupID: String = xsbti.ArtifactInfo.SbtOrganization
override val name: String = "sbt"
override def version(): String = StaticUtils.getProperty(getClass.getClassLoader, "xsbt.version.properties", "version") getOrElse "unknown"
override val mainClass: String = StaticUtils.MAIN
override val mainComponents: Array[String] = Array.empty
override val crossVersioned: Boolean = false
override val crossVersionedValue: xsbti.CrossValue = xsbti.CrossValue.Disabled
override val classpathExtra: Array[File] = Array.empty
}
private class WeakGlobalLock extends xsbti.GlobalLock {
override def apply[T](lockFile: File, run: java.util.concurrent.Callable[T]): T = run.call
}
private class StaticLauncher(appProvider: StaticAppProvider, scalaProvider: StaticScalaProvider) extends xsbti.Launcher {
override def getScala(version: String): xsbti.ScalaProvider = getScala(version, "")
override def getScala(version: String, reason: String): xsbti.ScalaProvider = getScala(version, reason, StaticUtils.SCALA_ORG)
override def getScala(version: String, reason: String, scalaOrg: String): xsbti.ScalaProvider = {
val myScalaVersion = scalaProvider.version
if (myScalaVersion == version) scalaProvider
else throw new InvalidComponent(s"This launcher can only provide scala $myScalaVersion, asked for scala $version")
}
override def app(id: xsbti.ApplicationID, version: String): xsbti.AppProvider = appProvider
override def topLoader(): ClassLoader = new URLClassLoader(Array.empty, null)
override def globalLock(): xsbti.GlobalLock = new WeakGlobalLock
override def bootDirectory(): File = new File(sys props "user.home") / ".sbt" / "boot"
override def ivyRepositories(): Array[xsbti.Repository] = appRepositories
override def appRepositories(): Array[xsbti.Repository] = Array(new FakeRepository(new FakeResolver("fakeresolver", bootDirectory / "fakeresolver-cache", modules)))
override def isOverrideRepositories(): Boolean = false
override def ivyHome(): File = null
override def checksums(): Array[String] = Array.empty
private lazy val modules = Map(
("org.scala-sbt", "sbt", sbtApplicationID.version) ->
Seq(FakeResolver.FakeArtifact("sbt", "jar", "jar", StaticUtils.thisJAR)),
("org.scala-sbt", "compiler-interface", sbtApplicationID.version) -> {
val file = scalaProvider.getComponent(StaticUtils.BRIDGE)
Seq(FakeResolver.FakeArtifact("compiler-interface", "src", "jar", file))
}
)
}
private class StaticScalaProvider(appProvider: StaticAppProvider) extends xsbti.ScalaProvider {
def getComponent(componentID: String): File = {
val component = appProvider.components.component(componentID)
assert(component.length == 1, s"""Component $componentID should have 1 file, ${component.length} files found: ${component.mkString(", ")}.""")
component(0)
}
override def launcher: xsbti.Launcher = new StaticLauncher(appProvider, this)
override def app(id: xsbti.ApplicationID): xsbti.AppProvider = appProvider
override def compilerJar(): File = getComponent(StaticUtils.COMPILER)
override def libraryJar(): File = getComponent(StaticUtils.LIBRARY)
override def jars(): Array[File] = Array(compilerJar, libraryJar, getComponent(StaticUtils.REFLECT))
override def loader(): ClassLoader = new URLClassLoader(jars map (_.toURI.toURL))
override def version(): String = StaticUtils.getProperty(loader, "compiler.properties", "version.number") getOrElse "unknown"
}
private class StaticAppProvider(appConfig: StaticAppConfiguration) extends xsbti.AppProvider {
if (components.component(StaticUtils.COMPILER).isEmpty) {
installFromResources(StaticUtils.COMPILER_JAR, StaticUtils.COMPILER)
}
if (components.component(StaticUtils.LIBRARY).isEmpty) {
installFromResources(StaticUtils.LIBRARY_JAR, StaticUtils.LIBRARY)
}
if (components.component(StaticUtils.REFLECT).isEmpty) {
installFromResources(StaticUtils.REFLECT_JAR, StaticUtils.REFLECT)
}
if (components.component(StaticUtils.XSBTI).isEmpty) {
installFromResources(StaticUtils.XSBTI_JAR, StaticUtils.XSBTI)
}
if (components.component(StaticUtils.BRIDGE).isEmpty) {
installFromResources(StaticUtils.BRIDGE_JAR, StaticUtils.BRIDGE)
}
override def components(): xsbti.ComponentProvider = new StaticComponentProvider(scalaProvider.launcher.bootDirectory)
override def entryPoint(): Class[_] = loader.getClass
override def id(): xsbti.ApplicationID = sbtApplicationID
override def loader(): ClassLoader = getClass.getClassLoader
override def mainClass(): Class[xsbti.AppMain] = loader.loadClass(id.mainClass).asInstanceOf[Class[xsbti.AppMain]]
override def mainClasspath(): Array[File] = Array(StaticUtils.thisJAR)
override def newMain(): xsbti.AppMain = new xMain
override def scalaProvider(): xsbti.ScalaProvider = new StaticScalaProvider(this)
/**
* Retrieves `fileName` from the resources and installs it in `componentID`.
* @param filename Name of the file to get from the resources.
* @param componentID ID of the component to create.
*/
private def installFromResources(filename: String, componentID: String): Unit =
IO.withTemporaryDirectory { tmp =>
Option(getClass.getClassLoader.getResourceAsStream(filename)) match {
case Some(stream) =>
val target = tmp / filename
val out = new java.io.FileOutputStream(target)
var read = 0
val content = new Array[Byte](1024)
while ({ read = stream.read(content); read != -1 }) {
out.write(content, 0, read)
}
components.defineComponent(componentID, Array(target))
case None =>
sys.error(s"Couldn't install component $componentID: $filename not found on resource path.")
}
}
}
private class StaticAppConfiguration(override val arguments: Array[String]) extends xsbti.AppConfiguration {
override val baseDirectory: File = new File(sys props "user.dir")
override val provider: xsbti.AppProvider = new StaticAppProvider(this)
}