Add `CompilerBridgeProvider` interface and misc.

* Add interface for the provider.
* Rename `IvyComponentCompiler` to `ZincComponentCompiler`.
* Split `ZincComponentCompiler` and `ZincComponentManager`.
* Define `IfMissing` and `InvalidComponent` into independent files.
* Rename variables and internal API to be clearer.
This commit is contained in:
jvican 2017-05-18 17:32:04 +02:00
parent 9c1268412a
commit 3f1cce379f
8 changed files with 281 additions and 237 deletions

View File

@ -1,199 +0,0 @@
/*
* Zinc - The incremental compiler for Scala.
* Copyright 2011 - 2017, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* This software is released under the terms written in LICENSE.
*/
package sbt
package internal
package inc
import java.io.File
import sbt.io.{ Hash, IO }
import sbt.internal.librarymanagement.{
IvyConfiguration,
JsonUtil,
IvySbt,
InlineConfiguration,
RetrieveConfiguration,
IvyActions,
UnresolvedWarningConfiguration,
LogicalClock,
UnresolvedWarning
}
import sbt.librarymanagement.{
Configurations,
ModuleID,
ModuleInfo,
Resolver,
UpdateConfiguration,
UpdateLogging,
UpdateOptions,
ArtifactTypeFilter
}
import sbt.librarymanagement.syntax._
import sbt.util.Logger
import sbt.internal.util.{ BufferedLogger, FullLogger }
private[sbt] object ComponentCompiler {
// val xsbtiID = "xsbti"
// val srcExtension = "-src"
val binSeparator = "-bin_"
val javaVersion = System.getProperty("java.class.version")
def interfaceProvider(manager: ZincComponentManager,
ivyConfiguration: IvyConfiguration,
sourcesModule: ModuleID): CompilerBridgeProvider =
new CompilerBridgeProvider {
def apply(scalaInstance: xsbti.compile.ScalaInstance, log: Logger): File = {
// this is the instance used to compile the interface component
val componentCompiler = new IvyComponentCompiler(
new RawCompiler(scalaInstance, ClasspathOptionsUtil.auto, log),
manager,
ivyConfiguration,
sourcesModule,
log)
log.debug(
"Getting " + sourcesModule + " from component compiler for Scala " + scalaInstance.version)
componentCompiler()
}
}
lazy val incrementalVersion = {
val properties = new java.util.Properties
val propertiesStream =
getClass.getResource("/incrementalcompiler.version.properties").openStream
try { properties.load(propertiesStream) } finally { propertiesStream.close() }
properties.getProperty("version")
}
}
/**
* Component compiler which is able to to retrieve the compiler bridge sources
* `sourceModule` using Ivy.
* The compiled classes are cached using the provided component manager according
* to the actualVersion field of the RawCompiler.
*/
private[inc] class IvyComponentCompiler(compiler: RawCompiler,
manager: ZincComponentManager,
ivyConfiguration: IvyConfiguration,
sourcesModule: ModuleID,
log: Logger) {
import ComponentCompiler._
// private val xsbtiInterfaceModuleName = "compiler-interface"
// private val xsbtiInterfaceID = s"interface-$incrementalVersion"
private val sbtOrgTemp = JsonUtil.sbtOrgTemp
private val modulePrefixTemp = "temp-module-"
private val ivySbt: IvySbt = new IvySbt(ivyConfiguration)
private val buffered = new BufferedLogger(FullLogger(log))
def apply(): File = {
// binID is of the form "org.example-compilerbridge-1.0.0-bin_2.11.7__50.0"
val binID = binaryID(
s"${sourcesModule.organization}-${sourcesModule.name}-${sourcesModule.revision}")
manager.file(binID)(IfMissing.define(true, compileAndInstall(binID)))
}
private def binaryID(id: String): String = {
val base = id + binSeparator + compiler.scalaInstance.actualVersion
base + "__" + javaVersion
}
private def compileAndInstall(binID: String): Unit =
IO.withTemporaryDirectory { binaryDirectory =>
val targetJar = new File(binaryDirectory, s"$binID.jar")
buffered bufferQuietly {
IO.withTemporaryDirectory { retrieveDirectory =>
update(getModule(sourcesModule), retrieveDirectory) match {
case Left(uw) =>
import sbt.util.ShowLines._
throw new InvalidComponent(
s"Couldn't retrieve source module: $sourcesModule\n" +
uw.lines.mkString("\n"))
case Right(allArtifacts) =>
val (sources, xsbtiJars) = allArtifacts partition (_.getName endsWith "-sources.jar")
AnalyzingCompiler.compileSources(sources,
targetJar,
xsbtiJars,
sourcesModule.name,
compiler,
log)
manager.define(binID, Seq(targetJar))
}
}
}
}
/**
* Returns a dummy module that depends on `moduleID`.
* Note: Sbt's implementation of Ivy requires us to do this, because only the dependencies
* of the specified module will be downloaded.
*/
private def getModule(moduleID: ModuleID): ivySbt.Module = {
val sha1 = Hash.toHex(Hash(moduleID.name))
val dummyID = ModuleID(sbtOrgTemp, modulePrefixTemp + sha1, moduleID.revision)
.withConfigurations(moduleID.configurations)
getModule(dummyID, Vector(moduleID))
}
private def getModule(moduleID: ModuleID,
deps: Vector[ModuleID],
uo: UpdateOptions = UpdateOptions()): ivySbt.Module = {
val moduleSetting = InlineConfiguration(
validate = false,
ivyScala = None,
module = moduleID,
moduleInfo = ModuleInfo(moduleID.name),
dependencies = deps
).withConfigurations(Vector(Configurations.Component))
new ivySbt.Module(moduleSetting)
}
private def dependenciesNames(module: ivySbt.Module): String = module.moduleSettings match {
// `module` is a dummy module, we will only fetch its dependencies.
case ic: InlineConfiguration =>
ic.dependencies map {
case mID: ModuleID =>
import mID._
s"$organization % $name % $revision"
} mkString ", "
case _ =>
s"unknown"
}
private def update(module: ivySbt.Module,
retrieveDirectory: File): Either[UnresolvedWarning, Vector[File]] = {
val retrieveConfiguration =
RetrieveConfiguration(retrieveDirectory, Resolver.defaultRetrievePattern, false, None)
val updateConfiguration = UpdateConfiguration(
Some(retrieveConfiguration),
missingOk = false,
UpdateLogging.DownloadOnly,
ArtifactTypeFilter.forbid(Set("doc"))
)
buffered.info(s"Attempting to fetch ${dependenciesNames(module)}. This operation may fail.")
IvyActions.updateEither(module,
updateConfiguration,
UnresolvedWarningConfiguration(),
LogicalClock.unknown,
None,
buffered) match {
case Left(unresolvedWarning) =>
buffered.debug(s"Couldn't retrieve module ${dependenciesNames(module)}.")
Left(unresolvedWarning)
case Right(updateReport) =>
val allFiles = updateReport.allFiles
buffered.debug(s"Files retrieved for ${dependenciesNames(module)}:")
buffered.debug(allFiles mkString ", ")
Right(allFiles.toVector)
}
}
}

View File

@ -0,0 +1,14 @@
package sbt.internal.inc
sealed trait IfMissing
object IfMissing {
def fail: IfMissing = Fail
/** f is expected to call ZincComponentManager.define. */
def define(useSecondaryCache: Boolean, f: => Unit): IfMissing = new Define(useSecondaryCache, f)
object Fail extends IfMissing
final class Define(val useSecondaryCache: Boolean, define: => Unit) extends IfMissing {
def run(): Unit = define
}
}

View File

@ -0,0 +1,5 @@
package sbt.internal.inc
class InvalidComponent(msg: String, cause: Throwable) extends RuntimeException(msg, cause) {
def this(msg: String) = this(msg, null)
}

View File

@ -0,0 +1,13 @@
package sbt.internal.inc
import java.util.Properties
/** Defines utilities to load Java properties from the JVM. */
private[inc] object ResourceLoader {
def getPropertiesFor(resource: String, classLoader: ClassLoader): Properties = {
val properties = new java.util.Properties
val propertiesStream = getClass.getResource(resource).openStream
try { properties.load(propertiesStream) } finally { propertiesStream.close() }
properties
}
}

View File

@ -0,0 +1,218 @@
/*
* Zinc - The incremental compiler for Scala.
* Copyright 2011 - 2017, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* This software is released under the terms written in LICENSE.
*/
package sbt
package internal
package inc
import java.io.File
import sbt.io.{ Hash, IO }
import sbt.internal.librarymanagement.{
InlineConfiguration,
IvyActions,
IvyConfiguration,
IvySbt,
JsonUtil,
LogicalClock,
RetrieveConfiguration,
UnresolvedWarning,
UnresolvedWarningConfiguration
}
import sbt.librarymanagement.{
ArtifactTypeFilter,
Configurations,
ModuleID,
ModuleInfo,
Resolver,
UpdateConfiguration,
UpdateLogging,
UpdateOptions
}
import sbt.librarymanagement.syntax._
import sbt.util.{ InterfaceUtil, Logger }
import xsbti.compile.CompilerBridgeProvider
private[sbt] object ZincComponentCompiler {
final val binSeparator = "-bin_"
final val javaClassVersion = System.getProperty("java.class.version")
private[inc] final val sbtOrgTemp = JsonUtil.sbtOrgTemp
private[inc] final val modulePrefixTemp = "temp-module-"
private final val ZincVersionPropertyFile = "/incrementalcompiler.version.properties"
private final val ZincVersionProperty = "version"
final lazy val incrementalVersion: String = {
val cl = this.getClass.getClassLoader
ResourceLoader.getPropertiesFor(ZincVersionPropertyFile, cl).getProperty(ZincVersionProperty)
}
private class ZincCompilerBridgeProvider(manager: ZincComponentManager,
ivyConfiguration: IvyConfiguration,
bridgeModule: ModuleID)
extends CompilerBridgeProvider {
override def getBridgeSources(scalaInstance: xsbti.compile.ScalaInstance,
logger: xsbti.Logger): File = {
val autoClasspath = ClasspathOptionsUtil.auto
val bridgeCompiler = new RawCompiler(scalaInstance, autoClasspath, logger)
val ivyComponent =
new ZincComponentCompiler(bridgeCompiler, manager, ivyConfiguration, bridgeModule, logger)
logger.debug(InterfaceUtil.f0(s"Getting $bridgeModule for Scala ${scalaInstance.version}"))
ivyComponent.getCompiledBridgeJar
}
}
def interfaceProvider(manager: ZincComponentManager,
ivyConfiguration: IvyConfiguration,
sourcesModule: ModuleID): CompilerBridgeProvider =
new ZincCompilerBridgeProvider(manager, ivyConfiguration, sourcesModule)
}
/**
* Component compiler which is able to to retrieve the compiler bridge sources
* `sourceModule` using Ivy.
* The compiled classes are cached using the provided component manager according
* to the actualVersion field of the RawCompiler.
*/
private[inc] class ZincComponentCompiler(
compiler: RawCompiler,
manager: ZincComponentManager,
ivyConfiguration: IvyConfiguration,
bridgeSources: ModuleID,
log: Logger
) {
import sbt.internal.util.{ BufferedLogger, FullLogger }
private final val ivySbt: IvySbt = new IvySbt(ivyConfiguration)
private final val buffered = new BufferedLogger(FullLogger(log))
def getCompiledBridgeJar: File = {
val jarBinaryName = createBridgeSourcesID(bridgeSources)
manager.file(jarBinaryName)(IfMissing.define(true, compileAndInstall(jarBinaryName)))
}
/**
* Returns the id for the compiler interface component.
*
* The ID contains the following parts:
* - The organization, name and revision.
* - The bin separator to make clear the jar represents binaries.
* - The Scala version for which the compiler interface is meant to.
* - The JVM class version.
*
* Example: "org.scala-sbt-compiler-bridge-1.0.0-bin_2.11.7__50.0".
*
* @param sources The moduleID representing the compiler bridge sources.
* @return The complete jar identifier for the bridge sources.
*/
private def createBridgeSourcesID(sources: ModuleID): String = {
import ZincComponentCompiler.{ binSeparator, javaClassVersion }
val id = s"${sources.organization}-${sources.name}-${sources.revision}"
val scalaVersion = compiler.scalaInstance.actualVersion()
s"$id$binSeparator${scalaVersion}__$javaClassVersion"
}
/**
* Resolves the compiler bridge sources, compiles them and installs the sbt component
* in the local filesystem to make sure that it's reused the next time is required.
*
* @param compilerBridgeId The identifier for the compiler bridge sources.
*/
private def compileAndInstall(compilerBridgeId: String): Unit = {
import UnresolvedWarning.unresolvedWarningLines
val ivyModuleForBridge = wrapDependencyInModule(bridgeSources)
IO.withTemporaryDirectory { binaryDirectory =>
val target = new File(binaryDirectory, s"$compilerBridgeId.jar")
buffered bufferQuietly {
IO.withTemporaryDirectory { retrieveDirectory =>
update(ivyModuleForBridge, retrieveDirectory) match {
case Left(uw) =>
val mod = bridgeSources.toString
val unresolvedLines = unresolvedWarningLines.showLines(uw)
val unretrievedMessage = s"The compiler bridge sources $mod could not be retrieved."
throw new InvalidComponent(s"$unresolvedLines\n$unresolvedLines")
case Right(allArtifacts) =>
val (srcs, xsbtiJars) = allArtifacts.partition(_.getName.endsWith("-sources.jar"))
val toCompileID = bridgeSources.name
AnalyzingCompiler.compileSources(srcs, target, xsbtiJars, toCompileID, compiler, log)
manager.define(compilerBridgeId, Seq(target))
}
}
}
}
}
/**
* Returns an ivy module that will wrap and download a given `moduleID`.
*
* @param moduleID The `moduleID` that needs to be wrapped in a dummy module to be downloaded.
*/
private def wrapDependencyInModule(moduleID: ModuleID): ivySbt.Module = {
def getModule(moduleID: ModuleID, deps: Vector[ModuleID], uo: UpdateOptions): ivySbt.Module = {
val moduleInfo = ModuleInfo(moduleID.name)
val componentIvySettings = InlineConfiguration(
validate = false,
ivyScala = None,
module = moduleID,
moduleInfo = moduleInfo,
dependencies = deps
).withConfigurations(Vector(Configurations.Component))
new ivySbt.Module(componentIvySettings)
}
import ZincComponentCompiler.{ sbtOrgTemp, modulePrefixTemp }
val sha1 = Hash.toHex(Hash(moduleID.name))
val dummyID = ModuleID(sbtOrgTemp, s"$modulePrefixTemp$sha1", moduleID.revision)
.withConfigurations(moduleID.configurations)
val defaultUpdateOptions = UpdateOptions()
getModule(dummyID, Vector(moduleID), defaultUpdateOptions)
}
// The implementation of this method is linked to `wrapDependencyInModule`
private def prettyPrintDependency(module: ivySbt.Module): String = {
module.moduleSettings match {
case ic: InlineConfiguration =>
// Pretty print the module as `ModuleIDExtra.toStringImpl` does.
val dependency =
ic.dependencies.map(m => s"${m.organization}:${m.name}:${m.revision}").headOption
dependency.getOrElse(sys.error("Fatal: more than one dependency in dummy bridge module."))
case _ => sys.error("Fatal: configuration to download was not inline.")
}
}
private final val warningConf = UnresolvedWarningConfiguration()
private final val defaultRetrievePattern = Resolver.defaultRetrievePattern
private def defaultUpdateConfiguration(retrieveDirectory: File): UpdateConfiguration = {
val retrieve = RetrieveConfiguration(retrieveDirectory, defaultRetrievePattern, false, None)
val logLevel = UpdateLogging.DownloadOnly
val noDocs = ArtifactTypeFilter.forbid(Set("doc"))
UpdateConfiguration(Some(retrieve), missingOk = false, logLevel, noDocs)
}
private def update(module: ivySbt.Module,
retrieveDirectory: File): Either[UnresolvedWarning, Vector[File]] = {
import IvyActions.updateEither
val updateConfiguration = defaultUpdateConfiguration(retrieveDirectory)
val dependencies = prettyPrintDependency(module)
buffered.info(s"Attempting to fetch $dependencies. This operation may fail.")
val clockForCache = LogicalClock.unknown
updateEither(module, updateConfiguration, warningConf, clockForCache, None, buffered) match {
case Left(unresolvedWarning) =>
buffered.debug(s"Couldn't retrieve module ${prettyPrintDependency(module)}.")
Left(unresolvedWarning)
case Right(updateReport) =>
val allFiles = updateReport.allFiles
buffered.debug(s"Files retrieved for ${prettyPrintDependency(module)}:")
buffered.debug(allFiles mkString ", ")
Right(allFiles)
}
}
}

View File

@ -30,18 +30,14 @@ class ZincComponentManager(globalLock: xsbti.GlobalLock,
/** Get all of the files for component 'id', throwing an exception if no files exist for the component. */
def files(id: String)(ifMissing: IfMissing): Iterable[File] = {
def fromSecondary =
lockSecondaryCache {
update(id)
getOrElse(createAndCache)
} getOrElse notFound
def notFound = invalid("Could not find required component '" + id + "'")
def getOrElse(orElse: => Iterable[File]): Iterable[File] = {
val existing = provider.component(id)
if (existing.isEmpty) orElse
else existing
}
def notFound = invalid("Could not find required component '" + id + "'")
def createAndCache =
def createAndCache = {
ifMissing match {
case IfMissing.Fail => notFound
case d: IfMissing.Define =>
@ -51,9 +47,31 @@ class ZincComponentManager(globalLock: xsbti.GlobalLock,
}
getOrElse(notFound)
}
lockLocalCache { getOrElse(fromSecondary) }
}
def fromSecondary: Iterable[File] = {
lockSecondaryCache {
update(id)
getOrElse(createAndCache)
}.getOrElse(notFound)
}
lockLocalCache(getOrElse(fromSecondary))
}
/** Get the file for component 'id', throwing an exception if no files or multiple files exist for the component. */
def file(id: String)(ifMissing: IfMissing): File = {
files(id)(ifMissing).toList match {
case x :: Nil => x
case xs =>
invalid("Expected single file for component '" + id + "', found: " + xs.mkString(", "))
}
}
/** Associate a component id to a series of jars. */
def define(id: String, files: Iterable[File]): Unit =
lockLocalCache(provider.defineComponent(id, files.toSeq.toArray))
/** This is used to lock the local cache in project/boot/. By checking the local cache first, we can avoid grabbing a global lock. */
private def lockLocalCache[T](action: => T): T = lock(provider.lockFile)(action)
@ -66,28 +84,16 @@ class ZincComponentManager(globalLock: xsbti.GlobalLock,
private def lock[T](file: File)(action: => T): T =
globalLock(file, new Callable[T] { def call = action })
/** Get the file for component 'id', throwing an exception if no files or multiple files exist for the component. */
def file(id: String)(ifMissing: IfMissing): File =
files(id)(ifMissing).toList match {
case x :: Nil => x
case xs =>
invalid("Expected single file for component '" + id + "', found: " + xs.mkString(", "))
}
private def invalid(msg: String) = throw new InvalidComponent(msg)
def define(id: String, files: Iterable[File]) = lockLocalCache {
provider.defineComponent(id, files.toSeq.toArray)
}
/** Retrieve the file for component 'id' from the secondary cache. */
private def update(id: String): Unit = {
secondaryCacheDir map { dir =>
secondaryCacheDir foreach { dir =>
val file = seondaryCacheFile(id, dir)
if (file.exists) {
define(id, Seq(file))
}
}
()
}
/** Install the files for component 'id' to the secondary cache. */
@ -107,20 +113,7 @@ class ZincComponentManager(globalLock: xsbti.GlobalLock,
new File(new File(dir, sbtOrg), fileName)
}
}
class InvalidComponent(msg: String, cause: Throwable) extends RuntimeException(msg, cause) {
def this(msg: String) = this(msg, null)
}
sealed trait IfMissing
object IfMissing {
def fail: IfMissing = Fail
/** f is expected to call ZincComponentManager.define. */
def define(useSecondaryCache: Boolean, f: => Unit): IfMissing = new Define(useSecondaryCache, f)
object Fail extends IfMissing
final class Define(val useSecondaryCache: Boolean, define: => Unit) extends IfMissing {
def run(): Unit = define
}
}
object ZincComponentManager {
lazy val (version, timestamp) = {
val properties = new java.util.Properties

View File

@ -48,14 +48,14 @@ abstract class BridgeProviderSpecification extends BaseIvySpecification {
val sourceModule = ModuleID(
xsbti.ArtifactInfo.SbtOrganization,
bridgeId,
ComponentCompiler.incrementalVersion).withConfigurations(Some("component")).sources()
ZincComponentCompiler.incrementalVersion).withConfigurations(Some("component")).sources()
val raw = new RawCompiler(instance, ClasspathOptionsUtil.auto, log)
val manager = new ZincComponentManager(lock, provider(targetDir), secondaryCacheOpt, log)
val componentCompiler =
new IvyComponentCompiler(raw, manager, ivyConfiguration, sourceModule, log)
new ZincComponentCompiler(raw, manager, ivyConfiguration, sourceModule, log)
val bridge = componentCompiler.apply()
val bridge = componentCompiler.getCompiledBridgeJar
val target = targetDir / s"target-bridge-$scalaVersion.jar"
IO.copyFile(bridge, target)
target

View File

@ -3,7 +3,7 @@ package sbt.internal.inc
import sbt.io.IO
import sbt.util.Logger
class IvyComponentCompilerSpec extends BridgeProviderSpecification {
class ZincComponentCompilerSpec extends BridgeProviderSpecification {
val scala210 = "2.10.5"
val scala211 = "2.11.8"