first part of semantic, multi-stage incremental compilation

This commit is contained in:
Mark Harrah 2010-06-27 09:18:35 -04:00
parent f9a7a0a28e
commit 9ad9df42b6
17 changed files with 939 additions and 34 deletions

62
compile/inc/APIs.scala Normal file
View File

@ -0,0 +1,62 @@
/* sbt -- Simple Build Tool
* Copyright 2010 Mark Harrah
*/
package sbt
package inc
import xsbti.api.Source
import java.io.File
import APIs.getAPI
trait APIs
{
/** The API for the source file `src` at the time represented by this instance.
* This method returns an empty API if the file had no API or is not known to this instance. */
def internalAPI(src: File): Source
/** The API for the external class `ext` at the time represented by this instance.
* This method returns an empty API if the file had no API or is not known to this instance. */
def externalAPI(ext: String): Source
def allExternals: collection.Set[String]
def allInternalSources: collection.Set[File]
def ++ (o: APIs): APIs
def markInternalSource(src: File, api: Source): APIs
def markExternalAPI(ext: String, api: Source): APIs
def removeInternal(remove: Iterable[File]): APIs
def filterExt(keep: String => Boolean): APIs
def internal: Map[File, Source]
def external: Map[String, Source]
}
object APIs
{
def apply(internal: Map[File, Source], external: Map[String, Source]): APIs = new MAPIs(internal, external)
def empty: APIs = apply(Map.empty, Map.empty)
val emptyAPI = new xsbti.api.Source(Array(), Array())
def getAPI[T](map: Map[T, Source], src: T): Source = map.getOrElse(src, emptyAPI)
}
private class MAPIs(val internal: Map[File, Source], val external: Map[String, Source]) extends APIs
{
def allInternalSources: collection.Set[File] = internal.keySet
def allExternals: collection.Set[String] = external.keySet
def ++ (o: APIs): APIs = new MAPIs(internal ++ o.internal, external ++ o.external)
def markInternalSource(src: File, api: Source): APIs =
new MAPIs(internal.updated(src, api), external)
def markExternalAPI(ext: String, api: Source): APIs =
new MAPIs(internal, external.updated(ext, api))
def removeInternal(remove: Iterable[File]): APIs = new MAPIs(internal -- remove, external)
def filterExt(keep: String => Boolean): APIs = new MAPIs(internal, external.filterKeys(keep))
def internalAPI(src: File) = getAPI(internal, src)
def externalAPI(ext: String) = getAPI(external, ext)
}

View File

@ -0,0 +1,55 @@
/* sbt -- Simple Build Tool
* Copyright 2010 Mark Harrah
*/
package sbt
package inc
import xsbti.api.Source
import java.io.File
trait Analysis
{
val stamps: Stamps
val apis: APIs
val relations: Relations
def ++(other: Analysis): Analysis
def -- (sources: Iterable[File]): Analysis
def copy(stamps: Stamps = stamps, apis: APIs = apis, relations: Relations = relations): Analysis
def addSource(src: File, api: Source, stamp: Stamp, internalDeps: Iterable[File]): Analysis
def addBinaryDep(src: File, dep: File, stamp: Stamp): Analysis
def addExternalDep(src: File, dep: String, api: Source): Analysis
def addProduct(src: File, product: File, stamp: Stamp): Analysis
}
object Analysis
{
lazy val Empty: Analysis = new MAnalysis(Stamps.empty, APIs.empty, Relations.empty)
}
private class MAnalysis(val stamps: Stamps, val apis: APIs, val relations: Relations) extends Analysis
{
def ++ (o: Analysis): Analysis = new MAnalysis(stamps ++ o.stamps, apis ++ o.apis, relations ++ o.relations)
def -- (sources: Iterable[File]): Analysis =
{
val newRelations = relations -- sources
def keep[T](f: (Relations, T) => Set[_]): T => Boolean = file => !f(newRelations, file).isEmpty
val newAPIs = apis.removeInternal(sources).filterExt( keep(_ usesExternal _) )
val newStamps = stamps.filter( keep(_ produced _), sources, keep(_ usesBinary _))
new MAnalysis(newStamps, newAPIs, newRelations)
}
def copy(stamps: Stamps, apis: APIs, relations: Relations): Analysis = new MAnalysis(stamps, apis, relations)
def addSource(src: File, api: Source, stamp: Stamp, internalDeps: Iterable[File]): Analysis =
copy( stamps.markInternalSource(src, stamp), apis.markInternalSource(src, api), relations.addInternalSrcDeps(src, internalDeps) )
def addBinaryDep(src: File, dep: File, stamp: Stamp): Analysis =
copy( stamps.markBinary(dep, stamp), apis, relations.addBinaryDep(src, dep) )
def addExternalDep(src: File, dep: String, depAPI: Source): Analysis =
copy( stamps, apis.markExternalAPI(dep, depAPI), relations.addExternalDep(src, dep) )
def addProduct(src: File, product: File, stamp: Stamp): Analysis =
copy( stamps.markProduct(product, stamp), apis, relations.addProduct(src, product) )
}

View File

@ -0,0 +1,33 @@
/* sbt -- Simple Build Tool
* Copyright 2010 Mark Harrah
*/
package sbt
package inc
trait AnalysisStore
{
def set(analysis: Analysis, setup: CompileSetup): Unit
def get(): (Analysis, CompileSetup)
}
object AnalysisStore
{
def cached(backing: AnalysisStore): AnalysisStore = new AnalysisStore {
private var last: Option[(Analysis, CompileSetup)] = None
def set(analysis: Analysis, setup: CompileSetup)
{
backing.set(analysis, setup)
last = Some( (analysis, setup) )
}
def get(): (Analysis, CompileSetup) =
{
if(last.isEmpty)
last = Some(backing.get())
last.get
}
}
def sync(backing: AnalysisStore): AnalysisStore = new AnalysisStore {
def set(analysis: Analysis, setup: CompileSetup): Unit = synchronized { backing.set(analysis, setup) }
def get(): (Analysis, CompileSetup) = synchronized { backing.get() }
}
}

24
compile/inc/Changes.scala Normal file
View File

@ -0,0 +1,24 @@
/* sbt -- Simple Build Tool
* Copyright 2010 Mark Harrah
*/
package sbt
package inc
import xsbt.api.NameChanges
import java.io.File
final case class InitialChanges(internalSrc: Changes[File], removedProducts: Set[File], binaryDeps: Set[File], external: APIChanges[String])
final case class APIChanges[T](modified: Set[T], names: NameChanges)
trait Changes[A]
{
def added: Set[A]
def removed: Set[A]
def changed: Set[A]
def unmodified: Set[A]
}
sealed abstract class Change(val file: File)
final class Removed(f: File) extends Change(f)
final class Added(f: File, newStamp: Stamp) extends Change(f)
final class Modified(f: File, oldStamp: Stamp, newStamp: Stamp) extends Change(f)

75
compile/inc/Compile.scala Normal file
View File

@ -0,0 +1,75 @@
/* sbt -- Simple Build Tool
* Copyright 2010 Mark Harrah
*/
package sbt
package inc
import xsbti.api.Source
import java.io.File
object IncrementalCompile
{
def apply(sources: Set[File], compile: (Set[File], xsbti.AnalysisCallback) => Unit, previous: Analysis, externalAPI: String => Source): Analysis =
{
val current = Stamps.initial(Stamp.exists, Stamp.hash, Stamp.lastModified)
val internalMap = (f: File) => previous.relations.produced(f).headOption
Incremental.compile(sources, previous, current, externalAPI, doCompile(compile, internalMap, current))
}
def doCompile(compile: (Set[File], xsbti.AnalysisCallback) => Unit, internalMap: File => Option[File], current: ReadStamps) = (srcs: Set[File]) => {
val callback = new AnalysisCallback(internalMap, current)
compile(srcs, callback)
callback.get
}
}
private final class AnalysisCallback(internalMap: File => Option[File], current: ReadStamps) extends xsbti.AnalysisCallback
{
import collection.mutable.{HashMap, HashSet, Map, Set}
private val apis = new HashMap[File, Source]
private val binaryDeps = new HashMap[File, Set[File]]
private val classes = new HashMap[File, Set[File]]
private val sourceDeps = new HashMap[File, Set[File]]
private def add[A,B](map: Map[A,Set[B]], a: A, b: B): Unit =
map.getOrElseUpdate(a, new HashSet[B]) += b
def sourceDependency(dependsOn: File, source: File) = add(sourceDeps, source, dependsOn)
def jarDependency(jar: File, source: File) = add(binaryDeps, source, jar)
def classDependency(clazz: File, source: File) = add(binaryDeps, source, clazz)
def productDependency(classFile: File, sourcePath: File) =
internalMap(classFile) match {
case Some(dependsOn) => sourceDependency(dependsOn, sourcePath)
case None => classDependency(classFile, sourcePath)
}
def generatedClass(source: File, module: File) = add(classes, source, module)
def api(sourceFile: File, source: Source) { apis(sourceFile) = source }
def endSource(sourcePath: File): Unit =
assert(apis.contains(sourcePath))
def get: Analysis = addBinaries( addProducts( addSources(Analysis.Empty) ) )
def addProducts(base: Analysis): Analysis = addAll(base, classes)( (a, src, prod) => a.addProduct(src, prod, current product prod ) )
def addBinaries(base: Analysis): Analysis = addAll(base, binaryDeps)( (a, src, bin) => a.addBinaryDep(src, bin, current binary bin) )
def addSources(base: Analysis): Analysis =
(base /: apis) { case (a, (src, api) ) =>
a.addSource(src, api, current.internalSource(src), sourceDeps.getOrElse(src, Nil: Iterable[File]))
}
def addAll[A,B](base: Analysis, m: Map[A, Set[B]])( f: (Analysis, A, B) => Analysis): Analysis =
(base /: m) { case (outer, (a, bs)) =>
(outer /: bs) { (inner, b) =>
f(inner, a, b)
} }
private def emptyS = new Array[String](0)
def superclassNames = emptyS
def annotationNames = emptyS
def superclassNotFound(superclassName: String) {}
def foundSubclass(source: File, subclassName: String, superclassName: String, isModule: Boolean) {}
def foundAnnotated(source: File, className: String, annotationName: String, isModule: Boolean) {}
def foundApplication(source: File, className: String) {}
def beginSource(source: File) {}
}

View File

@ -0,0 +1,44 @@
/* sbt -- Simple Build Tool
* Copyright 2010 Mark Harrah
*/
package sbt
package inc
import java.io.File
object CompileOrder extends Enumeration
{
val Mixed, JavaThenScala, ScalaThenJava = Value
}
// this class exists because of Scala's restriction on implicit parameter search.
// We cannot require an implicit parameter Equiv[Seq[String]] to construct Equiv[CompileSetup]
// because complexity(Equiv[Seq[String]]) > complexity(Equiv[CompileSetup])
// (6 > 4)
final class CompileOptions(val options: Seq[String])
final class CompileSetup(val outputDirectory: File, val options: CompileOptions, val compilerVersion: String, val order: CompileOrder.Value)
object CompileSetup
{
// Equiv[CompileOrder.Value] dominates Equiv[CompileSetup]
implicit def equivCompileSetup(implicit equivFile: Equiv[File], equivOpts: Equiv[CompileOptions], equivComp: Equiv[String]/*, equivOrder: Equiv[CompileOrder.Value]*/): Equiv[CompileSetup] = new Equiv[CompileSetup] {
def equiv(a: CompileSetup, b: CompileSetup) =
equivFile.equiv(a.outputDirectory, b.outputDirectory) &&
equivOpts.equiv(a.options, b.options) &&
equivComp.equiv(a.compilerVersion, b.compilerVersion) &&
a.order == b.order // equivOrder.equiv(a.order, b.order)
}
implicit val equivOutputDirectory: Equiv[File] = new Equiv[File] {
def equiv(a: File, b: File) = a.getAbsoluteFile == b.getAbsoluteFile
}
implicit val equivOpts: Equiv[CompileOptions] = new Equiv[CompileOptions] {
def equiv(a: CompileOptions, b: CompileOptions) = a.options sameElements b.options
}
implicit val equivCompilerVersion: Equiv[String] = new Equiv[String] {
def equiv(a: String, b: String) = a == b
}
implicit val equivOrder: Equiv[CompileOrder.Value] = new Equiv[CompileOrder.Value] {
def equiv(a: CompileOrder.Value, b: CompileOrder.Value) = a == b
}
}

View File

@ -0,0 +1,148 @@
/* sbt -- Simple Build Tool
* Copyright 2010 Mark Harrah
*/
package sbt
package inc
import xsbt.api.{NameChanges, SameAPI, TopLevel}
import annotation.tailrec
import xsbti.api.Source
import java.io.File
object Incremental
{
// TODO: the Analysis for the last successful compilation should get returned + Boolean indicating success
// TODO: full external name changes, scopeInvalidations
def compile(sources: Set[File], previous: Analysis, current: ReadStamps, externalAPI: String => Source, doCompile: Set[File] => Analysis)(implicit equivS: Equiv[Stamp]): Analysis =
{
def cycle(invalidated: Set[File], previous: Analysis): Analysis =
if(invalidated.isEmpty)
previous
else
{
val pruned = prune(invalidated, previous)
val fresh = doCompile(invalidated)
val merged = pruned ++ fresh//.copy(relations = pruned.relations ++ fresh.relations, apis = pruned.apis ++ fresh.apis)
val incChanges = changedIncremental(invalidated, previous.apis.internalAPI _, merged.apis.internalAPI _)
val incInv = invalidateIncremental(merged.relations, incChanges)
cycle(incInv, merged)
}
val initialChanges = changedInitial(sources, previous.stamps, previous.apis, current, externalAPI)
val initialInv = invalidateInitial(previous.relations, initialChanges)
cycle(initialInv, previous)
}
/**
* Accepts the sources that were recompiled during the last step and functions
* providing the API before and after the last step. The functions should return
* an empty API if the file did not/does not exist.
*/
def changedIncremental[T](lastSources: collection.Set[T], oldAPI: T => Source, newAPI: T => Source): APIChanges[T] =
{
val oldApis = lastSources map oldAPI
val newApis = lastSources map newAPI
val changes = (lastSources, oldApis, newApis).zipped.filter { (src, oldApi, newApi) => SameAPI(oldApi, newApi) }
val changedNames = TopLevel.nameChanges(changes._3, changes._2 )
val modifiedAPIs = changes._1.toSet
APIChanges(modifiedAPIs, changedNames)
}
def changedInitial(sources: Set[File], previous: Stamps, previousAPIs: APIs, current: ReadStamps, externalAPI: String => Source)(implicit equivS: Equiv[Stamp]): InitialChanges =
{
val srcChanges = changes(previous.allInternalSources.toSet, sources, f => !equivS.equiv( previous.internalSource(f), current.internalSource(f) ) )
val removedProducts = previous.allProducts.filter( p => !equivS.equiv( previous.product(p), current.product(p) ) ).toSet
val binaryDepChanges = previous.allBinaries.filter( f => !equivS.equiv( previous.binary(f), current.binary(f) ) ).toSet
val extChanges = changedIncremental(previousAPIs.allExternals, previousAPIs.externalAPI, externalAPI)
InitialChanges(srcChanges, removedProducts, binaryDepChanges, extChanges )
}
def changes(previous: Set[File], current: Set[File], existingModified: File => Boolean): Changes[File] =
new Changes[File]
{
private val inBoth = previous & current
val removed = previous -- inBoth
val added = current -- inBoth
val (changed, unmodified) = inBoth.partition(existingModified)
}
def invalidateIncremental(previous: Relations, changes: APIChanges[File]): Set[File] =
invalidateTransitive(previous.internalSrcDeps _, changes.modified )// ++ scopeInvalidations(previous.extAPI _, changes.modified, changes.names)
/** Only invalidates direct source dependencies. It excludes any sources that were recompiled during the previous run.
* Callers may want to augment the returned set with 'modified' or even all sources recompiled up to this point. */
def invalidateDirect(sourceDeps: File => Set[File], modified: Set[File]): Set[File] =
(modified flatMap sourceDeps) -- modified
/** Invalidates transitive source dependencies including `modified`. It excludes any sources that were recompiled during the previous run.*/
@tailrec def invalidateTransitive(sourceDeps: File => Set[File], modified: Set[File]): Set[File] =
{
val newInv = invalidateDirect(sourceDeps, modified)
if(newInv.isEmpty) modified else invalidateTransitive(sourceDeps, modified ++ newInv)
}
/** Invalidates sources based on initially detected 'changes' to the sources, products, and dependencies.*/
def invalidateInitial(previous: Relations, changes: InitialChanges): Set[File] =
{
val srcChanges = changes.internalSrc
val srcDirect = srcChanges.removed.flatMap(previous.usesInternalSrc) ++ srcChanges.added ++ srcChanges.changed
val byProduct = changes.removedProducts.flatMap(previous.produced)
val byBinaryDep = changes.binaryDeps.flatMap(previous.usesBinary)
val byExtSrcDep = changes.external.modified.flatMap(previous.usesExternal) // ++ scopeInvalidations
srcDirect ++ byProduct ++ byBinaryDep ++ byExtSrcDep
}
def prune(invalidatedSrcs: Set[File], previous: Analysis): Analysis =
{
IO.delete( invalidatedSrcs.flatMap(previous.relations.products) )
previous -- invalidatedSrcs
}
// unmodifiedSources should not contain any sources in the previous compilation run
// (this may unnecessarily invalidate them otherwise)
/*def scopeInvalidation(previous: Analysis, otherSources: Set[File], names: NameChanges): Set[File] =
{
val newNames = newTypes ++ names.newTerms
val newMap = pkgNameMap(newNames)
otherSources filter { src => scopeAffected(previous.extAPI(src), previous.srcDependencies(src), newNames, newMap) }
}
def scopeAffected(api: Source, srcDependencies: Iterable[Source], newNames: Set[String], newMap: Map[String, List[String]]): Boolean =
collisions_?(TopLevel.names(api.definitions), newNames) ||
pkgs(api) exists {p => shadowed_?(p, srcDependencies, newMap) }
def collisions_?(existing: Set[String], newNames: Map[String, List[String]]): Boolean =
!(existing ** newNames).isEmpty
// A proper implementation requires the actual symbol names used. This is a crude approximation in the meantime.
def shadowed_?(fromPkg: List[String], srcDependencies: Iterable[Source], newNames: Map[String, List[String]]): Boolean =
{
lazy val newPN = newNames.filter { pn => properSubPkg(fromPkg, pn._2) }
def isShadowed(usedName: String): Boolean =
{
val (usedPkg, name) = pkgAndName(usedName)
newPN.get(name).forall { nPkg => properSubPkg(usedPkg, nPkg) }
}
val usedNames = TopLevel.names(srcDependencies) // conservative approximation of referenced top-level names
usedNames exists isShadowed
}
def pkgNameMap(names: Iterable[String]): Map[String, List[String]] =
(names map pkgAndName).toMap
def pkgAndName(s: String) =
{
val period = s.lastIndexOf('.')
if(period < 0) (Nil, s) else (s.substring(0, period).split("\\."), s.substring(period+1))
}
def pkg(s: String) = pkgAndName(s)._1
def properSubPkg(testParent: Seq[String], testSub: Seq[String]) = testParent.length < testSub.length && testSub.startsWith(testParent)
def pkgs(api: Source) = names(api :: Nil).map(pkg)*/
}

79
compile/inc/Locate.scala Normal file
View File

@ -0,0 +1,79 @@
/* sbt -- Simple Build Tool
* Copyright 2010 Mark Harrah
*/
package sbt
package inc
import java.io.File
import java.util.zip.ZipFile
import Function.const
object Locate
{
/** Right(src) provides the value for the found class
* Left(true) means that the class was found, but it had no associated value
* Left(false) means that the class was not found */
def value[S](classpath: Seq[File], get: File => String => Option[S]): String => Either[Boolean, S] =
{
val gets = classpath.toStream.map(getValue(get))
className => find(className, gets)
}
def find[S](name: String, gets: Stream[String => Either[Boolean, S]]): Either[Boolean, S] =
if(gets.isEmpty)
Left(false)
else
gets.head(name) match
{
case Left(false) => find(name, gets.tail)
case x => x
}
def getValue[S](get: File => String => Option[S])(entry: File): String => Either[Boolean, S] =
{
val defClass = definesClass(entry)
val getF = get(entry)
className => if(defClass(className)) getF(className).toRight(true) else Left(false)
}
def definesClass(entry: File): String => Boolean =
if(entry.isDirectory)
directoryDefinesClass(entry)
else if(entry.exists)
jarDefinesClass(entry)
else
const(false)
def jarDefinesClass(entry: File): String => Boolean =
{
import collection.JavaConversions._
val jar = new ZipFile(entry, ZipFile.OPEN_READ)
val entries = try { jar.entries.map(e => toClassName(e.getName)).toSet } finally { jar.close() }
entries.contains _
}
def toClassName(entry: String): String =
entry.stripSuffix(ClassExt).replace('/', '.')
val ClassExt = ".class"
def directoryDefinesClass(entry: File): String => Boolean =
className => classFile(entry, className).isFile
def classFile(baseDir: File, className: String): File =
{
val (pkg, name) = components(className)
val dir = subDirectory(baseDir, pkg)
new File(dir, name + ClassExt)
}
def subDirectory(base: File, parts: Seq[String]): File =
(base /: parts) ( (b, p) => new File(b,p) )
def components(className: String): (Seq[String], String) =
{
assume(!className.isEmpty)
val parts = className.split("\\.")
if(parts.length == 1) (Nil, parts(0)) else (parts.init, parts.last)
}
}

105
compile/inc/Relations.scala Normal file
View File

@ -0,0 +1,105 @@
/* sbt -- Simple Build Tool
* Copyright 2010 Mark Harrah
*/
package sbt
package inc
import xsbti.api.Source
import java.io.File
trait Relations
{
/** All sources _with at least one product_ . */
def allSources: collection.Set[File]
def allProducts: collection.Set[File]
def allBinaryDeps: collection.Set[File]
def allInternalSrcDeps: collection.Set[File]
def allExternalDeps: collection.Set[String]
def products(src: File): Set[File]
def produced(prod: File): Set[File]
def binaryDeps(src: File): Set[File]
def usesBinary(dep: File): Set[File]
def internalSrcDeps(src: File): Set[File]
def usesInternalSrc(dep: File): Set[File]
def externalDeps(src: File): Set[String]
def usesExternal(dep: String): Set[File]
def addProduct(src: File, prod: File): Relations
def addExternalDep(src: File, dependsOn: String): Relations
def addInternalSrcDeps(src: File, dependsOn: Iterable[File]): Relations
def addBinaryDep(src: File, dependsOn: File): Relations
def ++ (o: Relations): Relations
def -- (sources: Iterable[File]): Relations
def srcProd: Relation[File, File]
def binaryDep: Relation[File, File]
def internalSrcDep: Relation[File, File]
def externalDep: Relation[File, String]
}
object Relations
{
lazy val e = Relation.empty[File, File]
def empty: Relations = new MRelations(e, e, e, Relation.empty[File, String])
def make(srcProd: Relation[File, File], binaryDep: Relation[File, File], internalSrcDep: Relation[File, File], externalDep: Relation[File, String]): Relations =
new MRelations(srcProd, binaryDep, internalSrcDep, externalDep)
}
/**
* `srcProd` is a relation between a source file and a product: (source, product).
* Note that some source files may not have a product and will not be included in this relation.
*
* `binaryDeps` is a relation between a source file and a binary dependency: (source, binary dependency).
* This only includes dependencies on classes and jars that do not have a corresponding source/API to track instead.
* A class or jar with a corresponding source should only be tracked in one of the source dependency relations.
*
* `internalSrcDeps` is a relation between a source file and a source dependency in the same compilation group.
* Dependencies on sources in other projects belong in external source dependencies.
*
* `externalSrcDeps` is a relation between a source file and a source dependency in another compilation group.
* Dependencies on sources in the same group belong in internal source dependencies.
*/
private class MRelations(val srcProd: Relation[File, File], val binaryDep: Relation[File, File],
val internalSrcDep: Relation[File, File], val externalDep: Relation[File, String]) extends Relations
{
def allSources: collection.Set[File] = srcProd._1s
def allProducts: collection.Set[File] = srcProd._2s
def allBinaryDeps: collection.Set[File] = binaryDep._2s
def allInternalSrcDeps: collection.Set[File] = internalSrcDep._2s
def allExternalDeps: collection.Set[String] = externalDep._2s
def products(src: File): Set[File] = srcProd.forward(src)
def produced(prod: File): Set[File] = srcProd.reverse(prod)
def binaryDeps(src: File): Set[File] = binaryDep.forward(src)
def usesBinary(dep: File): Set[File] = binaryDep.reverse(dep)
def internalSrcDeps(src: File): Set[File] = internalSrcDep.forward(src)
def usesInternalSrc(dep: File): Set[File] = internalSrcDep.reverse(dep)
def externalDeps(src: File): Set[String] = externalDep.forward(src)
def usesExternal(dep: String): Set[File] = externalDep.reverse(dep)
def addProduct(src: File, prod: File): Relations =
new MRelations( srcProd + (src, prod), binaryDep, internalSrcDep, externalDep )
def addExternalDep(src: File, dependsOn: String): Relations =
new MRelations( srcProd, binaryDep, internalSrcDep, externalDep + (src, dependsOn) )
def addInternalSrcDeps(src: File, dependsOn: Iterable[File]): Relations =
new MRelations( srcProd, binaryDep, internalSrcDep + (src, dependsOn ), externalDep )
def addBinaryDep(src: File, dependsOn: File): Relations =
new MRelations( srcProd, binaryDep + (src, dependsOn), internalSrcDep, externalDep )
def ++ (o: Relations): Relations =
new MRelations(srcProd ++ o.srcProd, binaryDep ++ o.binaryDep, internalSrcDep ++ o.internalSrcDep, externalDep ++ o.externalDep)
def -- (sources: Iterable[File]) =
new MRelations(srcProd -- sources, binaryDep -- sources, internalSrcDep -- sources, externalDep -- sources)
}

120
compile/inc/Stamp.scala Normal file
View File

@ -0,0 +1,120 @@
/* sbt -- Simple Build Tool
* Copyright 2010 Mark Harrah
*/
package sbt
package inc
import xsbti.api.Source
import java.io.{File, IOException}
import Stamp.getStamp
trait ReadStamps
{
/** The Stamp for the given product at the time represented by this Stamps instance.*/
def product(prod: File): Stamp
/** The Stamp for the given source file at the time represented by this Stamps instance.*/
def internalSource(src: File): Stamp
/** The Stamp for the given binary dependency at the time represented by this Stamps instance.*/
def binary(bin: File): Stamp
}
/** Provides information about files as they were at a specific time.*/
trait Stamps extends ReadStamps
{
def allInternalSources: collection.Set[File]
def allBinaries: collection.Set[File]
def allProducts: collection.Set[File]
def sources: Map[File, Stamp]
def binaries: Map[File, Stamp]
def products: Map[File, Stamp]
def markInternalSource(src: File, s: Stamp): Stamps
def markBinary(bin: File, s: Stamp): Stamps
def markProduct(prod: File, s: Stamp): Stamps
def filter(prod: File => Boolean, removeSources: Iterable[File], bin: File => Boolean): Stamps
def ++ (o: Stamps): Stamps
}
sealed trait Stamp
final class Hash(val value: Array[Byte]) extends Stamp
final class LastModified(val value: Long) extends Stamp
final class Exists(val value: Boolean) extends Stamp
object Stamp
{
implicit val equivStamp: Equiv[Stamp] = new Equiv[Stamp] {
def equiv(a: Stamp, b: Stamp) = (a,b) match {
case (h1: Hash, h2: Hash) => h1.value sameElements h2.value
case (e1: Exists, e2: Exists) => e1.value == e2.value
case (lm1: LastModified, lm2: LastModified) => lm1.value == lm2.value
case _ => false
}
}
val hash = (f: File) => tryStamp(new Hash(Hash(f)))
val lastModified = (f: File) => tryStamp(new LastModified(f.lastModified))
val exists = (f: File) => tryStamp(if(f.exists) present else notPresent)
def tryStamp(g: => Stamp): Stamp = try { g } catch { case i: IOException => notPresent }
val notPresent = new Exists(false)
val present = new Exists(true)
def getStamp(map: Map[File, Stamp], src: File): Stamp = map.getOrElse(src, notPresent)
}
object Stamps
{
def initial(prodStamp: File => Stamp, srcStamp: File => Stamp, binStamp: File => Stamp): ReadStamps = new InitialStamps(prodStamp, srcStamp, binStamp)
def empty: Stamps =
{
val eSt = Map.empty[File, Stamp]
apply(eSt, eSt, eSt)
}
def apply(products: Map[File, Stamp], sources: Map[File, Stamp], binaries: Map[File, Stamp]): Stamps =
new MStamps(products, sources, binaries)
}
private class MStamps(val products: Map[File, Stamp], val sources: Map[File, Stamp], val binaries: Map[File, Stamp]) extends Stamps
{
def allInternalSources: collection.Set[File] = sources.keySet
def allBinaries: collection.Set[File] = binaries.keySet
def allProducts: collection.Set[File] = products.keySet
def ++ (o: Stamps): Stamps =
new MStamps(products ++ o.products, sources ++ o.sources, binaries ++ o.binaries)
def markInternalSource(src: File, s: Stamp): Stamps =
new MStamps(products, sources.updated(src, s), binaries)
def markBinary(bin: File, s: Stamp): Stamps =
new MStamps(products, sources, binaries.updated(bin, s))
def markProduct(prod: File, s: Stamp): Stamps =
new MStamps(products.updated(prod, s), sources, binaries)
def filter(prod: File => Boolean, removeSources: Iterable[File], bin: File => Boolean): Stamps =
new MStamps(products.filterKeys(prod), sources -- removeSources, binaries.filterKeys(bin))
def product(prod: File) = getStamp(products, prod)
def internalSource(src: File) = getStamp(sources, src)
def binary(bin: File) = getStamp(binaries, bin)
}
private class InitialStamps(prodStamp: File => Stamp, srcStamp: File => Stamp, binStamp: File => Stamp) extends ReadStamps
{
import collection.mutable.{HashMap, Map}
// cached stamps
private val products: Map[File, Stamp] = new HashMap
private val sources: Map[File, Stamp] = new HashMap
private val binaries: Map[File, Stamp] = new HashMap
def product(prod: File): Stamp = synchronized { products.getOrElseUpdate(prod, prodStamp(prod)) }
def internalSource(src: File): Stamp = synchronized { sources.getOrElseUpdate(src, srcStamp(src)) }
def binary(bin: File): Stamp = synchronized { binaries.getOrElseUpdate(bin, binStamp(bin)) }
}

11
compile/inc/notes Normal file
View File

@ -0,0 +1,11 @@
each compilation group gets an Analysis
an sbt-style project could have multiple compilation groups or there could be multiple projects per compilation group.
Tradiationally, there has been a main group and a test group.
Each Analysis is associated with one or more classpath entries. Typically, it will be associated with the output directory and/or any artifacts produced from that output directory.
Need to be able to process Source and extract classes {C} s.t. C <: D for D in {D} or C or one of its non-private methods is annotated with annotation A in {A} or C.main(args: Array[String]) exists
For Java sources, need to write a (File, Set[File]) => Source function that reads an API from a class file. The compile function passed to IncrementalCompile needs to handle compiling Java sources in the proper order
Need to handle entries removed from classpath. Could be done similarly to how Locate is used for getting the API for a dependency. In this case, we'd get the Stamp for a binary dependency.

View File

@ -0,0 +1,68 @@
/* sbt -- Simple Build Tool
* Copyright 2010 Mark Harrah
*/
package sbt
import inc._
import java.io.File
import xsbt.{AnalyzingCompiler, CompileLogger, CompilerArguments}
import xsbti.api.Source
import xsbti.AnalysisCallback
import CompileSetup._
import sbinary.DefaultProtocol.{ immutableMapFormat, immutableSetFormat, StringFormat }
final class CompileConfiguration(val sources: Seq[File], val classpath: Seq[File], val previousAnalysis: Analysis,
val previousSetup: CompileSetup, val currentSetup: CompileSetup, val getAnalysis: File => Option[Analysis],
val maxErrors: Int, val compiler: AnalyzingCompiler)
class AggressiveCompile(cacheDirectory: File)
{
def apply(sources: Seq[File], classpath: Seq[File], outputDirectory: File, options: Seq[String], compiler: AnalyzingCompiler, log: CompileLogger): Analysis =
{
val setup = new CompileSetup(outputDirectory, new CompileOptions(options), compiler.scalaInstance.actualVersion, CompileOrder.Mixed)
compile1(sources, classpath, setup, store, Map.empty, compiler, log)
}
def withBootclasspath(args: CompilerArguments, classpath: Seq[File]): Seq[File] =
args.bootClasspath ++ classpath
def compile1(sources: Seq[File], classpath: Seq[File], setup: CompileSetup, store: AnalysisStore, analysis: Map[File, Analysis], compiler: AnalyzingCompiler, log: CompileLogger): Analysis =
{
val (previousAnalysis, previousSetup) = store.get()
val config = new CompileConfiguration(sources, classpath, previousAnalysis, previousSetup, setup, analysis.get _, 100, compiler)
val result = compile2(config, log)
store.set(result, setup)
result
}
def compile2(config: CompileConfiguration, log: CompileLogger)(implicit equiv: Equiv[CompileSetup]): Analysis =
{
import config._
import currentSetup._
val getAPI = (f: File) => {
val extApis = getAnalysis(f) match { case Some(a) => a.apis.external; case None => Map.empty[String, Source] }
extApis.get _
}
val apiOrEmpty = (api: Either[Boolean, Source]) => api.right.toOption.getOrElse( APIs.emptyAPI )
val cArgs = new CompilerArguments(compiler.scalaInstance, compiler.cp)
val externalAPI = apiOrEmpty compose Locate.value(withBootclasspath(cArgs, classpath), getAPI)
val compile0 = (include: Set[File], callback: AnalysisCallback) => {
val arguments = cArgs(sources.filter(include), classpath, outputDirectory, options.options)
compiler.compile(arguments, callback, maxErrors, log)
}
val sourcesSet = sources.toSet
val analysis = if(equiv.equiv(previousSetup, currentSetup)) previousAnalysis else Incremental.prune(sourcesSet, previousAnalysis)
IncrementalCompile(sourcesSet, compile0, analysis, externalAPI)
}
import AnalysisFormats._
// The following intermediate definitions are needed because of Scala's implicit parameter rules.
// implicit def a(implicit b: T[Int]): S = ...
// triggers a divierging expansion because T[Int] dominates S, even though they are unrelated
implicit val r = relationFormat[File,File]
implicit val map = immutableMapFormat[File, Stamp]
implicit val rF = relationsFormat(r,r,r, relationFormat[File, String])
implicit val aF = analysisFormat(stampsFormat, apisFormat, rF)
val store = AnalysisStore.sync(AnalysisStore.cached(FileBasedStore(cacheDirectory)(aF, setupFormat)))
}

View File

@ -1,44 +1,41 @@
/* sbt -- Simple Build Tool
* Copyright 2010 Mark Harrah
*/
package xsbt
package sbt
import xsbt.{AnalyzingCompiler, CompileLogger, ScalaInstance}
import java.io.File
import System.{currentTimeMillis => now}
import Path._
import GlobFilter._
class AggressiveCompiler extends xsbti.AppMain
{
final def run(configuration: xsbti.AppConfiguration): xsbti.MainResult =
{
System.setProperty("sbt.api.enable", "true")
val args = configuration.arguments.map(_.trim).toList
readLine("Press enter to compile... ")
val start = System.currentTimeMillis
val start = now
val success = run(args, configuration.baseDirectory, configuration.provider)
val end = System.currentTimeMillis
println("Compiled in " + ((end - start) / 1000.0) + " s")
println("Compiled in " + ((now - start) / 1000.0) + " s")
run(configuration)
}
def run(args: List[String], cwd: File, app: xsbti.AppProvider): Boolean =
def run(args: List[String], cwd: Path, app: xsbti.AppProvider): Boolean =
{
import Paths._
import GlobFilter._
val launcher = app.scalaProvider.launcher
val sources = Task(cwd ** "*.scala")
val outputDirectory = Task(cwd / "target" / "classes")
val classpath = outputDirectory map { _ ++ (cwd * "*.jar") ++(cwd * (-"project")).descendentsExcept( "*.jar", "project" || HiddenFileFilter) }
val cacheDirectory = cwd / "target" / "cache"
val options = Task(args.tail.toSeq)
val log = new ConsoleLogger with CompileLogger with sbt.IvyLogger { def verbose(msg: => String) = debug(msg) }
val componentManager = new sbt.ComponentManager(launcher.globalLock, app.components, log)
val compiler = Task(new AnalyzingCompiler(ScalaInstance(args.head, launcher), componentManager, log))
val compileTask = AggressiveCompile(sources, classpath, outputDirectory, options, cacheDirectory, compiler, log)
try { TaskRunner(compileTask.task); true }
catch
{
case w: TasksFailed => w.failures.foreach { f => handleException(f.exception) }; false
case e: Exception => handleException(e); false
}
val sources = cwd ** "*.scala"
val target = cwd / "target"
val outputDirectory = target / "classes"
val classpath = outputDirectory +++ (cwd * "*.jar") +++(cwd * (-"project")).descendentsExcept( "*.jar", "project" || HiddenFileFilter)
val cacheDirectory = target / "cache"
val options = args.tail.toSeq
val log = new ConsoleLogger with CompileLogger with sbt.IvyLogger
val componentManager = new ComponentManager(launcher.globalLock, app.components, log)
val compiler = new AnalyzingCompiler(ScalaInstance(args.head, launcher), componentManager, log)
val agg = new AggressiveCompile(cacheDirectory)
try { agg(sources.get.toSeq, classpath.get.toSeq, outputDirectory, options, compiler, log); true }
catch { case e: Exception => handleException(e); false }
}
def handleException(e: Throwable) =
{

View File

@ -0,0 +1,54 @@
/* sbt -- Simple Build Tool
* Copyright 2010 Mark Harrah
*/
package sbt
import inc._
import xsbti.api.Source
import xsbt.api.APIFormat
import java.io.File
import sbinary._
import DefaultProtocol._
object AnalysisFormats
{
type RFF = Relation[File, File]
type RFS = Relation[File, String]
implicit def analysisFormat(implicit stampsF: Format[Stamps], apisF: Format[APIs], relationsF: Format[Relations]): Format[Analysis] =
asProduct3( Analysis.Empty.copy _)( a => (a.stamps, a.apis, a.relations))(stampsF, apisF, relationsF)
implicit def setupFormat(implicit outDirF: Format[File], optionF: Format[CompileOptions], compilerVersion: Format[String], orderF: Format[CompileOrder.Value]): Format[CompileSetup] =
asProduct4[CompileSetup, File, CompileOptions, String, CompileOrder.Value]( (a,b,c,d) => new CompileSetup(a,b,c,d) )(s => (s.outputDirectory, s.options, s.compilerVersion, s.order))(outDirF, optionF, compilerVersion, orderF)
implicit def stampsFormat(implicit prodF: Format[Map[File, Stamp]], srcF: Format[Map[File, Stamp]], binF: Format[Map[File, Stamp]]): Format[Stamps] =
asProduct3( Stamps.apply _ )( s => (s.products, s.sources, s.binaries) )(prodF, srcF, binF)
implicit def stampFormat(implicit hashF: Format[Hash], modF: Format[LastModified], existsF: Format[Exists]): Format[Stamp] =
asUnion(hashF, modF, existsF)
implicit def apisFormat(implicit internalF: Format[Map[File, Source]], externalF: Format[Map[String, Source]]): Format[APIs] =
asProduct2( APIs.apply _)( as => (as.internal, as.external) )(internalF, externalF)
implicit def relationsFormat(implicit prodF: Format[RFF], binF: Format[RFF], intF: Format[RFF], extF: Format[RFS]): Format[Relations] =
asProduct4[Relations, RFF, RFF, RFF, RFS]( (a,b,c,d) => Relations.make(a,b,c,d) )( rs => (rs.srcProd, rs.binaryDep, rs.internalSrcDep, rs.externalDep) )(prodF, binF, intF, extF)
implicit def relationFormat[A,B](implicit af: Format[Map[A, Set[B]]], bf: Format[Map[B, Set[A]]]): Format[Relation[A,B]] =
asProduct2[Relation[A,B], Map[A, Set[B]], Map[B, Set[A]]]( Relation.make _ )( r => (r.forwardMap, r.reverseMap) )(af, bf)
implicit val sourceFormat: Format[Source] =
wrap[Source, Array[Byte]]( APIFormat.write _, APIFormat.read _)
implicit def fileFormat: Format[File] = wrap[File, String](_.getAbsolutePath, s => new File(s))
// can't require Format[Seq[String]] because its complexity is higher than Format[CompileOptions]
implicit def optsFormat(implicit strF: Format[String]): Format[CompileOptions] =
wrap[CompileOptions, Seq[String]](_.options, os => new CompileOptions(os))(seqFormat[String])
implicit val orderFormat: Format[CompileOrder.Value] = enumerationFormat(CompileOrder)
implicit def seqFormat[T](implicit optionFormat: Format[T]): Format[Seq[T]] = viaSeq[Seq[T], T](x => x)
implicit def hashStampFormat: Format[Hash] = wrap[Hash, Array[Byte]](_.value, new Hash(_))
implicit def lastModFormat: Format[LastModified] = wrap[LastModified, Long](_.value, new LastModified(_))
implicit def existsFormat: Format[Exists] = wrap[Exists, Boolean](_.value, new Exists(_))
}

27
main/FileBasedStore.scala Normal file
View File

@ -0,0 +1,27 @@
/* sbt -- Simple Build Tool
* Copyright 2010 Mark Harrah
*/
package sbt
import inc.{Analysis, AnalysisStore, CompileSetup}
import java.io.File
import sbinary._
import Operations.{read, write}
import DefaultProtocol._
import JavaIO._
object FileBasedStore
{
def apply(file: File)(implicit analysisF: Format[Analysis], setupF: Format[CompileSetup]): AnalysisStore = new AnalysisStore {
def set(analysis: Analysis, setup: CompileSetup): Unit =
Using.fileOutputStream()(file) { out =>
write[(Analysis, CompileSetup)](out, (analysis, setup) )
}
def get(): (Analysis, CompileSetup) =
Using.fileInputStream(file) { in =>
read[(Analysis, CompileSetup)]( in )
}
}
}

View File

@ -1,14 +1,13 @@
[scala]
version: 2.7.7
version: 2.8.0.RC6
[app]
org: org.scala-tools.sbt
name: alternate-compiler-test
version: 0.6.12-SNAPSHOT
class: xsbt.AggressiveCompiler
version: 0.9.0-SNAPSHOT
class: sbt.AggressiveCompiler
components: xsbti
cross-versioned: true
resources: project/boot/sbinary.jar
[repositories]
local

View File

@ -23,19 +23,21 @@ class XSbt(info: ProjectInfo) extends ParentProject(info) with NoCrossPaths
val logSub = project(utilPath / "log", "Logging", new LogProject(_), interfaceSub)
val datatypeSub = baseProject("util" /"datatype", "Datatype Generator", ioSub)
val testSub = project("scripted", "Test", new TestProject(_), ioSub)
val compileInterfaceSub = project(compilePath / "interface", "Compiler Interface", new CompilerInterfaceProject(_), interfaceSub)
val compileIncrementalSub = project(compilePath / "inc", "Incremental Compiler", new IncrementalProject(_), collectionSub, apiSub, ioSub)
val compilerSub = project(compilePath, "Compile", new CompileProject(_),
launchInterfaceSub, interfaceSub, ivySub, ioSub, classpathSub, compileInterfaceSub)
val taskSub = project(tasksPath, "Tasks", new TaskProject(_), controlSub, collectionSub)
val cacheSub = project(cachePath, "Cache", new CacheProject(_), ioSub, collectionSub)
/** following are not updated for 2.8 or 0.9 */
val testSub = project("scripted", "Test", new TestProject(_), ioSub)
val trackingSub = baseProject(cachePath / "tracking", "Tracking", cacheSub)
val compilerSub = project(compilePath, "Compile", new CompileProject(_),
launchInterfaceSub, interfaceSub, ivySub, ioSub, classpathSub, compileInterfaceSub)
val stdTaskSub = project(tasksPath / "standard", "Standard Tasks", new StandardTaskProject(_), trackingSub, taskSub, compilerSub, apiSub)
val altCompilerSub = baseProject("main", "Alternate Compiler Test", stdTaskSub, logSub)
val altCompilerSub = project("main", "Alternate Compiler Test", new AlternateProject(_), compileIncrementalSub, compilerSub, ioSub, logSub)
val sbtSub = project(sbtPath, "Simple Build Tool", new SbtProject(_) {}, compilerSub, launchInterfaceSub)
val installerSub = project(sbtPath / "install", "Installer", new InstallerProject(_) {}, sbtSub)
@ -105,7 +107,9 @@ class XSbt(info: ProjectInfo) extends ParentProject(info) with NoCrossPaths
class TaskProject(info: ProjectInfo) extends Base(info) with TestDependencies
class ClassfileProject(info: ProjectInfo) extends Base(info) with TestDependencies
class CompletionProject(info: ProjectInfo) extends Base(info) with TestDependencies
class CacheProject(info: ProjectInfo) extends Base(info)
class CacheProject(info: ProjectInfo) extends Base(info) with SBinaryDep
class AlternateProject(info: ProjectInfo) extends Base(info) with SBinaryDep
trait SBinaryDep extends BasicManagedProject
{
// these compilation options are useful for debugging caches and task composition
//override def compileOptions = super.compileOptions ++ List(Unchecked,ExplainTypes, CompileOption("-Xlog-implicits"))