* Basic API serialization

* Fixes to API extraction and equality checking
 * Reworked tracking
 * New compile infrastructure based on API changes
 * Example application for testing
This commit is contained in:
Mark Harrah 2010-01-05 19:50:43 -05:00
parent 833688cdd9
commit 6f6b795b39
15 changed files with 500 additions and 171 deletions

View File

@ -1,3 +1,6 @@
/* sbt -- Simple Build Tool
* Copyright 2009, 2010 Mark Harrah
*/
package xsbt
object ChangeReport

View File

@ -1,3 +1,6 @@
/* sbt -- Simple Build Tool
* Copyright 2009, 2010 Mark Harrah
*/
package xsbt
private object DependencyTracking
@ -16,14 +19,22 @@ trait UpdateTracking[T] extends NotNull
def product(source: T, output: T): Unit
def tag(source: T, t: Array[Byte]): Unit
def read: ReadTracking[T]
// removes files from all maps, both keys and values
def removeAll(files: Iterable[T]): Unit
// removes sources as keys/values in source, product maps and as values in reverseDependencies map
def pending(sources: Iterable[T]): Unit
}
import scala.collection.Set
trait ReadTracking[T] extends NotNull
{
def isProduct(file: T): Boolean
def isSource(file: T): Boolean
def isUsed(file: T): Boolean
def dependsOn(file: T): Set[T]
def products(file: T): Set[T]
def sources(file: T): Set[T]
def usedBy(file: T): Set[T]
def tag(file: T): Array[Byte]
def allProducts: Set[T]
def allSources: Set[T]
def allUsed: Set[T]
@ -42,6 +53,7 @@ private final class DefaultTracking[T](translateProducts: Boolean)
val productMap: DMap[T] = forward(sourceMap) // map from a source to its products. Keep in sync with sourceMap
}
// if translateProducts is true, dependencies on a product are translated to dependencies on a source
// if there is a source recorded as generating that product
private abstract class DependencyTracking[T](translateProducts: Boolean) extends ReadTracking[T] with UpdateTracking[T]
{
val reverseDependencies: DMap[T] // map from a file to the files that depend on it
@ -58,11 +70,17 @@ private abstract class DependencyTracking[T](translateProducts: Boolean) extends
final def usedBy(file: T): Set[T] = get(reverseUses, file)
final def tag(file: T): Array[Byte] = tagMap.getOrElse(file, new Array[Byte](0))
def isProduct(file: T): Boolean = exists(sourceMap, file)
def isSource(file: T): Boolean = exists(productMap, file)
def isUsed(file: T): Boolean = exists(reverseUses, file)
final def allProducts = Set() ++ sourceMap.keys
final def allSources = Set() ++ productMap.keys
final def allUsed = Set() ++ reverseUses.keys
final def allTags = tagMap.toSeq
private def exists(map: DMap[T], value: T): Boolean = map.contains(value)
private def get(map: DMap[T], value: T): Set[T] = map.getOrElse(value, Set.empty[T])
final def dependency(sourceFile: T, dependsOn: T)
@ -82,22 +100,38 @@ private abstract class DependencyTracking[T](translateProducts: Boolean) extends
final def use(sourceFile: T, usesFile: T) { reverseUses.add(usesFile, sourceFile) }
final def tag(sourceFile: T, t: Array[Byte]) { tagMap(sourceFile) = t }
private def removeOneWay(a: DMap[T], files: Iterable[T]): Unit =
a.values.foreach { _ --= files }
private def remove(a: DMap[T], b: DMap[T], file: T): Unit =
for(x <- a.removeKey(file)) b --= x
private def removeAll(files: Iterable[T], a: DMap[T], b: DMap[T]): Unit =
files.foreach { file => remove(a, b, file); remove(b, a, file) }
final def removeAll(files: Iterable[T])
{
def remove(a: DMap[T], b: DMap[T], file: T): Unit =
for(x <- a.removeKey(file)) b --= x
def removeAll(a: DMap[T], b: DMap[T]): Unit =
files.foreach { file => remove(a, b, file); remove(b, a, file) }
removeAll(forward(reverseDependencies), reverseDependencies)
removeAll(productMap, sourceMap)
removeAll(forward(reverseUses), reverseUses)
removeAll(files, forward(reverseDependencies), reverseDependencies)
removeAll(files, productMap, sourceMap)
removeAll(files, forward(reverseUses), reverseUses)
tagMap --= files
}
def pending(sources: Iterable[T])
{
removeOneWay(reverseDependencies, sources)
removeOneWay(reverseUses, sources)
removeAll(sources, productMap, sourceMap)
tagMap --= sources
}
protected final def forward(map: DMap[T]): DMap[T] =
{
val f = newMap[T]
for( (key, values) <- map; value <- values) f.add(value, key)
f
}
override def toString =
(graph("Reverse source dependencies", reverseDependencies) ::
graph("Sources and products", productMap) ::
graph("Reverse uses", reverseUses) ::
Nil) mkString "\n"
def graph(title: String, map: DMap[T]) =
"\"" + title + "\" {\n\t" + graphEntries(map) + "\n}"
def graphEntries(map: DMap[T]) = map.map{ case (key, values) => values.map(key + " -> " + _).mkString("\n\t") }.mkString("\n\t")
}

View File

@ -1,3 +1,6 @@
/* sbt -- Simple Build Tool
* Copyright 2009, 2010 Mark Harrah
*/
package xsbt
import java.io.{File,IOException}
@ -79,60 +82,98 @@ class Difference(val filesTask: Task[Set[File]], val style: FilesInfo.Style, val
}
}
}
object InvalidateFiles
class DependencyTracked[T](val cacheDirectory: File, val translateProducts: Boolean, cleanT: T => Unit)(implicit format: Format[T], mf: Manifest[T]) extends Tracked
{
def apply(cacheDirectory: File): Invalidate[File] = apply(cacheDirectory, true)
def apply(cacheDirectory: File, translateProducts: Boolean): Invalidate[File] =
{
import sbinary.DefaultProtocol.FileFormat
new Invalidate[File](cacheDirectory, translateProducts, FileUtilities.delete)
}
}
class Invalidate[T](val cacheDirectory: File, val translateProducts: Boolean, cleanT: T => Unit)
(implicit format: Format[T], mf: Manifest[T]) extends Tracked
{
def this(cacheDirectory: File, translateProducts: Boolean)(implicit format: Format[T], mf: Manifest[T]) =
this(cacheDirectory, translateProducts, x => ())
private val trackFormat = new TrackingFormat[T](cacheDirectory, translateProducts)
private def cleanAll(fs: Set[T]) = fs.foreach(cleanT)
val clean = Task(cleanAll(trackFormat.read.allProducts))
val clear = Clean(cacheDirectory)
def apply[R](f: UpdateTracking[T] => Task[R]): Task[R] =
{
val tracker = trackFormat.read
f(tracker) map { result =>
trackFormat.write(tracker)
result
}
}
}
object InvalidateFiles
{
def apply(cacheDirectory: File): InvalidateTransitive[File] = apply(cacheDirectory, true)
def apply(cacheDirectory: File, translateProducts: Boolean): InvalidateTransitive[File] =
{
import sbinary.DefaultProtocol.FileFormat
new InvalidateTransitive[File](cacheDirectory, translateProducts, FileUtilities.delete)
}
}
object InvalidateTransitive
{
import scala.collection.Set
def apply[T](tracker: UpdateTracking[T], files: Set[T]): InvalidationReport[T] =
{
val readTracker = tracker.read
val invalidated = Set() ++ invalidate(readTracker, files)
val invalidatedProducts = Set() ++ invalidated.filter(readTracker.isProduct)
new InvalidationReport[T]
{
val invalid = invalidated
val invalidProducts = invalidatedProducts
val valid = Set() ++ files -- invalid
}
}
def andClean[T](tracker: UpdateTracking[T], cleanImpl: Set[T] => Unit, files: Set[T]): InvalidationReport[T] =
{
val report = apply(tracker, files)
clean(tracker, cleanImpl, report)
report
}
def clear[T](tracker: UpdateTracking[T], report: InvalidationReport[T]): Unit =
tracker.removeAll(report.invalid)
def clean[T](tracker: UpdateTracking[T], cleanImpl: Set[T] => Unit, report: InvalidationReport[T])
{
clear(tracker, report)
cleanImpl(report.invalidProducts)
}
private def invalidate[T](tracker: ReadTracking[T], files: Iterable[T]): Set[T] =
{
import scala.collection.mutable.HashSet
val invalidated = new HashSet[T]
def invalidate0(files: Iterable[T]): Unit =
for(file <- files if !invalidated(file))
{
invalidated += file
invalidate0(invalidatedBy(tracker, file))
}
invalidate0(files)
invalidated
}
private def invalidatedBy[T](tracker: ReadTracking[T], file: T) =
tracker.products(file) ++ tracker.sources(file) ++ tracker.usedBy(file) ++ tracker.dependsOn(file)
}
class InvalidateTransitive[T](cacheDirectory: File, translateProducts: Boolean, cleanT: T => Unit)
(implicit format: Format[T], mf: Manifest[T]) extends Tracked
{
def this(cacheDirectory: File, translateProducts: Boolean)(implicit format: Format[T], mf: Manifest[T]) =
this(cacheDirectory, translateProducts, (_: T) => ())
private val tracked = new DependencyTracked(cacheDirectory, translateProducts, cleanT)
def clean = tracked.clean
def clear = tracked.clear
def apply[R](changes: ChangeReport[T])(f: (InvalidationReport[T], UpdateTracking[T]) => Task[R]): Task[R] =
apply(Task(changes))(f)
def apply[R](changesTask: Task[ChangeReport[T]])(f: (InvalidationReport[T], UpdateTracking[T]) => Task[R]): Task[R] =
{
changesTask bind { changes =>
val tracker = trackFormat.read
def invalidatedBy(file: T) = tracker.products(file) ++ tracker.sources(file) ++ tracker.usedBy(file) ++ tracker.dependsOn(file)
import scala.collection.mutable.HashSet
val invalidated = new HashSet[T]
val invalidatedProducts = new HashSet[T]
def invalidate(files: Iterable[T]): Unit =
for(file <- files if !invalidated(file))
{
invalidated += file
if(!tracker.sources(file).isEmpty) invalidatedProducts += file
invalidate(invalidatedBy(file))
}
invalidate(changes.modified)
tracker.removeAll(invalidated)
val report = new InvalidationReport[T]
{
val invalid = Set(invalidated.toSeq : _*)
val invalidProducts = Set(invalidatedProducts.toSeq : _*)
val valid = changes.unmodified -- invalid
}
cleanAll(report.invalidProducts)
f(report, tracker) map { result =>
trackFormat.write(tracker)
result
tracked { tracker =>
val report = InvalidateTransitive.andClean[T](tracker, _.foreach(cleanT), changes.modified)
f(report, tracker)
}
}
}

View File

@ -1,3 +1,6 @@
/* sbt -- Simple Build Tool
* Copyright 2009, 2010 Mark Harrah
*/
package xsbt
import java.io.File

View File

@ -0,0 +1,34 @@
/* sbt -- Simple Build Tool
* Copyright 2010 Mark Harrah
*/
package xsbt.api
import xsbti.api._
import java.io.{ByteArrayInputStream, ByteArrayOutputStream, InputStream, ObjectInputStream, ObjectOutputStream, OutputStream}
object APIFormat
{
def write(api: Source): Array[Byte] =
{
val baos = new ByteArrayOutputStream
write(api, baos)
baos.toByteArray
}
def write(api: Source, out: OutputStream)
{
val objOut = new ObjectOutputStream(out)
try { objOut.writeObject(api) }
finally { objOut.close() }
}
def read(bytes: Array[Byte]): Source = read(new ByteArrayInputStream(bytes))
def read(in: InputStream): Source =
{
try
{
val objIn = new ObjectInputStream(in)
try { objIn.readObject().asInstanceOf[Source] }
finally { objIn.close() }
}
catch { case e: java.io.EOFException => new xsbti.api.Source(Array(), Array())}
}
}

View File

@ -142,8 +142,12 @@ private class SameAPI
def sameAnnotations(a: Seq[Annotation], b: Seq[Annotation]): Boolean =
sameSeq(a, b)(sameAnnotation)
def sameAnnotation(a: Annotation, b: Annotation): Boolean =
sameSimpleType(a.base, b.base) &&
sameSeq(a.arguments, b.arguments)(defaultEquals)
debug(sameSimpleType(a.base, b.base), "Annotation base type differed") &&
debug(sameAnnotationArguments(a.arguments, b.arguments), "Annotation arguments differed (" + a + ") and (" + b + ")")
def sameAnnotationArguments(a: Seq[AnnotationArgument], b: Seq[AnnotationArgument]): Boolean =
argumentMap(a) == argumentMap(b)
def argumentMap(a: Seq[AnnotationArgument]): Map[String,String] =
Map() ++ a.map(arg => (arg.name, arg.value))
def sameDefinitionSpecificAPI(a: Definition, b: Definition): Boolean =
(a, b) match
@ -256,7 +260,7 @@ private class SameAPI
case (sa: Singleton, sb: Singleton) => debug(sameSingleton(sa, sb), "Different singleton")
case (_: EmptyType, _: EmptyType) => true
case (pa: Parameterized, pb: Parameterized) => debug(sameParameterized(pa, pb), "Different parameterized")
case _ => debug(false, "Different category of simple type")
case _ => debug(false, "Different category of simple type (" + a.getClass.getName + " and " + b.getClass.getName + ") for (" + a + " and " + b + ")")
}
def sameParameterized(a: Parameterized, b: Parameterized): Boolean =

View File

@ -67,7 +67,9 @@ final class API(val global: Global, val callback: xsbti.AnalysisCallback) extend
}
private def annotations(as: List[AnnotationInfo]): Array[xsbti.api.Annotation] = as.toArray[AnnotationInfo].map(annotation)
private def annotation(a: AnnotationInfo) = new xsbti.api.Annotation(simpleType(a.atp), a.args.map(_.hashCode.toString).toArray[String])
private def annotation(a: AnnotationInfo) =
new xsbti.api.Annotation(simpleType(a.atp),
a.assocs.map { case (name, value) => new xsbti.api.AnnotationArgument(name.toString, value.toString) }.toArray[xsbti.api.AnnotationArgument] )
private def annotated(as: List[AnnotationInfo], tpe: Type) = new xsbti.api.Annotated(simpleType(tpe), annotations(as))
private def defDef(s: Symbol) =
@ -180,7 +182,7 @@ final class API(val global: Global, val callback: xsbti.AnalysisCallback) extend
else
{
val within = c.privateWithin
val qualifier = if(within == NoSymbol) Constants.unqualified else new xsbti.api.IdQualifier(c.fullNameString)
val qualifier = if(within == NoSymbol) Constants.unqualified else new xsbti.api.IdQualifier(within.fullNameString)
if(c.hasFlag(Flags.PRIVATE)) new xsbti.api.Private(qualifier)
else if(c.hasFlag(Flags.PROTECTED)) new xsbti.api.Protected(qualifier)
else new xsbti.api.Pkg(qualifier)

View File

@ -96,7 +96,10 @@ TypeParameter
Annotation
base: SimpleType
arguments: String*
arguments: AnnotationArgument*
AnnotationArgument
name: String
value: String
enum Variance : Contravariant, Covariant, Invariant
enum ParameterModifier : Repeated, Plain, ByName

View File

@ -0,0 +1,51 @@
/* sbt -- Simple Build Tool
* Copyright 2010 Mark Harrah
*/
package xsbt
import java.io.File
class AggressiveCompiler extends xsbti.AppMain
{
final def run(configuration: xsbti.AppConfiguration): xsbti.MainResult =
{
System.setProperty("sbt.api.enable", "true")
val args = configuration.arguments.map(_.trim).toList
readLine("Press enter to compile... ")
val start = System.currentTimeMillis
val success = run(args, configuration.baseDirectory, configuration.provider)
val end = System.currentTimeMillis
println("Compiled in " + ((end - start) / 1000.0) + " s")
run(configuration)
}
def run(args: List[String], cwd: File, app: xsbti.AppProvider): Boolean =
{
import Paths._
import GlobFilter._
val launcher = app.scalaProvider.launcher
val sources = Task(cwd ** "*.scala")
val outputDirectory = Task(cwd / "target" / "classes")
val classpath = outputDirectory map { _ ++ (cwd ** "*.jar") }
val cacheDirectory = cwd / "target" / "cache"
val options = Task(args.tail.toSeq)
val log = new ConsoleLogger with CompileLogger with IvyLogger { def verbose(msg: => String) = debug(msg) }
val componentManager = new ComponentManager(launcher.globalLock, app.components, log)
val compiler = Task(new AnalyzingCompiler(ScalaInstance(args.head, launcher), componentManager))
val compileTask = AggressiveCompile(sources, classpath, outputDirectory, options, cacheDirectory, compiler, log)
try { TaskRunner(compileTask.task); true }
catch
{
case w: TasksFailed => w.failures.foreach { f => handleException(f.exception) }; false
case e: Exception => handleException(e); false
}
}
def handleException(e: Throwable) =
{
if(!e.isInstanceOf[xsbti.CompileFailed])
{
e.printStackTrace
System.err.println(e.toString)
}
}
}

20
main/alt.boot.properties Normal file
View File

@ -0,0 +1,20 @@
[scala]
version: 2.7.7
[app]
org: org.scala-tools.sbt
name: alternate-compiler-test
version: 0.6.10-SNAPSHOT
class: xsbt.AggressiveCompiler
components: xsbti
cross-versioned: true
resources: project/boot/sbinary.jar
[repositories]
local
maven-local
scala-tools-releases
scala-tools-snapshots
[boot]
directory: project/boot

View File

@ -30,7 +30,9 @@ class XSbt(info: ProjectInfo) extends ParentProject(info)
val trackingSub = baseProject(cachePath / "tracking", "Tracking", cacheSub)
val compilerSub = project(compilePath, "Compile", new CompileProject(_),
launchInterfaceSub, interfaceSub, ivySub, ioSub, classpathSub, compileInterfaceSub)
val stdTaskSub = project(tasksPath / "standard", "Standard Tasks", new StandardTaskProject(_), trackingSub, compilerSub)
val stdTaskSub = project(tasksPath / "standard", "Standard Tasks", new StandardTaskProject(_), trackingSub, compilerSub, apiSub)
val altCompilerSub = baseProject("main", "Alternate Compiler Test", stdTaskSub, logSub)
val distSub = project("dist", "Distribution", new DistProject(_))

View File

@ -93,30 +93,26 @@ object Task
}
}
implicit def twoToBuilder[A,B](t: (Task[A], Task[B]) ): Builder2[A,B] =
t match { case (a,b) => new Builder2(a,b) }
implicit def twoToBuilder[A,B](t: (Task[A], Task[B]) ): Builder2[A,B] = new Builder2(t._1,t._2)
final class Builder2[A,B] private[Task](a: Task[A], b: Task[B]) extends NotNull
{
private def extract = (r: Results) => (r(a), r(b))
private def compute[T](f: (A,B) => T) = tupled(f) compose extract
private def compute[T](f: (A,B) => T) = (r: Results) => f(r(a), r(b))
def map[X](f: (A,B) => X): Task[X] = mapTask(a,b)(compute(f))
def bind[X](f: (A,B) => Result[X]): Task[X] = bindTask(a,b)(compute(f))
}
implicit def threeToBuilder[A,B,C](t: (Task[A], Task[B], Task[C])): Builder3[A,B,C] = t match { case (a,b,c) => new Builder3(a,b,c) }
implicit def threeToBuilder[A,B,C](t: (Task[A], Task[B], Task[C])): Builder3[A,B,C] = new Builder3(t._1,t._2,t._3)
final class Builder3[A,B,C] private[Task](a: Task[A], b: Task[B], c: Task[C]) extends NotNull
{
private def extract = (r: Results) => (r(a), r(b), r(c))
private def compute[T](f: (A,B,C) => T) = tupled(f) compose extract
private def compute[T](f: (A,B,C) => T) = (r: Results) => f(r(a), r(b), r(c))
def map[X](f: (A,B,C) => X): Task[X] = mapTask(a,b,c)(compute(f))
def bind[X](f: (A,B,C) => Result[X]): Task[X] = bindTask(a,b,c)(compute(f))
}
implicit def fourToBuilder[A,B,C,D](t: (Task[A], Task[B], Task[C], Task[D])): Builder4[A,B,C,D] = t match { case (a,b,c,d) => new Builder4(a,b,c,d) }
implicit def fourToBuilder[A,B,C,D](t: (Task[A], Task[B], Task[C], Task[D])): Builder4[A,B,C,D] = new Builder4(t._1,t._2,t._3,t._4)
final class Builder4[A,B,C,D] private[Task](a: Task[A], b: Task[B], c: Task[C], d: Task[D]) extends NotNull
{
private def extract = (r: Results) => (r(a), r(b), r(c), r(d))
private def compute[T](f: (A,B,C,D) => T) = tupled(f) compose extract
private def compute[T](f: (A,B,C,D) => T) = (r: Results) => f(r(a), r(b), r(c), r(d))
def map[X](f: (A,B,C,D) => X): Task[X] = mapTask(a,b,c,d)( compute(f) )
def bind[X](f: (A,B,C,D) => Result[X]): Task[X] = bindTask(a,b,c,d)( compute(f) )
}

View File

@ -1,14 +1,20 @@
/* sbt -- Simple Build Tool
* Copyright 2009, 2010 Mark Harrah
*/
package xsbt
import java.io.File
import xsbt.api.{APIFormat, SameAPI}
import xsbti.api.Source
trait Compile extends TrackedTaskDefinition[CompileReport]
trait CompileImpl[R]
{
def apply(sourceChanges: ChangeReport[File], classpathChanges: ChangeReport[File], outputDirectory: File, options: Seq[String]): Task[R]
def tracked: Seq[Tracked]
}
final class Compile[R](val cacheDirectory: File, val sources: Task[Set[File]], val classpath: Task[Set[File]],
val outputDirectory: Task[File], val options: Task[Seq[String]], compileImpl: CompileImpl[R]) extends TrackedTaskDefinition[R]
{
val sources: Task[Set[File]]
val classpath: Task[Set[File]]
val outputDirectory: Task[File]
val options: Task[Seq[String]]
val trackedClasspath = Difference.inputs(classpath, FilesInfo.lastModified, cacheFile("classpath"))
val trackedSource = Difference.inputs(sources, FilesInfo.hash, cacheFile("sources"))
val trackedOptions =
@ -17,121 +23,127 @@ trait Compile extends TrackedTaskDefinition[CompileReport]
import Task._
new Changed((outputDirectory, options) map ( "-d" :: _.getAbsolutePath :: _.toList), cacheFile("options"))
}
val invalidation = InvalidateFiles(cacheFile("dependencies/"))
lazy val task = create
def create =
val task =
trackedClasspath { rawClasspathChanges => // detect changes to the classpath (last modified only)
trackedSource { rawSourceChanges =>// detect changes to sources (hash only)
trackedSource { rawSourceChanges => // detect changes to sources (hash only)
val newOpts = (opts: Seq[String]) => (opts, rawSourceChanges.markAllModified, rawClasspathChanges.markAllModified) // if options changed, mark everything changed
val sameOpts = (opts: Seq[String]) => (opts, rawSourceChanges, rawClasspathChanges)
trackedOptions(newOpts, sameOpts) bind { // detect changes to options
case (options, sourceChanges, classpathChanges) =>
invalidation( classpathChanges +++ sourceChanges ) { (report, tracking) => // invalidation based on changes
outputDirectory bind { outDir => compile(sourceChanges, classpathChanges, outDir, options, report, tracking) }
outputDirectory bind { outDir =>
FileUtilities.createDirectory(outDir)
compileImpl(sourceChanges, classpathChanges, outDir, options)
}
}
}
} dependsOn(sources, options, outputDirectory)// raise these dependencies to the top for parallelism
} dependsOn(sources, classpath, options, outputDirectory)// raise these dependencies to the top for parallelism
def compile(sourceChanges: ChangeReport[File], classpathChanges: ChangeReport[File], outputDirectory: File, options: Seq[String], report: InvalidationReport[File], tracking: UpdateTracking[File]): Task[CompileReport]
lazy val tracked = getTracked
protected def getTracked = Seq(trackedClasspath, trackedSource, trackedOptions, invalidation)
lazy val tracked = Seq(trackedClasspath, trackedSource, trackedOptions) ++ compileImpl.tracked
}
class StandardCompile(val sources: Task[Set[File]], val classpath: Task[Set[File]], val outputDirectory: Task[File], val options: Task[Seq[String]],
val superclassNames: Task[Set[String]], val compilerTask: Task[AnalyzingCompiler], val cacheDirectory: File, val log: CompileLogger) extends Compile
{
import Task._
import scala.collection.mutable.{ArrayBuffer, Buffer, HashMap, HashSet, Map, Set => mSet}
override def create = super.create dependsOn(superclassNames, compilerTask) // raise these dependencies to the top for parallelism
def compile(sourceChanges: ChangeReport[File], classpathChanges: ChangeReport[File], outputDirectory: File,
options: Seq[String], report: InvalidationReport[File], tracking: UpdateTracking[File]): Task[CompileReport] =
object AggressiveCompile
{
def apply(sources: Task[Set[File]], classpath: Task[Set[File]], outputDirectory: Task[File], options: Task[Seq[String]],
cacheDirectory: File, compilerTask: Task[AnalyzingCompiler], log: CompileLogger): Compile[Set[File]] =
{
val sources = report.invalid ** sourceChanges.checked // determine the sources that need recompiling (report.invalid also contains classes and libraries)
val classpath = classpathChanges.checked
compile(sources, classpath, outputDirectory, options, tracking)
val implCache = new File(cacheDirectory, "deps")
val baseCache = new File(cacheDirectory, "inputs")
val impl = new AggressiveCompile(implCache, compilerTask, log)
new Compile(baseCache, sources, classpath, outputDirectory, options, impl)
}
def compile(sources: Set[File], classpath: Set[File], outputDirectory: File, options: Seq[String], tracking: UpdateTracking[File]): Task[CompileReport] =
{
(compilerTask, superclassNames) map { (compiler, superClasses) =>
if(!sources.isEmpty)
{
val callback = new CompileAnalysisCallback(superClasses.toArray, tracking)
log.debug("Compile task calling compiler " + compiler)
compiler(sources, classpath, outputDirectory, options, callback, 100, log)
}
val readTracking = tracking.read
val applicationSet = new HashSet[String]
val subclassMap = new HashMap[String, Buffer[DetectedSubclass]]
readTags(applicationSet, subclassMap, readTracking)
new CompileReport
{
val superclasses = superClasses
def subclasses(superclass: String) = Set() ++ subclassMap.getOrElse(superclass, Nil)
val applications = Set() ++ applicationSet
val classes = Set() ++ readTracking.allProducts
override def toString =
{
val superStrings = superclasses.map(superC => superC + " >: \n\t\t" + subclasses(superC).mkString("\n\t\t"))
val applicationsPart = if(applications.isEmpty) Nil else Seq("Applications") ++ applications
val lines = Seq("Compilation Report:", sources.size + " sources", classes.size + " classes") ++ superStrings
lines.mkString("\n\t")
}
class AggressiveCompile(val cacheDirectory: File, val compilerTask: Task[AnalyzingCompiler], val log: CompileLogger) extends CompileImpl[Set[File]]
{
def apply(sourceChanges: ChangeReport[File], classpathChanges: ChangeReport[File], outputDirectory: File, options: Seq[String]): Task[Set[File]] =
compilerTask bind { compiler =>
tracking { tracker =>
Task {
log.info("Removed sources: \n\t" + sourceChanges.removed.mkString("\n\t"))
log.info("Added sources: \n\t" + sourceChanges.added.mkString("\n\t"))
log.info("Modified sources: \n\t" + (sourceChanges.modified -- sourceChanges.added -- sourceChanges.removed).mkString("\n\t"))
val classpath = classpathChanges.checked
val readTracker = tracker.read
def uptodate(time: Long, files: Iterable[File]) = files.forall(_.lastModified <= time)
def isProductOutofdate(product: File) = !product.exists || !uptodate(product.lastModified, readTracker.sources(product))
val outofdateProducts = readTracker.allProducts.filter(isProductOutofdate)
val rawInvalidatedSources =
classpathChanges.modified.flatMap(readTracker.usedBy) ++
sourceChanges.removed.flatMap(readTracker.dependsOn) ++
sourceChanges.modified ++
outofdateProducts.flatMap(readTracker.sources)
val invalidatedSources = scc(readTracker, rawInvalidatedSources)
val sources = invalidatedSources.filter(_.exists)
val previousAPIMap = Map() ++ sources.map { src => (src, APIFormat.read(readTracker.tag(src))) }
val invalidatedProducts = outofdateProducts ++ products(readTracker, invalidatedSources)
val transitiveIfNeeded = InvalidateTransitive(tracker, sources)
tracker.removeAll(invalidatedProducts ++ classpathChanges.modified ++ (invalidatedSources -- sources))
tracker.pending(sources)
FileUtilities.delete(invalidatedProducts)
log.info("Initially invalidated sources:\n\t" + sources.mkString("\n\t"))
if(!sources.isEmpty)
{
val newAPIMap = doCompile(sources, classpath, outputDirectory, options, tracker, compiler, log)
val apiChanged = sources filter { src => !sameAPI(previousAPIMap, newAPIMap, src) }
log.info("Sources with API changes:\n\t" + apiChanged.mkString("\n\t"))
val finalAPIMap =
if(apiChanged.isEmpty || apiChanged.size == sourceChanges.checked.size) newAPIMap
else
{
InvalidateTransitive.clean(tracker, FileUtilities.delete, transitiveIfNeeded)
val sources = transitiveIfNeeded.invalid ** sourceChanges.checked
log.info("All sources invalidated by API changes:\n\t" + sources.mkString("\n\t"))
doCompile(sources, classpath, outputDirectory, options, tracker, compiler, log)
}
finalAPIMap.foreach { case (src, api) => tracker.tag(src, APIFormat.write(api)) }
}
Set() ++ tracker.read.allProducts
}
}
}
}
private def abs(f: Set[File]) = f.map(_.getAbsolutePath)
private def readTags(allApplications: mSet[String], subclassMap: Map[String, Buffer[DetectedSubclass]], readTracking: ReadTracking[File])
def products(tracker: ReadTracking[File], srcs: Set[File]): Set[File] = srcs.flatMap(tracker.products)
def doCompile(sources: Set[File], classpath: Set[File], outputDirectory: File, options: Seq[String], tracker: UpdateTracking[File], compiler: AnalyzingCompiler, log: CompileLogger): scala.collection.Map[File, Source] =
{
for((source, tag) <- readTracking.allTags) if(tag.length > 0)
{
val (applications, subclasses) = Tag.fromBytes(tag)
allApplications ++= applications
subclasses.foreach(subclass => subclassMap.getOrElseUpdate(subclass.superclassName, new ArrayBuffer[DetectedSubclass]) += subclass)
}
}
private final class CompileAnalysisCallback(superClasses: Array[String], tracking: UpdateTracking[File]) extends xsbti.AnalysisCallback
{
private var applications = List[String]()
private var subclasses = List[DetectedSubclass]()
def superclassNames = superClasses
def superclassNotFound(superclassName: String) = error("Superclass not found: " + superclassName)
def beginSource(source: File) {}
def endSource(source: File)
{
if(!applications.isEmpty || !subclasses.isEmpty)
{
tracking.tag(source, Tag.toBytes(applications, subclasses) )
applications = Nil
subclasses = Nil
}
}
def foundApplication(source: File, className: String) { applications ::= className }
def foundSubclass(source: File, subclassName: String, superclassName: String, isModule: Boolean): Unit =
subclasses ::= DetectedSubclass(source, subclassName, superclassName, isModule)
def sourceDependency(dependsOn: File, source: File) { tracking.dependency(source, dependsOn) }
def jarDependency(jar: File, source: File) { tracking.use(source, jar) }
def classDependency(clazz: File, source: File) { tracking.dependency(source, clazz) }
def generatedClass(source: File, clazz: File) { tracking.product(source, clazz) }
def api(source: File, api: xsbti.api.Source) = ()
val callback = new APIAnalysisCallback(tracker)
log.debug("Compiling using compiler " + compiler)
compiler(sources, classpath, outputDirectory, options, callback, 100, log)
callback.apiMap
}
import sbinary.DefaultProtocol.FileFormat
val tracking = new DependencyTracked(cacheDirectory, true, (files: File) => FileUtilities.delete(files))
def tracked = Seq(tracking)
def sameAPI[T](a: scala.collection.Map[T, Source], b: scala.collection.Map[T, Source], t: T): Boolean = sameAPI(a.get(t), b.get(t))
def sameAPI(a: Option[Source], b: Option[Source]): Boolean =
if(a.isEmpty) b.isEmpty else (b.isDefined && SameAPI(a.get, b.get))
// TODO: implement
def scc(readTracker: ReadTracking[File], sources: Set[File]) = sources
}
object Tag
private final class APIAnalysisCallback(tracking: UpdateTracking[File]) extends xsbti.AnalysisCallback
{
import sbinary.{DefaultProtocol, Format, Operations}
import DefaultProtocol._
private implicit val subclassFormat: Format[DetectedSubclass] =
asProduct4(DetectedSubclass.apply)( ds => Some(ds.source, ds.subclassName, ds.superclassName, ds.isModule))
def toBytes(applications: List[String], subclasses: List[DetectedSubclass]) = CacheIO.toBytes((applications, subclasses))
def fromBytes(bytes: Array[Byte]) = CacheIO.fromBytes( ( List[String](), List[DetectedSubclass]() ) )(bytes)
val apiMap = new scala.collection.mutable.HashMap[File, Source]
def sourceDependency(dependsOn: File, source: File) { tracking.dependency(source, dependsOn) }
def jarDependency(jar: File, source: File) { tracking.use(source, jar) }
def classDependency(clazz: File, source: File) { tracking.dependency(source, clazz) }
def generatedClass(source: File, clazz: File) { tracking.product(source, clazz) }
def api(source: File, api: xsbti.api.Source) { apiMap(source) = api }
def superclassNames = Array()
def superclassNotFound(superclassName: String) {}
def beginSource(source: File) {}
def endSource(source: File) {}
def foundApplication(source: File, className: String) {}
def foundSubclass(source: File, subclassName: String, superclassName: String, isModule: Boolean) {}
}
trait CompileReport extends NotNull
{
def classes: Set[File]
def applications: Set[String]
def superclasses: Set[String]
def subclasses(superclass: String): Set[DetectedSubclass]
}
final case class DetectedSubclass(source: File, subclassName: String, superclassName: String, isModule: Boolean) extends NotNull

View File

@ -1,3 +1,6 @@
/* sbt -- Simple Build Tool
* Copyright 2009, 2010 Mark Harrah
*/
package xsbt
import java.io.File
@ -7,12 +10,15 @@ trait TaskDefinition[T]
val task: Task[T]
val clean: Task[Unit]
}
trait TrackedTaskDefinition[T] extends TaskDefinition[T]
trait TrackedTaskDefinition[T] extends TaskDefinition[T] with WithCache
{
def cacheDirectory: File
def cacheFile(relative: String) = new File(cacheDirectory, relative)
val tracked: Seq[Tracked]
lazy val clean: Task[Unit] = onTracked(_.clean).bind( u => onTracked(_.clear) )
import Task._
private def onTracked(f: Tracked => Task[Unit]) = tracked.forkTasks(f).joinIgnore
}
trait WithCache
{
def cacheDirectory: File
def cacheFile(relative: String) = new File(cacheDirectory, relative)
}

View File

@ -0,0 +1,118 @@
/* sbt -- Simple Build Tool
* Copyright 2009, 2010 Mark Harrah
*/
package xsbt
import java.io.File
trait TransitiveCompile extends CompileImpl[CompileReport] with WithCache
{
final val invalidation = InvalidateFiles(cacheFile("dependencies/"))
def tracked = Seq(invalidation)
def apply(sourceChanges: ChangeReport[File], classpathChanges: ChangeReport[File], outputDirectory: File, options: Seq[String]): Task[CompileReport] =
// doesn't notice if classes are removed
invalidation( classpathChanges +++ sourceChanges ) { (report, tracking) => // invalidation based on changes
compile(sourceChanges, classpathChanges, outputDirectory, options, report, tracking)
}
def compile(sourceChanges: ChangeReport[File], classpathChanges: ChangeReport[File], outputDirectory: File,
options: Seq[String], report: InvalidationReport[File], tracking: UpdateTracking[File]): Task[CompileReport]
}
class StandardCompile(val sources: Task[Set[File]], val classpath: Task[Set[File]], val outputDirectory: Task[File], val options: Task[Seq[String]],
val superclassNames: Task[Set[String]], val compilerTask: Task[AnalyzingCompiler], val cacheDirectory: File, val log: CompileLogger) extends TransitiveCompile
{
import Task._
import scala.collection.mutable.{ArrayBuffer, Buffer, HashMap, HashSet, Map, Set => mSet}
def compile(sourceChanges: ChangeReport[File], classpathChanges: ChangeReport[File], outputDirectory: File,
options: Seq[String], report: InvalidationReport[File], tracking: UpdateTracking[File]): Task[CompileReport] =
{
val sources = report.invalid ** sourceChanges.checked // determine the sources that need recompiling (report.invalid also contains classes and libraries)
val classpath = classpathChanges.checked
compile(sources, classpath, outputDirectory, options, tracking)
}
def compile(sources: Set[File], classpath: Set[File], outputDirectory: File, options: Seq[String], tracking: UpdateTracking[File]): Task[CompileReport] =
{
(compilerTask, superclassNames) map { (compiler, superClasses) =>
if(!sources.isEmpty)
{
val callback = new CompileAnalysisCallback(superClasses.toArray, tracking)
log.debug("Compile task calling compiler " + compiler)
compiler(sources, classpath, outputDirectory, options, callback, 100, log)
}
val readTracking = tracking.read
val applicationSet = new HashSet[String]
val subclassMap = new HashMap[String, Buffer[DetectedSubclass]]
readTags(applicationSet, subclassMap, readTracking)
new CompileReport
{
val superclasses = superClasses
def subclasses(superclass: String) = Set() ++ subclassMap.getOrElse(superclass, Nil)
val applications = Set() ++ applicationSet
val classes = Set() ++ readTracking.allProducts
override def toString =
{
val superStrings = superclasses.map(superC => superC + " >: \n\t\t" + subclasses(superC).mkString("\n\t\t"))
val applicationsPart = if(applications.isEmpty) Nil else Seq("Applications") ++ applications
val lines = Seq("Compilation Report:", sources.size + " sources", classes.size + " classes") ++ superStrings
lines.mkString("\n\t")
}
}
}
}
private def abs(f: Set[File]) = f.map(_.getAbsolutePath)
private def readTags(allApplications: mSet[String], subclassMap: Map[String, Buffer[DetectedSubclass]], readTracking: ReadTracking[File])
{
for((source, tag) <- readTracking.allTags) if(tag.length > 0)
{
val (applications, subclasses) = Tag.fromBytes(tag)
allApplications ++= applications
subclasses.foreach(subclass => subclassMap.getOrElseUpdate(subclass.superclassName, new ArrayBuffer[DetectedSubclass]) += subclass)
}
}
private final class CompileAnalysisCallback(superClasses: Array[String], tracking: UpdateTracking[File]) extends xsbti.AnalysisCallback
{
private var applications = List[String]()
private var subclasses = List[DetectedSubclass]()
def superclassNames = superClasses
def superclassNotFound(superclassName: String) = error("Superclass not found: " + superclassName)
def beginSource(source: File) {}
def endSource(source: File)
{
if(!applications.isEmpty || !subclasses.isEmpty)
{
tracking.tag(source, Tag.toBytes(applications, subclasses) )
applications = Nil
subclasses = Nil
}
}
def foundApplication(source: File, className: String) { applications ::= className }
def foundSubclass(source: File, subclassName: String, superclassName: String, isModule: Boolean): Unit =
subclasses ::= DetectedSubclass(source, subclassName, superclassName, isModule)
def sourceDependency(dependsOn: File, source: File) { tracking.dependency(source, dependsOn) }
def jarDependency(jar: File, source: File) { tracking.use(source, jar) }
def classDependency(clazz: File, source: File) { tracking.dependency(source, clazz) }
def generatedClass(source: File, clazz: File) { tracking.product(source, clazz) }
def api(source: File, api: xsbti.api.Source) = ()
}
}
object Tag
{
import sbinary.{DefaultProtocol, Format, Operations}
import DefaultProtocol._
private implicit val subclassFormat: Format[DetectedSubclass] =
asProduct4(DetectedSubclass.apply)( ds => Some(ds.source, ds.subclassName, ds.superclassName, ds.isModule))
def toBytes(applications: List[String], subclasses: List[DetectedSubclass]) = CacheIO.toBytes((applications, subclasses))
def fromBytes(bytes: Array[Byte]) = CacheIO.fromBytes( ( List[String](), List[DetectedSubclass]() ) )(bytes)
}
trait CompileReport extends NotNull
{
def classes: Set[File]
def applications: Set[String]
def superclasses: Set[String]
def subclasses(superclass: String): Set[DetectedSubclass]
}
final case class DetectedSubclass(source: File, subclassName: String, superclassName: String, isModule: Boolean) extends NotNull