From 6f6b795b39d88a8eba8612c535c4bdd6ab3386a1 Mon Sep 17 00:00:00 2001 From: Mark Harrah Date: Tue, 5 Jan 2010 19:50:43 -0500 Subject: [PATCH] * Basic API serialization * Fixes to API extraction and equality checking * Reworked tracking * New compile infrastructure based on API changes * Example application for testing --- cache/tracking/ChangeReport.scala | 3 + cache/tracking/DependencyTracking.scala | 50 +++++- cache/tracking/Tracked.scala | 125 +++++++++----- cache/tracking/TrackingFormat.scala | 3 + compile/api/APIFormat.scala | 34 ++++ compile/api/SameAPI.scala | 10 +- compile/interface/API.scala | 6 +- interface/definition | 5 +- main/AggressiveCompiler.scala | 51 ++++++ main/alt.boot.properties | 20 +++ project/build/XSbt.scala | 4 +- tasks/Task.scala | 16 +- tasks/standard/Compile.scala | 214 +++++++++++++----------- tasks/standard/TaskDefinition.scala | 12 +- tasks/standard/TransitiveCompile.scala | 118 +++++++++++++ 15 files changed, 500 insertions(+), 171 deletions(-) create mode 100644 compile/api/APIFormat.scala create mode 100644 main/AggressiveCompiler.scala create mode 100644 main/alt.boot.properties create mode 100644 tasks/standard/TransitiveCompile.scala diff --git a/cache/tracking/ChangeReport.scala b/cache/tracking/ChangeReport.scala index 41f99ca1a..c8f3a52eb 100644 --- a/cache/tracking/ChangeReport.scala +++ b/cache/tracking/ChangeReport.scala @@ -1,3 +1,6 @@ +/* sbt -- Simple Build Tool + * Copyright 2009, 2010 Mark Harrah + */ package xsbt object ChangeReport diff --git a/cache/tracking/DependencyTracking.scala b/cache/tracking/DependencyTracking.scala index 5d61f020e..e34930f5a 100644 --- a/cache/tracking/DependencyTracking.scala +++ b/cache/tracking/DependencyTracking.scala @@ -1,3 +1,6 @@ +/* sbt -- Simple Build Tool + * Copyright 2009, 2010 Mark Harrah + */ package xsbt private object DependencyTracking @@ -16,14 +19,22 @@ trait UpdateTracking[T] extends NotNull def product(source: T, output: T): Unit def tag(source: T, t: Array[Byte]): Unit def read: ReadTracking[T] + // removes files from all maps, both keys and values + def removeAll(files: Iterable[T]): Unit + // removes sources as keys/values in source, product maps and as values in reverseDependencies map + def pending(sources: Iterable[T]): Unit } import scala.collection.Set trait ReadTracking[T] extends NotNull { + def isProduct(file: T): Boolean + def isSource(file: T): Boolean + def isUsed(file: T): Boolean def dependsOn(file: T): Set[T] def products(file: T): Set[T] def sources(file: T): Set[T] def usedBy(file: T): Set[T] + def tag(file: T): Array[Byte] def allProducts: Set[T] def allSources: Set[T] def allUsed: Set[T] @@ -42,6 +53,7 @@ private final class DefaultTracking[T](translateProducts: Boolean) val productMap: DMap[T] = forward(sourceMap) // map from a source to its products. Keep in sync with sourceMap } // if translateProducts is true, dependencies on a product are translated to dependencies on a source +// if there is a source recorded as generating that product private abstract class DependencyTracking[T](translateProducts: Boolean) extends ReadTracking[T] with UpdateTracking[T] { val reverseDependencies: DMap[T] // map from a file to the files that depend on it @@ -58,11 +70,17 @@ private abstract class DependencyTracking[T](translateProducts: Boolean) extends final def usedBy(file: T): Set[T] = get(reverseUses, file) final def tag(file: T): Array[Byte] = tagMap.getOrElse(file, new Array[Byte](0)) + def isProduct(file: T): Boolean = exists(sourceMap, file) + def isSource(file: T): Boolean = exists(productMap, file) + def isUsed(file: T): Boolean = exists(reverseUses, file) + + final def allProducts = Set() ++ sourceMap.keys final def allSources = Set() ++ productMap.keys final def allUsed = Set() ++ reverseUses.keys final def allTags = tagMap.toSeq + private def exists(map: DMap[T], value: T): Boolean = map.contains(value) private def get(map: DMap[T], value: T): Set[T] = map.getOrElse(value, Set.empty[T]) final def dependency(sourceFile: T, dependsOn: T) @@ -82,22 +100,38 @@ private abstract class DependencyTracking[T](translateProducts: Boolean) extends final def use(sourceFile: T, usesFile: T) { reverseUses.add(usesFile, sourceFile) } final def tag(sourceFile: T, t: Array[Byte]) { tagMap(sourceFile) = t } + private def removeOneWay(a: DMap[T], files: Iterable[T]): Unit = + a.values.foreach { _ --= files } + private def remove(a: DMap[T], b: DMap[T], file: T): Unit = + for(x <- a.removeKey(file)) b --= x + private def removeAll(files: Iterable[T], a: DMap[T], b: DMap[T]): Unit = + files.foreach { file => remove(a, b, file); remove(b, a, file) } final def removeAll(files: Iterable[T]) { - def remove(a: DMap[T], b: DMap[T], file: T): Unit = - for(x <- a.removeKey(file)) b --= x - def removeAll(a: DMap[T], b: DMap[T]): Unit = - files.foreach { file => remove(a, b, file); remove(b, a, file) } - - removeAll(forward(reverseDependencies), reverseDependencies) - removeAll(productMap, sourceMap) - removeAll(forward(reverseUses), reverseUses) + removeAll(files, forward(reverseDependencies), reverseDependencies) + removeAll(files, productMap, sourceMap) + removeAll(files, forward(reverseUses), reverseUses) tagMap --= files } + def pending(sources: Iterable[T]) + { + removeOneWay(reverseDependencies, sources) + removeOneWay(reverseUses, sources) + removeAll(sources, productMap, sourceMap) + tagMap --= sources + } protected final def forward(map: DMap[T]): DMap[T] = { val f = newMap[T] for( (key, values) <- map; value <- values) f.add(value, key) f } + override def toString = + (graph("Reverse source dependencies", reverseDependencies) :: + graph("Sources and products", productMap) :: + graph("Reverse uses", reverseUses) :: + Nil) mkString "\n" + def graph(title: String, map: DMap[T]) = + "\"" + title + "\" {\n\t" + graphEntries(map) + "\n}" + def graphEntries(map: DMap[T]) = map.map{ case (key, values) => values.map(key + " -> " + _).mkString("\n\t") }.mkString("\n\t") } diff --git a/cache/tracking/Tracked.scala b/cache/tracking/Tracked.scala index 72be00ac6..dc8c6095e 100644 --- a/cache/tracking/Tracked.scala +++ b/cache/tracking/Tracked.scala @@ -1,3 +1,6 @@ +/* sbt -- Simple Build Tool + * Copyright 2009, 2010 Mark Harrah + */ package xsbt import java.io.{File,IOException} @@ -79,60 +82,98 @@ class Difference(val filesTask: Task[Set[File]], val style: FilesInfo.Style, val } } } -object InvalidateFiles +class DependencyTracked[T](val cacheDirectory: File, val translateProducts: Boolean, cleanT: T => Unit)(implicit format: Format[T], mf: Manifest[T]) extends Tracked { - def apply(cacheDirectory: File): Invalidate[File] = apply(cacheDirectory, true) - def apply(cacheDirectory: File, translateProducts: Boolean): Invalidate[File] = - { - import sbinary.DefaultProtocol.FileFormat - new Invalidate[File](cacheDirectory, translateProducts, FileUtilities.delete) - } -} -class Invalidate[T](val cacheDirectory: File, val translateProducts: Boolean, cleanT: T => Unit) - (implicit format: Format[T], mf: Manifest[T]) extends Tracked -{ - def this(cacheDirectory: File, translateProducts: Boolean)(implicit format: Format[T], mf: Manifest[T]) = - this(cacheDirectory, translateProducts, x => ()) - private val trackFormat = new TrackingFormat[T](cacheDirectory, translateProducts) private def cleanAll(fs: Set[T]) = fs.foreach(cleanT) val clean = Task(cleanAll(trackFormat.read.allProducts)) val clear = Clean(cacheDirectory) + def apply[R](f: UpdateTracking[T] => Task[R]): Task[R] = + { + val tracker = trackFormat.read + f(tracker) map { result => + trackFormat.write(tracker) + result + } + } +} +object InvalidateFiles +{ + def apply(cacheDirectory: File): InvalidateTransitive[File] = apply(cacheDirectory, true) + def apply(cacheDirectory: File, translateProducts: Boolean): InvalidateTransitive[File] = + { + import sbinary.DefaultProtocol.FileFormat + new InvalidateTransitive[File](cacheDirectory, translateProducts, FileUtilities.delete) + } +} + +object InvalidateTransitive +{ + import scala.collection.Set + def apply[T](tracker: UpdateTracking[T], files: Set[T]): InvalidationReport[T] = + { + val readTracker = tracker.read + val invalidated = Set() ++ invalidate(readTracker, files) + val invalidatedProducts = Set() ++ invalidated.filter(readTracker.isProduct) + + new InvalidationReport[T] + { + val invalid = invalidated + val invalidProducts = invalidatedProducts + val valid = Set() ++ files -- invalid + } + } + def andClean[T](tracker: UpdateTracking[T], cleanImpl: Set[T] => Unit, files: Set[T]): InvalidationReport[T] = + { + val report = apply(tracker, files) + clean(tracker, cleanImpl, report) + report + } + def clear[T](tracker: UpdateTracking[T], report: InvalidationReport[T]): Unit = + tracker.removeAll(report.invalid) + def clean[T](tracker: UpdateTracking[T], cleanImpl: Set[T] => Unit, report: InvalidationReport[T]) + { + clear(tracker, report) + cleanImpl(report.invalidProducts) + } + + private def invalidate[T](tracker: ReadTracking[T], files: Iterable[T]): Set[T] = + { + import scala.collection.mutable.HashSet + val invalidated = new HashSet[T] + def invalidate0(files: Iterable[T]): Unit = + for(file <- files if !invalidated(file)) + { + invalidated += file + invalidate0(invalidatedBy(tracker, file)) + } + invalidate0(files) + invalidated + } + private def invalidatedBy[T](tracker: ReadTracking[T], file: T) = + tracker.products(file) ++ tracker.sources(file) ++ tracker.usedBy(file) ++ tracker.dependsOn(file) + +} +class InvalidateTransitive[T](cacheDirectory: File, translateProducts: Boolean, cleanT: T => Unit) + (implicit format: Format[T], mf: Manifest[T]) extends Tracked +{ + def this(cacheDirectory: File, translateProducts: Boolean)(implicit format: Format[T], mf: Manifest[T]) = + this(cacheDirectory, translateProducts, (_: T) => ()) + + private val tracked = new DependencyTracked(cacheDirectory, translateProducts, cleanT) + def clean = tracked.clean + def clear = tracked.clear + def apply[R](changes: ChangeReport[T])(f: (InvalidationReport[T], UpdateTracking[T]) => Task[R]): Task[R] = apply(Task(changes))(f) def apply[R](changesTask: Task[ChangeReport[T]])(f: (InvalidationReport[T], UpdateTracking[T]) => Task[R]): Task[R] = { changesTask bind { changes => - val tracker = trackFormat.read - def invalidatedBy(file: T) = tracker.products(file) ++ tracker.sources(file) ++ tracker.usedBy(file) ++ tracker.dependsOn(file) - - import scala.collection.mutable.HashSet - val invalidated = new HashSet[T] - val invalidatedProducts = new HashSet[T] - def invalidate(files: Iterable[T]): Unit = - for(file <- files if !invalidated(file)) - { - invalidated += file - if(!tracker.sources(file).isEmpty) invalidatedProducts += file - invalidate(invalidatedBy(file)) - } - - invalidate(changes.modified) - tracker.removeAll(invalidated) - - val report = new InvalidationReport[T] - { - val invalid = Set(invalidated.toSeq : _*) - val invalidProducts = Set(invalidatedProducts.toSeq : _*) - val valid = changes.unmodified -- invalid - } - cleanAll(report.invalidProducts) - - f(report, tracker) map { result => - trackFormat.write(tracker) - result + tracked { tracker => + val report = InvalidateTransitive.andClean[T](tracker, _.foreach(cleanT), changes.modified) + f(report, tracker) } } } diff --git a/cache/tracking/TrackingFormat.scala b/cache/tracking/TrackingFormat.scala index c1bb3da56..c0106d3a0 100644 --- a/cache/tracking/TrackingFormat.scala +++ b/cache/tracking/TrackingFormat.scala @@ -1,3 +1,6 @@ +/* sbt -- Simple Build Tool + * Copyright 2009, 2010 Mark Harrah + */ package xsbt import java.io.File diff --git a/compile/api/APIFormat.scala b/compile/api/APIFormat.scala new file mode 100644 index 000000000..6e3f7ab82 --- /dev/null +++ b/compile/api/APIFormat.scala @@ -0,0 +1,34 @@ +/* sbt -- Simple Build Tool + * Copyright 2010 Mark Harrah + */ +package xsbt.api + + import xsbti.api._ + import java.io.{ByteArrayInputStream, ByteArrayOutputStream, InputStream, ObjectInputStream, ObjectOutputStream, OutputStream} + +object APIFormat +{ + def write(api: Source): Array[Byte] = + { + val baos = new ByteArrayOutputStream + write(api, baos) + baos.toByteArray + } + def write(api: Source, out: OutputStream) + { + val objOut = new ObjectOutputStream(out) + try { objOut.writeObject(api) } + finally { objOut.close() } + } + def read(bytes: Array[Byte]): Source = read(new ByteArrayInputStream(bytes)) + def read(in: InputStream): Source = + { + try + { + val objIn = new ObjectInputStream(in) + try { objIn.readObject().asInstanceOf[Source] } + finally { objIn.close() } + } + catch { case e: java.io.EOFException => new xsbti.api.Source(Array(), Array())} + } +} \ No newline at end of file diff --git a/compile/api/SameAPI.scala b/compile/api/SameAPI.scala index 1213f3007..60f0ea355 100644 --- a/compile/api/SameAPI.scala +++ b/compile/api/SameAPI.scala @@ -142,8 +142,12 @@ private class SameAPI def sameAnnotations(a: Seq[Annotation], b: Seq[Annotation]): Boolean = sameSeq(a, b)(sameAnnotation) def sameAnnotation(a: Annotation, b: Annotation): Boolean = - sameSimpleType(a.base, b.base) && - sameSeq(a.arguments, b.arguments)(defaultEquals) + debug(sameSimpleType(a.base, b.base), "Annotation base type differed") && + debug(sameAnnotationArguments(a.arguments, b.arguments), "Annotation arguments differed (" + a + ") and (" + b + ")") + def sameAnnotationArguments(a: Seq[AnnotationArgument], b: Seq[AnnotationArgument]): Boolean = + argumentMap(a) == argumentMap(b) + def argumentMap(a: Seq[AnnotationArgument]): Map[String,String] = + Map() ++ a.map(arg => (arg.name, arg.value)) def sameDefinitionSpecificAPI(a: Definition, b: Definition): Boolean = (a, b) match @@ -256,7 +260,7 @@ private class SameAPI case (sa: Singleton, sb: Singleton) => debug(sameSingleton(sa, sb), "Different singleton") case (_: EmptyType, _: EmptyType) => true case (pa: Parameterized, pb: Parameterized) => debug(sameParameterized(pa, pb), "Different parameterized") - case _ => debug(false, "Different category of simple type") + case _ => debug(false, "Different category of simple type (" + a.getClass.getName + " and " + b.getClass.getName + ") for (" + a + " and " + b + ")") } def sameParameterized(a: Parameterized, b: Parameterized): Boolean = diff --git a/compile/interface/API.scala b/compile/interface/API.scala index a060f036f..4271408c1 100644 --- a/compile/interface/API.scala +++ b/compile/interface/API.scala @@ -67,7 +67,9 @@ final class API(val global: Global, val callback: xsbti.AnalysisCallback) extend } private def annotations(as: List[AnnotationInfo]): Array[xsbti.api.Annotation] = as.toArray[AnnotationInfo].map(annotation) - private def annotation(a: AnnotationInfo) = new xsbti.api.Annotation(simpleType(a.atp), a.args.map(_.hashCode.toString).toArray[String]) + private def annotation(a: AnnotationInfo) = + new xsbti.api.Annotation(simpleType(a.atp), + a.assocs.map { case (name, value) => new xsbti.api.AnnotationArgument(name.toString, value.toString) }.toArray[xsbti.api.AnnotationArgument] ) private def annotated(as: List[AnnotationInfo], tpe: Type) = new xsbti.api.Annotated(simpleType(tpe), annotations(as)) private def defDef(s: Symbol) = @@ -180,7 +182,7 @@ final class API(val global: Global, val callback: xsbti.AnalysisCallback) extend else { val within = c.privateWithin - val qualifier = if(within == NoSymbol) Constants.unqualified else new xsbti.api.IdQualifier(c.fullNameString) + val qualifier = if(within == NoSymbol) Constants.unqualified else new xsbti.api.IdQualifier(within.fullNameString) if(c.hasFlag(Flags.PRIVATE)) new xsbti.api.Private(qualifier) else if(c.hasFlag(Flags.PROTECTED)) new xsbti.api.Protected(qualifier) else new xsbti.api.Pkg(qualifier) diff --git a/interface/definition b/interface/definition index f2696c863..2365e9ee5 100644 --- a/interface/definition +++ b/interface/definition @@ -96,7 +96,10 @@ TypeParameter Annotation base: SimpleType - arguments: String* + arguments: AnnotationArgument* +AnnotationArgument + name: String + value: String enum Variance : Contravariant, Covariant, Invariant enum ParameterModifier : Repeated, Plain, ByName diff --git a/main/AggressiveCompiler.scala b/main/AggressiveCompiler.scala new file mode 100644 index 000000000..bbeb50e16 --- /dev/null +++ b/main/AggressiveCompiler.scala @@ -0,0 +1,51 @@ +/* sbt -- Simple Build Tool + * Copyright 2010 Mark Harrah + */ +package xsbt + + import java.io.File + +class AggressiveCompiler extends xsbti.AppMain +{ + final def run(configuration: xsbti.AppConfiguration): xsbti.MainResult = + { + System.setProperty("sbt.api.enable", "true") + val args = configuration.arguments.map(_.trim).toList + readLine("Press enter to compile... ") + val start = System.currentTimeMillis + val success = run(args, configuration.baseDirectory, configuration.provider) + val end = System.currentTimeMillis + println("Compiled in " + ((end - start) / 1000.0) + " s") + run(configuration) + } + def run(args: List[String], cwd: File, app: xsbti.AppProvider): Boolean = + { + import Paths._ + import GlobFilter._ + val launcher = app.scalaProvider.launcher + val sources = Task(cwd ** "*.scala") + val outputDirectory = Task(cwd / "target" / "classes") + val classpath = outputDirectory map { _ ++ (cwd ** "*.jar") } + val cacheDirectory = cwd / "target" / "cache" + val options = Task(args.tail.toSeq) + val log = new ConsoleLogger with CompileLogger with IvyLogger { def verbose(msg: => String) = debug(msg) } + val componentManager = new ComponentManager(launcher.globalLock, app.components, log) + val compiler = Task(new AnalyzingCompiler(ScalaInstance(args.head, launcher), componentManager)) + val compileTask = AggressiveCompile(sources, classpath, outputDirectory, options, cacheDirectory, compiler, log) + + try { TaskRunner(compileTask.task); true } + catch + { + case w: TasksFailed => w.failures.foreach { f => handleException(f.exception) }; false + case e: Exception => handleException(e); false + } + } + def handleException(e: Throwable) = + { + if(!e.isInstanceOf[xsbti.CompileFailed]) + { + e.printStackTrace + System.err.println(e.toString) + } + } +} \ No newline at end of file diff --git a/main/alt.boot.properties b/main/alt.boot.properties new file mode 100644 index 000000000..a24e8e815 --- /dev/null +++ b/main/alt.boot.properties @@ -0,0 +1,20 @@ +[scala] + version: 2.7.7 + +[app] + org: org.scala-tools.sbt + name: alternate-compiler-test + version: 0.6.10-SNAPSHOT + class: xsbt.AggressiveCompiler + components: xsbti + cross-versioned: true + resources: project/boot/sbinary.jar + +[repositories] + local + maven-local + scala-tools-releases + scala-tools-snapshots + +[boot] + directory: project/boot diff --git a/project/build/XSbt.scala b/project/build/XSbt.scala index 8e2808542..401b847df 100644 --- a/project/build/XSbt.scala +++ b/project/build/XSbt.scala @@ -30,7 +30,9 @@ class XSbt(info: ProjectInfo) extends ParentProject(info) val trackingSub = baseProject(cachePath / "tracking", "Tracking", cacheSub) val compilerSub = project(compilePath, "Compile", new CompileProject(_), launchInterfaceSub, interfaceSub, ivySub, ioSub, classpathSub, compileInterfaceSub) - val stdTaskSub = project(tasksPath / "standard", "Standard Tasks", new StandardTaskProject(_), trackingSub, compilerSub) + val stdTaskSub = project(tasksPath / "standard", "Standard Tasks", new StandardTaskProject(_), trackingSub, compilerSub, apiSub) + + val altCompilerSub = baseProject("main", "Alternate Compiler Test", stdTaskSub, logSub) val distSub = project("dist", "Distribution", new DistProject(_)) diff --git a/tasks/Task.scala b/tasks/Task.scala index 322b76ec2..1b9c2a344 100644 --- a/tasks/Task.scala +++ b/tasks/Task.scala @@ -93,30 +93,26 @@ object Task } } - implicit def twoToBuilder[A,B](t: (Task[A], Task[B]) ): Builder2[A,B] = - t match { case (a,b) => new Builder2(a,b) } + implicit def twoToBuilder[A,B](t: (Task[A], Task[B]) ): Builder2[A,B] = new Builder2(t._1,t._2) final class Builder2[A,B] private[Task](a: Task[A], b: Task[B]) extends NotNull { - private def extract = (r: Results) => (r(a), r(b)) - private def compute[T](f: (A,B) => T) = tupled(f) compose extract + private def compute[T](f: (A,B) => T) = (r: Results) => f(r(a), r(b)) def map[X](f: (A,B) => X): Task[X] = mapTask(a,b)(compute(f)) def bind[X](f: (A,B) => Result[X]): Task[X] = bindTask(a,b)(compute(f)) } - implicit def threeToBuilder[A,B,C](t: (Task[A], Task[B], Task[C])): Builder3[A,B,C] = t match { case (a,b,c) => new Builder3(a,b,c) } + implicit def threeToBuilder[A,B,C](t: (Task[A], Task[B], Task[C])): Builder3[A,B,C] = new Builder3(t._1,t._2,t._3) final class Builder3[A,B,C] private[Task](a: Task[A], b: Task[B], c: Task[C]) extends NotNull { - private def extract = (r: Results) => (r(a), r(b), r(c)) - private def compute[T](f: (A,B,C) => T) = tupled(f) compose extract + private def compute[T](f: (A,B,C) => T) = (r: Results) => f(r(a), r(b), r(c)) def map[X](f: (A,B,C) => X): Task[X] = mapTask(a,b,c)(compute(f)) def bind[X](f: (A,B,C) => Result[X]): Task[X] = bindTask(a,b,c)(compute(f)) } - implicit def fourToBuilder[A,B,C,D](t: (Task[A], Task[B], Task[C], Task[D])): Builder4[A,B,C,D] = t match { case (a,b,c,d) => new Builder4(a,b,c,d) } + implicit def fourToBuilder[A,B,C,D](t: (Task[A], Task[B], Task[C], Task[D])): Builder4[A,B,C,D] = new Builder4(t._1,t._2,t._3,t._4) final class Builder4[A,B,C,D] private[Task](a: Task[A], b: Task[B], c: Task[C], d: Task[D]) extends NotNull { - private def extract = (r: Results) => (r(a), r(b), r(c), r(d)) - private def compute[T](f: (A,B,C,D) => T) = tupled(f) compose extract + private def compute[T](f: (A,B,C,D) => T) = (r: Results) => f(r(a), r(b), r(c), r(d)) def map[X](f: (A,B,C,D) => X): Task[X] = mapTask(a,b,c,d)( compute(f) ) def bind[X](f: (A,B,C,D) => Result[X]): Task[X] = bindTask(a,b,c,d)( compute(f) ) } diff --git a/tasks/standard/Compile.scala b/tasks/standard/Compile.scala index 0b1210cfd..406d4266e 100644 --- a/tasks/standard/Compile.scala +++ b/tasks/standard/Compile.scala @@ -1,14 +1,20 @@ +/* sbt -- Simple Build Tool + * Copyright 2009, 2010 Mark Harrah + */ package xsbt import java.io.File + import xsbt.api.{APIFormat, SameAPI} + import xsbti.api.Source -trait Compile extends TrackedTaskDefinition[CompileReport] +trait CompileImpl[R] +{ + def apply(sourceChanges: ChangeReport[File], classpathChanges: ChangeReport[File], outputDirectory: File, options: Seq[String]): Task[R] + def tracked: Seq[Tracked] +} +final class Compile[R](val cacheDirectory: File, val sources: Task[Set[File]], val classpath: Task[Set[File]], + val outputDirectory: Task[File], val options: Task[Seq[String]], compileImpl: CompileImpl[R]) extends TrackedTaskDefinition[R] { - val sources: Task[Set[File]] - val classpath: Task[Set[File]] - val outputDirectory: Task[File] - val options: Task[Seq[String]] - val trackedClasspath = Difference.inputs(classpath, FilesInfo.lastModified, cacheFile("classpath")) val trackedSource = Difference.inputs(sources, FilesInfo.hash, cacheFile("sources")) val trackedOptions = @@ -17,121 +23,127 @@ trait Compile extends TrackedTaskDefinition[CompileReport] import Task._ new Changed((outputDirectory, options) map ( "-d" :: _.getAbsolutePath :: _.toList), cacheFile("options")) } - val invalidation = InvalidateFiles(cacheFile("dependencies/")) - lazy val task = create - def create = + val task = trackedClasspath { rawClasspathChanges => // detect changes to the classpath (last modified only) - trackedSource { rawSourceChanges =>// detect changes to sources (hash only) + trackedSource { rawSourceChanges => // detect changes to sources (hash only) val newOpts = (opts: Seq[String]) => (opts, rawSourceChanges.markAllModified, rawClasspathChanges.markAllModified) // if options changed, mark everything changed val sameOpts = (opts: Seq[String]) => (opts, rawSourceChanges, rawClasspathChanges) trackedOptions(newOpts, sameOpts) bind { // detect changes to options case (options, sourceChanges, classpathChanges) => - invalidation( classpathChanges +++ sourceChanges ) { (report, tracking) => // invalidation based on changes - outputDirectory bind { outDir => compile(sourceChanges, classpathChanges, outDir, options, report, tracking) } + outputDirectory bind { outDir => + FileUtilities.createDirectory(outDir) + compileImpl(sourceChanges, classpathChanges, outDir, options) } } } - } dependsOn(sources, options, outputDirectory)// raise these dependencies to the top for parallelism + } dependsOn(sources, classpath, options, outputDirectory)// raise these dependencies to the top for parallelism - def compile(sourceChanges: ChangeReport[File], classpathChanges: ChangeReport[File], outputDirectory: File, options: Seq[String], report: InvalidationReport[File], tracking: UpdateTracking[File]): Task[CompileReport] - lazy val tracked = getTracked - protected def getTracked = Seq(trackedClasspath, trackedSource, trackedOptions, invalidation) + lazy val tracked = Seq(trackedClasspath, trackedSource, trackedOptions) ++ compileImpl.tracked } -class StandardCompile(val sources: Task[Set[File]], val classpath: Task[Set[File]], val outputDirectory: Task[File], val options: Task[Seq[String]], - val superclassNames: Task[Set[String]], val compilerTask: Task[AnalyzingCompiler], val cacheDirectory: File, val log: CompileLogger) extends Compile -{ - import Task._ - import scala.collection.mutable.{ArrayBuffer, Buffer, HashMap, HashSet, Map, Set => mSet} - override def create = super.create dependsOn(superclassNames, compilerTask) // raise these dependencies to the top for parallelism - def compile(sourceChanges: ChangeReport[File], classpathChanges: ChangeReport[File], outputDirectory: File, - options: Seq[String], report: InvalidationReport[File], tracking: UpdateTracking[File]): Task[CompileReport] = +object AggressiveCompile +{ + def apply(sources: Task[Set[File]], classpath: Task[Set[File]], outputDirectory: Task[File], options: Task[Seq[String]], + cacheDirectory: File, compilerTask: Task[AnalyzingCompiler], log: CompileLogger): Compile[Set[File]] = { - val sources = report.invalid ** sourceChanges.checked // determine the sources that need recompiling (report.invalid also contains classes and libraries) - val classpath = classpathChanges.checked - compile(sources, classpath, outputDirectory, options, tracking) + val implCache = new File(cacheDirectory, "deps") + val baseCache = new File(cacheDirectory, "inputs") + val impl = new AggressiveCompile(implCache, compilerTask, log) + new Compile(baseCache, sources, classpath, outputDirectory, options, impl) } - def compile(sources: Set[File], classpath: Set[File], outputDirectory: File, options: Seq[String], tracking: UpdateTracking[File]): Task[CompileReport] = - { - (compilerTask, superclassNames) map { (compiler, superClasses) => - if(!sources.isEmpty) - { - val callback = new CompileAnalysisCallback(superClasses.toArray, tracking) - log.debug("Compile task calling compiler " + compiler) - compiler(sources, classpath, outputDirectory, options, callback, 100, log) - } - val readTracking = tracking.read - val applicationSet = new HashSet[String] - val subclassMap = new HashMap[String, Buffer[DetectedSubclass]] - readTags(applicationSet, subclassMap, readTracking) - new CompileReport - { - val superclasses = superClasses - def subclasses(superclass: String) = Set() ++ subclassMap.getOrElse(superclass, Nil) - val applications = Set() ++ applicationSet - val classes = Set() ++ readTracking.allProducts - override def toString = - { - val superStrings = superclasses.map(superC => superC + " >: \n\t\t" + subclasses(superC).mkString("\n\t\t")) - val applicationsPart = if(applications.isEmpty) Nil else Seq("Applications") ++ applications - val lines = Seq("Compilation Report:", sources.size + " sources", classes.size + " classes") ++ superStrings - lines.mkString("\n\t") +} + +class AggressiveCompile(val cacheDirectory: File, val compilerTask: Task[AnalyzingCompiler], val log: CompileLogger) extends CompileImpl[Set[File]] +{ + def apply(sourceChanges: ChangeReport[File], classpathChanges: ChangeReport[File], outputDirectory: File, options: Seq[String]): Task[Set[File]] = + compilerTask bind { compiler => + tracking { tracker => + Task { + log.info("Removed sources: \n\t" + sourceChanges.removed.mkString("\n\t")) + log.info("Added sources: \n\t" + sourceChanges.added.mkString("\n\t")) + log.info("Modified sources: \n\t" + (sourceChanges.modified -- sourceChanges.added -- sourceChanges.removed).mkString("\n\t")) + + val classpath = classpathChanges.checked + val readTracker = tracker.read + + def uptodate(time: Long, files: Iterable[File]) = files.forall(_.lastModified <= time) + def isProductOutofdate(product: File) = !product.exists || !uptodate(product.lastModified, readTracker.sources(product)) + val outofdateProducts = readTracker.allProducts.filter(isProductOutofdate) + + val rawInvalidatedSources = + classpathChanges.modified.flatMap(readTracker.usedBy) ++ + sourceChanges.removed.flatMap(readTracker.dependsOn) ++ + sourceChanges.modified ++ + outofdateProducts.flatMap(readTracker.sources) + val invalidatedSources = scc(readTracker, rawInvalidatedSources) + val sources = invalidatedSources.filter(_.exists) + val previousAPIMap = Map() ++ sources.map { src => (src, APIFormat.read(readTracker.tag(src))) } + val invalidatedProducts = outofdateProducts ++ products(readTracker, invalidatedSources) + + val transitiveIfNeeded = InvalidateTransitive(tracker, sources) + tracker.removeAll(invalidatedProducts ++ classpathChanges.modified ++ (invalidatedSources -- sources)) + tracker.pending(sources) + FileUtilities.delete(invalidatedProducts) + + log.info("Initially invalidated sources:\n\t" + sources.mkString("\n\t")) + if(!sources.isEmpty) + { + val newAPIMap = doCompile(sources, classpath, outputDirectory, options, tracker, compiler, log) + val apiChanged = sources filter { src => !sameAPI(previousAPIMap, newAPIMap, src) } + log.info("Sources with API changes:\n\t" + apiChanged.mkString("\n\t")) + val finalAPIMap = + if(apiChanged.isEmpty || apiChanged.size == sourceChanges.checked.size) newAPIMap + else + { + InvalidateTransitive.clean(tracker, FileUtilities.delete, transitiveIfNeeded) + val sources = transitiveIfNeeded.invalid ** sourceChanges.checked + log.info("All sources invalidated by API changes:\n\t" + sources.mkString("\n\t")) + doCompile(sources, classpath, outputDirectory, options, tracker, compiler, log) + } + finalAPIMap.foreach { case (src, api) => tracker.tag(src, APIFormat.write(api)) } + } + Set() ++ tracker.read.allProducts } } } - } - private def abs(f: Set[File]) = f.map(_.getAbsolutePath) - private def readTags(allApplications: mSet[String], subclassMap: Map[String, Buffer[DetectedSubclass]], readTracking: ReadTracking[File]) + def products(tracker: ReadTracking[File], srcs: Set[File]): Set[File] = srcs.flatMap(tracker.products) + + def doCompile(sources: Set[File], classpath: Set[File], outputDirectory: File, options: Seq[String], tracker: UpdateTracking[File], compiler: AnalyzingCompiler, log: CompileLogger): scala.collection.Map[File, Source] = { - for((source, tag) <- readTracking.allTags) if(tag.length > 0) - { - val (applications, subclasses) = Tag.fromBytes(tag) - allApplications ++= applications - subclasses.foreach(subclass => subclassMap.getOrElseUpdate(subclass.superclassName, new ArrayBuffer[DetectedSubclass]) += subclass) - } - } - private final class CompileAnalysisCallback(superClasses: Array[String], tracking: UpdateTracking[File]) extends xsbti.AnalysisCallback - { - private var applications = List[String]() - private var subclasses = List[DetectedSubclass]() - def superclassNames = superClasses - def superclassNotFound(superclassName: String) = error("Superclass not found: " + superclassName) - def beginSource(source: File) {} - def endSource(source: File) - { - if(!applications.isEmpty || !subclasses.isEmpty) - { - tracking.tag(source, Tag.toBytes(applications, subclasses) ) - applications = Nil - subclasses = Nil - } - } - def foundApplication(source: File, className: String) { applications ::= className } - def foundSubclass(source: File, subclassName: String, superclassName: String, isModule: Boolean): Unit = - subclasses ::= DetectedSubclass(source, subclassName, superclassName, isModule) - def sourceDependency(dependsOn: File, source: File) { tracking.dependency(source, dependsOn) } - def jarDependency(jar: File, source: File) { tracking.use(source, jar) } - def classDependency(clazz: File, source: File) { tracking.dependency(source, clazz) } - def generatedClass(source: File, clazz: File) { tracking.product(source, clazz) } - def api(source: File, api: xsbti.api.Source) = () + val callback = new APIAnalysisCallback(tracker) + log.debug("Compiling using compiler " + compiler) + compiler(sources, classpath, outputDirectory, options, callback, 100, log) + callback.apiMap } + + import sbinary.DefaultProtocol.FileFormat + val tracking = new DependencyTracked(cacheDirectory, true, (files: File) => FileUtilities.delete(files)) + def tracked = Seq(tracking) + + def sameAPI[T](a: scala.collection.Map[T, Source], b: scala.collection.Map[T, Source], t: T): Boolean = sameAPI(a.get(t), b.get(t)) + def sameAPI(a: Option[Source], b: Option[Source]): Boolean = + if(a.isEmpty) b.isEmpty else (b.isDefined && SameAPI(a.get, b.get)) + + // TODO: implement + def scc(readTracker: ReadTracking[File], sources: Set[File]) = sources } -object Tag + +private final class APIAnalysisCallback(tracking: UpdateTracking[File]) extends xsbti.AnalysisCallback { - import sbinary.{DefaultProtocol, Format, Operations} - import DefaultProtocol._ - private implicit val subclassFormat: Format[DetectedSubclass] = - asProduct4(DetectedSubclass.apply)( ds => Some(ds.source, ds.subclassName, ds.superclassName, ds.isModule)) - def toBytes(applications: List[String], subclasses: List[DetectedSubclass]) = CacheIO.toBytes((applications, subclasses)) - def fromBytes(bytes: Array[Byte]) = CacheIO.fromBytes( ( List[String](), List[DetectedSubclass]() ) )(bytes) + val apiMap = new scala.collection.mutable.HashMap[File, Source] + + def sourceDependency(dependsOn: File, source: File) { tracking.dependency(source, dependsOn) } + def jarDependency(jar: File, source: File) { tracking.use(source, jar) } + def classDependency(clazz: File, source: File) { tracking.dependency(source, clazz) } + def generatedClass(source: File, clazz: File) { tracking.product(source, clazz) } + def api(source: File, api: xsbti.api.Source) { apiMap(source) = api } + + def superclassNames = Array() + def superclassNotFound(superclassName: String) {} + def beginSource(source: File) {} + def endSource(source: File) {} + def foundApplication(source: File, className: String) {} + def foundSubclass(source: File, subclassName: String, superclassName: String, isModule: Boolean) {} } -trait CompileReport extends NotNull -{ - def classes: Set[File] - def applications: Set[String] - def superclasses: Set[String] - def subclasses(superclass: String): Set[DetectedSubclass] -} -final case class DetectedSubclass(source: File, subclassName: String, superclassName: String, isModule: Boolean) extends NotNull \ No newline at end of file diff --git a/tasks/standard/TaskDefinition.scala b/tasks/standard/TaskDefinition.scala index 8c35fcb43..2eec9d487 100644 --- a/tasks/standard/TaskDefinition.scala +++ b/tasks/standard/TaskDefinition.scala @@ -1,3 +1,6 @@ +/* sbt -- Simple Build Tool + * Copyright 2009, 2010 Mark Harrah + */ package xsbt import java.io.File @@ -7,12 +10,15 @@ trait TaskDefinition[T] val task: Task[T] val clean: Task[Unit] } -trait TrackedTaskDefinition[T] extends TaskDefinition[T] +trait TrackedTaskDefinition[T] extends TaskDefinition[T] with WithCache { - def cacheDirectory: File - def cacheFile(relative: String) = new File(cacheDirectory, relative) val tracked: Seq[Tracked] lazy val clean: Task[Unit] = onTracked(_.clean).bind( u => onTracked(_.clear) ) import Task._ private def onTracked(f: Tracked => Task[Unit]) = tracked.forkTasks(f).joinIgnore +} +trait WithCache +{ + def cacheDirectory: File + def cacheFile(relative: String) = new File(cacheDirectory, relative) } \ No newline at end of file diff --git a/tasks/standard/TransitiveCompile.scala b/tasks/standard/TransitiveCompile.scala new file mode 100644 index 000000000..93852d89c --- /dev/null +++ b/tasks/standard/TransitiveCompile.scala @@ -0,0 +1,118 @@ +/* sbt -- Simple Build Tool + * Copyright 2009, 2010 Mark Harrah + */ +package xsbt + + import java.io.File + + +trait TransitiveCompile extends CompileImpl[CompileReport] with WithCache +{ + final val invalidation = InvalidateFiles(cacheFile("dependencies/")) + def tracked = Seq(invalidation) + + def apply(sourceChanges: ChangeReport[File], classpathChanges: ChangeReport[File], outputDirectory: File, options: Seq[String]): Task[CompileReport] = + // doesn't notice if classes are removed + invalidation( classpathChanges +++ sourceChanges ) { (report, tracking) => // invalidation based on changes + compile(sourceChanges, classpathChanges, outputDirectory, options, report, tracking) + } + def compile(sourceChanges: ChangeReport[File], classpathChanges: ChangeReport[File], outputDirectory: File, + options: Seq[String], report: InvalidationReport[File], tracking: UpdateTracking[File]): Task[CompileReport] +} + +class StandardCompile(val sources: Task[Set[File]], val classpath: Task[Set[File]], val outputDirectory: Task[File], val options: Task[Seq[String]], + val superclassNames: Task[Set[String]], val compilerTask: Task[AnalyzingCompiler], val cacheDirectory: File, val log: CompileLogger) extends TransitiveCompile +{ + import Task._ + import scala.collection.mutable.{ArrayBuffer, Buffer, HashMap, HashSet, Map, Set => mSet} + + def compile(sourceChanges: ChangeReport[File], classpathChanges: ChangeReport[File], outputDirectory: File, + options: Seq[String], report: InvalidationReport[File], tracking: UpdateTracking[File]): Task[CompileReport] = + { + val sources = report.invalid ** sourceChanges.checked // determine the sources that need recompiling (report.invalid also contains classes and libraries) + val classpath = classpathChanges.checked + compile(sources, classpath, outputDirectory, options, tracking) + } + def compile(sources: Set[File], classpath: Set[File], outputDirectory: File, options: Seq[String], tracking: UpdateTracking[File]): Task[CompileReport] = + { + (compilerTask, superclassNames) map { (compiler, superClasses) => + if(!sources.isEmpty) + { + val callback = new CompileAnalysisCallback(superClasses.toArray, tracking) + log.debug("Compile task calling compiler " + compiler) + compiler(sources, classpath, outputDirectory, options, callback, 100, log) + } + val readTracking = tracking.read + val applicationSet = new HashSet[String] + val subclassMap = new HashMap[String, Buffer[DetectedSubclass]] + readTags(applicationSet, subclassMap, readTracking) + new CompileReport + { + val superclasses = superClasses + def subclasses(superclass: String) = Set() ++ subclassMap.getOrElse(superclass, Nil) + val applications = Set() ++ applicationSet + val classes = Set() ++ readTracking.allProducts + override def toString = + { + val superStrings = superclasses.map(superC => superC + " >: \n\t\t" + subclasses(superC).mkString("\n\t\t")) + val applicationsPart = if(applications.isEmpty) Nil else Seq("Applications") ++ applications + val lines = Seq("Compilation Report:", sources.size + " sources", classes.size + " classes") ++ superStrings + lines.mkString("\n\t") + } + } + } + } + private def abs(f: Set[File]) = f.map(_.getAbsolutePath) + private def readTags(allApplications: mSet[String], subclassMap: Map[String, Buffer[DetectedSubclass]], readTracking: ReadTracking[File]) + { + for((source, tag) <- readTracking.allTags) if(tag.length > 0) + { + val (applications, subclasses) = Tag.fromBytes(tag) + allApplications ++= applications + subclasses.foreach(subclass => subclassMap.getOrElseUpdate(subclass.superclassName, new ArrayBuffer[DetectedSubclass]) += subclass) + } + } + private final class CompileAnalysisCallback(superClasses: Array[String], tracking: UpdateTracking[File]) extends xsbti.AnalysisCallback + { + private var applications = List[String]() + private var subclasses = List[DetectedSubclass]() + def superclassNames = superClasses + def superclassNotFound(superclassName: String) = error("Superclass not found: " + superclassName) + def beginSource(source: File) {} + def endSource(source: File) + { + if(!applications.isEmpty || !subclasses.isEmpty) + { + tracking.tag(source, Tag.toBytes(applications, subclasses) ) + applications = Nil + subclasses = Nil + } + } + def foundApplication(source: File, className: String) { applications ::= className } + def foundSubclass(source: File, subclassName: String, superclassName: String, isModule: Boolean): Unit = + subclasses ::= DetectedSubclass(source, subclassName, superclassName, isModule) + def sourceDependency(dependsOn: File, source: File) { tracking.dependency(source, dependsOn) } + def jarDependency(jar: File, source: File) { tracking.use(source, jar) } + def classDependency(clazz: File, source: File) { tracking.dependency(source, clazz) } + def generatedClass(source: File, clazz: File) { tracking.product(source, clazz) } + def api(source: File, api: xsbti.api.Source) = () + } +} + +object Tag +{ + import sbinary.{DefaultProtocol, Format, Operations} + import DefaultProtocol._ + private implicit val subclassFormat: Format[DetectedSubclass] = + asProduct4(DetectedSubclass.apply)( ds => Some(ds.source, ds.subclassName, ds.superclassName, ds.isModule)) + def toBytes(applications: List[String], subclasses: List[DetectedSubclass]) = CacheIO.toBytes((applications, subclasses)) + def fromBytes(bytes: Array[Byte]) = CacheIO.fromBytes( ( List[String](), List[DetectedSubclass]() ) )(bytes) +} +trait CompileReport extends NotNull +{ + def classes: Set[File] + def applications: Set[String] + def superclasses: Set[String] + def subclasses(superclass: String): Set[DetectedSubclass] +} +final case class DetectedSubclass(source: File, subclassName: String, superclassName: String, isModule: Boolean) extends NotNull \ No newline at end of file