Compile task with dependency tracking. Checkpoint: compiles successfully.

This commit is contained in:
Mark Harrah 2009-08-30 11:10:37 -04:00
parent 573994dd4e
commit 6e414e96c5
10 changed files with 292 additions and 44 deletions

View File

@ -5,18 +5,17 @@ import CacheIO.{fromFile, toFile}
import sbinary.Format
import scala.reflect.Manifest
object DependencyTracking
trait Tracked extends NotNull
{
def trackBasic[T, F <: FileInfo](filesTask: Task[Set[File]], style: FilesInfo.Style[F], cacheDirectory: File)
(f: (ChangeReport[File], InvalidationReport[File], UpdateTracking[File]) => Task[T])(implicit mf: Manifest[F]): Task[T] =
{
changed(filesTask, style, new File(cacheDirectory, "files")) { sourceChanges =>
invalidate(sourceChanges, cacheDirectory) { (report, tracking) =>
f(sourceChanges, report, tracking)
}
}
}
def changed[O,O2](task: Task[O], file: File)(ifChanged: O => O2, ifUnchanged: O => O2)(implicit input: InputCache[O]): Task[O2] { type Input = O } =
def clear: Task[Unit]
def clean: Task[Unit]
}
class Changed[O](val task: Task[O], val file: File)(implicit input: InputCache[O]) extends Tracked
{
def clean = Task.empty
def clear = Clean(file)
def apply[O2](ifChanged: O => O2, ifUnchanged: O => O2): Task[O2] { type Input = O } =
task map { value =>
val cache = OpenResource.fileInputStream(file)(input.uptodate(value))
if(cache.uptodate)
@ -27,14 +26,24 @@ object DependencyTracking
ifChanged(value)
}
}
def changed[T, F <: FileInfo](files: Set[File], style: FilesInfo.Style[F], cache: File)
(f: ChangeReport[File] => Task[T])(implicit mf: Manifest[F]): Task[T] =
changed(Task(files), style, cache)(f)
def changed[T, F <: FileInfo](filesTask: Task[Set[File]], style: FilesInfo.Style[F], cache: File)
(f: ChangeReport[File] => Task[T])(implicit mf: Manifest[F]): Task[T] =
}
class Difference[F <: FileInfo](val filesTask: Task[Set[File]], val style: FilesInfo.Style[F], val cache: File, val shouldClean: Boolean)(implicit mf: Manifest[F]) extends Tracked
{
def this(filesTask: Task[Set[File]], style: FilesInfo.Style[F], cache: File)(implicit mf: Manifest[F]) = this(filesTask, style, cache, false)
def this(files: Set[File], style: FilesInfo.Style[F], cache: File, shouldClean: Boolean)(implicit mf: Manifest[F]) = this(Task(files), style, cache)
def this(files: Set[File], style: FilesInfo.Style[F], cache: File)(implicit mf: Manifest[F]) = this(Task(files), style, cache, false)
val clear = Clean(cache)
val clean = if(shouldClean) cleanTask else Task.empty
def cleanTask = Clean(Task(raw(cachedFilesInfo)))
private def cachedFilesInfo = fromFile(style.formats)(cache).files
private def raw(fs: Set[F]): Set[File] = fs.map(_.file)
def apply[T](f: ChangeReport[File] => Task[T]): Task[T] =
filesTask bind { files =>
val lastFilesInfo = fromFile(style.formats)(cache).files
val lastFiles = lastFilesInfo.map(_.file)
val lastFilesInfo = cachedFilesInfo
val lastFiles = raw(lastFilesInfo)
val currentFiles = files.map(_.getAbsoluteFile)
val currentFilesInfo = style(files)
@ -43,7 +52,7 @@ object DependencyTracking
lazy val allInputs = currentFiles
lazy val removed = lastFiles -- allInputs
lazy val added = allInputs -- lastFiles
lazy val modified = (lastFilesInfo -- currentFilesInfo.files).map(_.file)
lazy val modified = raw(lastFilesInfo -- currentFilesInfo.files)
lazy val unmodified = allInputs -- modified
}
@ -52,20 +61,32 @@ object DependencyTracking
result
}
}
def invalidate[R](changes: ChangeReport[File], cacheDirectory: File)(f: (InvalidationReport[File], UpdateTracking[File]) => Task[R]): Task[R] =
}
object InvalidateFiles
{
def apply(cacheDirectory: File): Invalidate[File] = apply(cacheDirectory, true)
def apply(cacheDirectory: File, translateProducts: Boolean): Invalidate[File] =
{
val pruneAndF = (report: InvalidationReport[File], tracking: UpdateTracking[File]) => {
report.invalidProducts.foreach(_.delete)
f(report, tracking)
}
implicit val format = sbinary.DefaultProtocol.FileFormat
invalidate(Task(changes), cacheDirectory, true)(pruneAndF)
import sbinary.DefaultProtocol.FileFormat
new Invalidate[File](cacheDirectory, translateProducts, FileUtilities.delete)
}
def invalidate[T,R](changesTask: Task[ChangeReport[T]], cacheDirectory: File, translateProducts: Boolean)
(f: (InvalidationReport[T], UpdateTracking[T]) => Task[R])(implicit format: Format[T], mf: Manifest[T]): Task[R] =
}
class Invalidate[T](val cacheDirectory: File, val translateProducts: Boolean, cleanT: T => Unit)
(implicit format: Format[T], mf: Manifest[T]) extends Tracked
{
def this(cacheDirectory: File, translateProducts: Boolean)(implicit format: Format[T], mf: Manifest[T]) =
this(cacheDirectory, translateProducts, x => ())
private val trackFormat = new TrackingFormat[T](cacheDirectory, translateProducts)
private def cleanAll(fs: Set[T]) = fs.foreach(cleanT)
def clear = Clean(cacheDirectory)
def clean = Task(cleanAll(trackFormat.read.allProducts))
def apply[R](changes: ChangeReport[T])(f: (InvalidationReport[T], UpdateTracking[T]) => Task[R]): Task[R] =
apply(Task(changes))(f)
def apply[R](changesTask: Task[ChangeReport[T]])(f: (InvalidationReport[T], UpdateTracking[T]) => Task[R]): Task[R] =
{
changesTask bind { changes =>
val trackFormat = new TrackingFormat[T](cacheDirectory, translateProducts)
val tracker = trackFormat.read
def invalidatedBy(file: T) = tracker.products(file) ++ tracker.sources(file) ++ tracker.usedBy(file) ++ tracker.dependsOn(file)
@ -89,17 +110,36 @@ object DependencyTracking
val invalidProducts = Set(invalidatedProducts.toSeq : _*)
val valid = changes.unmodified -- invalid
}
cleanAll(report.invalidProducts)
f(report, tracker) map { result =>
trackFormat.write(tracker)
result
}
}
}
import scala.collection.mutable.{Set, HashMap, MultiMap}
private[xsbt] type DependencyMap[T] = HashMap[T, Set[T]] with MultiMap[T, T]
private[xsbt] def newMap[T]: DependencyMap[T] = new HashMap[T, Set[T]] with MultiMap[T, T]
}
class BasicTracked[F <: FileInfo](filesTask: Task[Set[File]], style: FilesInfo.Style[F], cacheDirectory: File)(implicit mf: Manifest[F]) extends Tracked
{
private val changed = new Difference(filesTask, style, new File(cacheDirectory, "files"))
private val invalidation = InvalidateFiles(cacheDirectory)
val clean = invalidation.clean
val clear = Clean(cacheDirectory)
def apply[R](f: (ChangeReport[File], InvalidationReport[File], UpdateTracking[File]) => Task[R]): Task[R] =
changed { sourceChanges =>
invalidation(sourceChanges) { (report, tracking) =>
f(sourceChanges, report, tracking)
}
}
}
private object DependencyTracking
{
import scala.collection.mutable.{Set, HashMap, Map, MultiMap}
type DependencyMap[T] = HashMap[T, Set[T]] with MultiMap[T, T]
def newMap[T]: DependencyMap[T] = new HashMap[T, Set[T]] with MultiMap[T, T]
type TagMap[T] = Map[T, Array[Byte]]
def newTagMap[T] = new HashMap[T, Array[Byte]]
}
trait UpdateTracking[T] extends NotNull
@ -107,6 +147,14 @@ trait UpdateTracking[T] extends NotNull
def dependency(source: T, dependsOn: T): Unit
def use(source: T, uses: T): Unit
def product(source: T, output: T): Unit
def tag(source: T, t: Array[Byte]): Unit
def read: ReadTracking[T]
}
object Clean
{
def apply(src: Task[Set[File]]): Task[Unit] = src map FileUtilities.delete
def apply(srcs: File*): Task[Unit] = Task(FileUtilities.delete(srcs))
def apply(srcs: Set[File]): Task[Unit] = Task(FileUtilities.delete(srcs))
}
import scala.collection.Set
trait ReadTracking[T] extends NotNull
@ -115,9 +163,15 @@ trait ReadTracking[T] extends NotNull
def products(file: T): Set[T]
def sources(file: T): Set[T]
def usedBy(file: T): Set[T]
def allProducts: Set[T]
def allSources: Set[T]
def allUsed: Set[T]
def allTags: Seq[(T,Array[Byte])]
}
import DependencyTracking.{DependencyMap => DMap, newMap}
private final class DefaultTracking[T](translateProducts: Boolean)(val reverseDependencies: DMap[T], val reverseUses: DMap[T], val sourceMap: DMap[T]) extends DependencyTracking[T](translateProducts)
import DependencyTracking.{DependencyMap => DMap, newMap, TagMap}
private final class DefaultTracking[T](translateProducts: Boolean)
(val reverseDependencies: DMap[T], val reverseUses: DMap[T], val sourceMap: DMap[T], val tagMap: TagMap[T])
extends DependencyTracking[T](translateProducts)
{
val productMap: DMap[T] = forward(sourceMap) // map from a source to its products. Keep in sync with sourceMap
}
@ -128,11 +182,20 @@ private abstract class DependencyTracking[T](translateProducts: Boolean) extends
val reverseUses: DMap[T] // map from a file to the files that use it
val sourceMap: DMap[T] // map from a product to its sources. Keep in sync with productMap
val productMap: DMap[T] // map from a source to its products. Keep in sync with sourceMap
val tagMap: TagMap[T]
def read = this
final def dependsOn(file: T): Set[T] = get(reverseDependencies, file)
final def products(file: T): Set[T] = get(productMap, file)
final def sources(file: T): Set[T] = get(sourceMap, file)
final def usedBy(file: T): Set[T] = get(reverseUses, file)
final def tag(file: T): Array[Byte] = tagMap.getOrElse(file, new Array[Byte](0))
final def allProducts = Set() ++ sourceMap.keys
final def allSources = Set() ++ productMap.keys
final def allUsed = Set() ++ reverseUses.keys
final def allTags = tagMap.toSeq
private def get(map: DMap[T], value: T): Set[T] = map.getOrElse(value, Set.empty[T])
@ -151,6 +214,7 @@ private abstract class DependencyTracking[T](translateProducts: Boolean) extends
sourceMap.add(product, sourceFile)
}
final def use(sourceFile: T, usesFile: T) { reverseUses.add(usesFile, sourceFile) }
final def tag(sourceFile: T, t: Array[Byte]) { tagMap(sourceFile) = t }
final def removeAll(files: Iterable[T])
{
@ -162,6 +226,7 @@ private abstract class DependencyTracking[T](translateProducts: Boolean) extends
removeAll(forward(reverseDependencies), reverseDependencies)
removeAll(productMap, sourceMap)
removeAll(forward(reverseUses), reverseUses)
tagMap --= files
}
protected final def forward(map: DMap[T]): DMap[T] =
{
@ -169,4 +234,4 @@ private abstract class DependencyTracking[T](translateProducts: Boolean) extends
for( (key, values) <- map; value <- values) f.add(value, key)
f
}
}
}

View File

@ -7,7 +7,7 @@ import sbinary.{DefaultProtocol, Format}
import DefaultProtocol._
import TrackingFormat._
import CacheIO.{fromFile, toFile}
import DependencyTracking.{DependencyMap => DMap, newMap}
import DependencyTracking.{DependencyMap => DMap, newMap, TagMap}
private class TrackingFormat[T](directory: File, translateProducts: Boolean)(implicit tFormat: Format[T], mf: Manifest[T]) extends NotNull
{
@ -42,7 +42,11 @@ private object TrackingFormat
}
}
def trackingFormat[T](translateProducts: Boolean)(implicit tFormat: Format[T]): Format[DependencyTracking[T]] =
asProduct3((a: DMap[T],b: DMap[T],c: DMap[T]) => new DefaultTracking(translateProducts)(a,b,c) : DependencyTracking[T])(dt => Some(dt.reverseDependencies, dt.reverseUses, dt.sourceMap))
{
implicit val arrayFormat = sbinary.Operations.format[Array[Byte]]
asProduct4((a: DMap[T],b: DMap[T],c: DMap[T], d:TagMap[T]) => new DefaultTracking(translateProducts)(a,b,c,d) : DependencyTracking[T]
)(dt => Some(dt.reverseDependencies, dt.reverseUses, dt.sourceMap, dt.tagMap))
}
}
private final class IndexMap[T] extends NotNull

View File

@ -29,7 +29,7 @@ class RawCompiler(scalaLoader: ClassLoader)
}
}
/** Interface to the compiler that uses the dependency analysis plugin.*/
class AnalyzeCompile(scalaVersion: String, scalaLoader: ClassLoader, manager: ComponentManager) extends NotNull
class AnalyzeCompiler(scalaVersion: String, scalaLoader: ClassLoader, manager: ComponentManager) extends NotNull
{
def this(scalaVersion: String, provider: xsbti.ScalaProvider, manager: ComponentManager) =
this(scalaVersion, provider.getScalaLoader(scalaVersion), manager)

View File

@ -17,12 +17,12 @@ object CompileTest extends Specification
val manager = new ComponentManager(launch.getSbtHome(sbtVersion, scalaVersion), log)
prepare(manager, ComponentCompiler.compilerInterfaceSrcID, "CompilerInterface.scala")
prepare(manager, ComponentCompiler.xsbtiID, classOf[xsbti.AnalysisCallback])
testCompileAnalysis(new AnalyzeCompile(scalaVersion, launch, manager), log)
testCompileAnalysis(new AnalyzeCompiler(scalaVersion, launch, manager), log)
}
}
}
}
private def testCompileAnalysis(compiler: AnalyzeCompile, log: xsbti.Logger)
private def testCompileAnalysis(compiler: AnalyzeCompiler, log: xsbti.Logger)
{
WithFiles( new File("Test.scala") -> "object Test" ) { sources =>
withTemporaryDirectory { temp =>

View File

@ -21,9 +21,9 @@ class XSbt(info: ProjectInfo) extends ParentProject(info)
val taskSub = project(tasksPath, "Tasks", new TaskProject(_), controlSub, collectionSub)
val cacheSub = project("cache", "Cache", new CacheProject(_), taskSub, ioSub)
val stdTaskSub = project(tasksPath / "standard", "Standard Tasks", new Base(_), taskSub, cacheSub)
val compilerSub = project(compilePath, "Compile", new CompileProject(_),
launchInterfaceSub, interfaceSub, ivySub, ioSub, classpathSub, compileInterfaceSub)
val stdTaskSub = project(tasksPath / "standard", "Standard Tasks", new Base(_), taskSub, cacheSub, compilerSub)
/* Multi-subproject paths */

View File

@ -47,6 +47,8 @@ private trait Results extends NotNull
object Task
{
val empty = Task(())
type ITask[I,O] = Task[O] { type Input = I }
import Function.tupled
def apply[O](o: => O): ITask[Unit,O] =

View File

@ -0,0 +1,119 @@
package xsbt
import java.io.File
trait Compile extends TrackedTaskDefinition[CompileReport]
{
val sources: Task[Set[File]]
val classpath: Task[Set[File]]
val options: Task[Seq[String]]
val trackedClasspath = new Difference(classpath, FilesInfo.lastModified, cacheFile("classpath"))
val trackedSource = new Difference(sources, FilesInfo.hash, cacheFile("sources"))
val trackedOptions =
{
import Cache._
new Changed(options.map(_.toList), cacheFile("options"))
}
val invalidation = InvalidateFiles(cacheFile("dependencies/"))
lazy val task = create
def create =
trackedClasspath { rawClasspathChanges => // detect changes to the classpath (last modified only)
trackedSource { rawSourceChanges =>// detect changes to sources (hash only)
val newOpts = (opts: Seq[String]) => (opts, rawSourceChanges.markAllModified, rawClasspathChanges.markAllModified) // if options changed, mark everything changed
val sameOpts = (opts: Seq[String]) => (opts, rawSourceChanges, rawClasspathChanges)
trackedOptions(newOpts, sameOpts) bind { // detect changes to options
case (options, classpathChanges, sourceChanges) =>
invalidation( classpathChanges +++ sourceChanges ) { (report, tracking) => // invalidation based on changes
compile(sourceChanges, classpathChanges, options, report, tracking)
}
}
}
} dependsOn(sources, options)// raise these dependencies to the top for parallelism
def compile(sourceChanges: ChangeReport[File], classpathChanges: ChangeReport[File], options: Seq[String], report: InvalidationReport[File], tracking: UpdateTracking[File]): Task[CompileReport]
lazy val tracked = getTracked
protected def getTracked = Seq(trackedClasspath, trackedSource, trackedOptions, invalidation)
}
class StandardCompile(val sources: Task[Set[File]], val classpath: Task[Set[File]], val options: Task[Seq[String]],
val superclassNames: Task[Set[String]], val compilerTask: Task[AnalyzeCompiler], val cacheDirectory: File, val log: xsbti.Logger) extends Compile
{
import Task._
import sbinary.{DefaultProtocol, Format, Operations}
import DefaultProtocol._
import Operations.{fromByteArray, toByteArray}
import scala.collection.mutable.{ArrayBuffer, Buffer, HashMap, HashSet, Map, Set => mSet}
private implicit val subclassFormat: Format[DetectedSubclass] =
asProduct4(DetectedSubclass.apply)( ds => Some(ds.source, ds.subclassName, ds.superclassName, ds.isModule))
override def create = super.create dependsOn(superclassNames, compilerTask) // raise these dependencies to the top for parallelism
def compile(sourceChanges: ChangeReport[File], classpathChanges: ChangeReport[File], options: Seq[String], report: InvalidationReport[File], tracking: UpdateTracking[File]): Task[CompileReport] =
{
val sources = report.invalid ** sourceChanges.allInputs // determine the sources that need recompiling (report.invalid also contains classes and libraries)
val classpath = classpathChanges.allInputs
(compilerTask, superclassNames) map { (compiler, superClasses) =>
val callback = new CompileAnalysisCallback(superClasses.toArray, tracking)
val arguments = Seq("-cp", abs(classpath).mkString(File.pathSeparator)) ++ options ++ abs(sources).toSeq
compiler(arguments, callback, 100, log)
val readTracking = tracking.read
val applicationSet = new HashSet[String]
val subclassMap = new HashMap[String, Buffer[DetectedSubclass]]
readTags(applicationSet, subclassMap, readTracking)
new CompileReport
{
val superclasses = superClasses
def subclasses(superclass: String) = Set() ++ subclassMap.getOrElse(superclass, Nil)
val applications = Set() ++ applicationSet
val classes = Set() ++ readTracking.allProducts
}
}
}
private def abs(f: Set[File]) = f.map(_.getAbsolutePath)
private def readTags(allApplications: mSet[String], subclassMap: Map[String, Buffer[DetectedSubclass]], readTracking: ReadTracking[File])
{
for((source, tag) <- readTracking.allTags) if(tag.length > 0)
{
val (applications, subclasses) = fromByteArray[(List[String], List[DetectedSubclass])](tag)
allApplications ++= applications
subclasses.foreach(subclass => subclassMap.getOrElseUpdate(subclass.superclassName, new ArrayBuffer[DetectedSubclass]) += subclass)
}
}
private final class CompileAnalysisCallback(superClasses: Array[String], tracking: UpdateTracking[File]) extends xsbti.AnalysisCallback
{
private var applications = List[(File,String)]()
private var subclasses = List[DetectedSubclass]()
def superclassNames = superClasses
def superclassNotFound(superclassName: String) = error("Superclass not found: " + superclassName)
def beginSource(source: File) {}
def endSource(source: File)
{
if(!applications.isEmpty || !subclasses.isEmpty)
{
tracking.tag(source, toByteArray( (applications, subclasses) ) )
applications = Nil
subclasses = Nil
}
}
def foundApplication(source: File, className: String) { applications ::= ( (source, className) ) }
def foundSubclass(source: File, subclassName: String, superclassName: String, isModule: Boolean): Unit =
subclasses ::= DetectedSubclass(source, subclassName, superclassName, isModule)
def sourceDependency(dependsOn: File, source: File) { tracking.dependency(source, dependsOn) }
def jarDependency(jar: File, source: File) { tracking.use(source, jar) }
def classDependency(clazz: File, source: File) { tracking.dependency(source, clazz) }
def generatedClass(source: File, clazz: File) { tracking.product(source, clazz) }
}
}
trait CompileReport extends NotNull
{
def classes: Set[File]
def applications: Set[String]
def superclasses: Set[String]
def subclasses(superclass: String): Set[DetectedSubclass]
}
final case class DetectedSubclass(source: File, subclassName: String, superclassName: String, isModule: Boolean) extends NotNull

39
tasks/standard/Sync.scala Normal file
View File

@ -0,0 +1,39 @@
package xsbt
import java.io.File
import Task._
object Sync
{
def sources(inputDirectory: Task[File], outputDirectory: Task[File]) =
{
import Paths._
(inputDirectory, outputDirectory) map { (in, out) =>
FileUtilities.assertDirectories(in, out)
(in ***) x FileMapper.rebase(in, out)
}
}
}
class Sync(val sources: Task[Iterable[(File,File)]], val cacheDirectory: File) extends TrackedTaskDefinition[Set[File]]
{
val tracking = new BasicTracked(sources.map(Set() ++ _.map(_._1)), FilesInfo.hash, cacheFile("sources"))
val tracked = Seq(tracking)
def this(inputDirectory: Task[File], outputDirectory: Task[File], cacheFile: File) = this(Sync.sources(inputDirectory, outputDirectory), cacheFile)
lazy val task =
sources bind { srcs =>
val sourcesTargets = srcs.toSeq
tracking { (sourceChanges, report, tracking) =>
Task
{
val changed = report.invalid ** sourceChanges.allInputs
for((source,target) <- sourcesTargets if changed(source))
{
FileUtilities.copyFile(source, target)
tracking.product(source, target)
}
Set( sourcesTargets.map(_._2) : _*)
}
}
}
}

View File

@ -0,0 +1,19 @@
package xsbt
import java.io.File
trait TaskDefinition[T]
{
val task: Task[T]
val clean: Task[Unit]
val clear: Task[Unit]
}
trait TrackedTaskDefinition[T] extends TaskDefinition[T]
{
def cacheDirectory: File
def cacheFile(relative: String) = new File(cacheDirectory, relative)
val tracked: Seq[Tracked]
val clear: Task[Unit] = foreachCache(_.clear)
val clean: Task[Unit] = foreachCache(_.clean)
private def foreachCache(f: Tracked => Task[Unit]): Task[Unit] = tracked.map(f).join.map(i => ())
}

View File

@ -54,7 +54,7 @@ object FileUtilities
}
}
def assertDirectory(file: File) { assert(file.isDirectory, (if(file.exists) "Not a directory: " else "Directory not found: ") + file) }
def assertDirectory(file: File*) { file.foreach(assertDirectory) }
def assertDirectories(file: File*) { file.foreach(assertDirectory) }
// "base.extension" -> (base, extension)
def split(name: String): (String, String) =