From 56547aad299b726a280e421ad6f6c128f6084fbf Mon Sep 17 00:00:00 2001 From: Mark Harrah Date: Wed, 26 Aug 2009 08:38:20 -0400 Subject: [PATCH] Composable dependency tracking on top of Tasks. --- cache/ChangeReport.scala | 25 +++++ cache/DependencyTracking.scala | 154 ++++++++++++++++++++++++++++ cache/TrackingFormat.scala | 52 ++++++++++ cache/src/test/scala/Tracking.scala | 45 ++++++++ project/build/XSbt.scala | 21 +++- util/io/PathMapper.scala | 34 ++++-- 6 files changed, 317 insertions(+), 14 deletions(-) create mode 100644 cache/ChangeReport.scala create mode 100644 cache/DependencyTracking.scala create mode 100644 cache/TrackingFormat.scala create mode 100644 cache/src/test/scala/Tracking.scala diff --git a/cache/ChangeReport.scala b/cache/ChangeReport.scala new file mode 100644 index 000000000..baa078034 --- /dev/null +++ b/cache/ChangeReport.scala @@ -0,0 +1,25 @@ +package xsbt + +trait ChangeReport[T] extends NotNull +{ + def allInputs: Set[T] + def unmodified: Set[T] + def modified: Set[T] // all changes, including added + def added: Set[T] + def removed: Set[T] + def +++(other: ChangeReport[T]): ChangeReport[T] = new CompoundChangeReport(this, other) +} +trait InvalidationReport[T] extends NotNull +{ + def valid: Set[T] + def invalid: Set[T] + def invalidProducts: Set[T] +} +private class CompoundChangeReport[T](a: ChangeReport[T], b: ChangeReport[T]) extends ChangeReport[T] +{ + lazy val allInputs = a.allInputs ++ b.allInputs + lazy val unmodified = a.unmodified ++ b.unmodified + lazy val modified = a.modified ++ b.modified + lazy val added = a.added ++ b.added + lazy val removed = a.removed ++ b.removed +} \ No newline at end of file diff --git a/cache/DependencyTracking.scala b/cache/DependencyTracking.scala new file mode 100644 index 000000000..931efba1c --- /dev/null +++ b/cache/DependencyTracking.scala @@ -0,0 +1,154 @@ +package xsbt + +import java.io.File +import sbinary.{Format, Operations} + +object DependencyTracking +{ + def trackBasic[T, F <: FileInfo](filesTask: Task[Set[File]], style: FilesInfo.Style[F], cacheDirectory: File) + (f: (ChangeReport[File], InvalidationReport[File], UpdateTracking[File]) => Task[T]): Task[T] = + { + changed(filesTask, style, new File(cacheDirectory, "files")) { sourceChanges => + invalidate(sourceChanges, cacheDirectory) { (report, tracking) => + f(sourceChanges, report, tracking) + } + } + } + def changed[T, F <: FileInfo](filesTask: Task[Set[File]], style: FilesInfo.Style[F], cache: File)(f: ChangeReport[File] => Task[T]): Task[T] = + filesTask bind { files => + val lastFilesInfo = Operations.fromFile(cache)(style.format).files + val lastFiles = lastFilesInfo.map(_.file) + val currentFiles = files.map(_.getAbsoluteFile) + val currentFilesInfo = style(files) + + val report = new ChangeReport[File] + { + lazy val allInputs = currentFiles + lazy val removed = lastFiles -- allInputs + lazy val added = allInputs -- lastFiles + lazy val modified = (lastFilesInfo -- currentFilesInfo.files).map(_.file) + lazy val unmodified = allInputs -- modified + } + + f(report) map { result => + Operations.toFile(currentFilesInfo)(cache)(style.format) + result + } + } + def invalidate[R](changes: ChangeReport[File], cacheDirectory: File)(f: (InvalidationReport[File], UpdateTracking[File]) => Task[R]): Task[R] = + { + val pruneAndF = (report: InvalidationReport[File], tracking: UpdateTracking[File]) => { + report.invalidProducts.foreach(_.delete) + f(report, tracking) + } + invalidate(Task(changes), cacheDirectory, true)(pruneAndF)(sbinary.DefaultProtocol.FileFormat) + } + def invalidate[T,R](changesTask: Task[ChangeReport[T]], cacheDirectory: File, translateProducts: Boolean) + (f: (InvalidationReport[T], UpdateTracking[T]) => Task[R])(implicit format: Format[T]): Task[R] = + { + changesTask bind { changes => + val trackFormat = new TrackingFormat[T](cacheDirectory, translateProducts) + val tracker = trackFormat.read + def invalidatedBy(file: T) = tracker.products(file) ++ tracker.sources(file) ++ tracker.usedBy(file) ++ tracker.dependsOn(file) + + import scala.collection.mutable.HashSet + val invalidated = new HashSet[T] + val invalidatedProducts = new HashSet[T] + def invalidate(files: Iterable[T]): Unit = + for(file <- files if !invalidated(file)) + { + invalidated += file + if(!tracker.sources(file).isEmpty) invalidatedProducts += file + invalidate(invalidatedBy(file)) + } + + invalidate(changes.modified) + tracker.removeAll(invalidated) + + val report = new InvalidationReport[T] + { + val invalid = Set(invalidated.toSeq : _*) + val invalidProducts = Set(invalidatedProducts.toSeq : _*) + val valid = changes.unmodified -- invalid + } + + f(report, tracker) map { result => + trackFormat.write(tracker) + result + } + } + } + + import scala.collection.mutable.{Set, HashMap, MultiMap} + private[xsbt] type DependencyMap[T] = HashMap[T, Set[T]] with MultiMap[T, T] + private[xsbt] def newMap[T]: DependencyMap[T] = new HashMap[T, Set[T]] with MultiMap[T, T] +} + +trait UpdateTracking[T] extends NotNull +{ + def dependency(source: T, dependsOn: T): Unit + def use(source: T, uses: T): Unit + def product(source: T, output: T): Unit +} +import scala.collection.Set +trait ReadTracking[T] extends NotNull +{ + def dependsOn(file: T): Set[T] + def products(file: T): Set[T] + def sources(file: T): Set[T] + def usedBy(file: T): Set[T] +} +import DependencyTracking.{DependencyMap => DMap, newMap} +private final class DefaultTracking[T](translateProducts: Boolean)(val reverseDependencies: DMap[T], val reverseUses: DMap[T], val sourceMap: DMap[T]) extends DependencyTracking[T](translateProducts) +{ + val productMap: DMap[T] = forward(sourceMap) // map from a source to its products. Keep in sync with sourceMap +} +// if translateProducts is true, dependencies on a product are translated to dependencies on a source +private abstract class DependencyTracking[T](translateProducts: Boolean) extends ReadTracking[T] with UpdateTracking[T] +{ + val reverseDependencies: DMap[T] // map from a file to the files that depend on it + val reverseUses: DMap[T] // map from a file to the files that use it + val sourceMap: DMap[T] // map from a product to its sources. Keep in sync with productMap + val productMap: DMap[T] // map from a source to its products. Keep in sync with sourceMap + + final def dependsOn(file: T): Set[T] = get(reverseDependencies, file) + final def products(file: T): Set[T] = get(productMap, file) + final def sources(file: T): Set[T] = get(sourceMap, file) + final def usedBy(file: T): Set[T] = get(reverseUses, file) + + private def get(map: DMap[T], value: T): Set[T] = map.getOrElse(value, Set.empty[T]) + + final def dependency(sourceFile: T, dependsOn: T) + { + val actualDependencies = + if(!translateProducts) + Seq(dependsOn) + else + sourceMap.getOrElse(dependsOn, Seq(dependsOn)) + actualDependencies.foreach { actualDependency => reverseDependencies.add(actualDependency, sourceFile) } + } + final def product(sourceFile: T, product: T) + { + productMap.add(sourceFile, product) + sourceMap.add(product, sourceFile) + } + final def use(sourceFile: T, usesFile: T) { reverseUses.add(usesFile, sourceFile) } + + final def removeAll(files: Iterable[T]) + { + def remove(a: DMap[T], b: DMap[T], file: T): Unit = + for(x <- a.removeKey(file)) b --= x + def removeAll(a: DMap[T], b: DMap[T]): Unit = + files.foreach { file => remove(a, b, file); remove(b, a, file) } + + removeAll(forward(reverseDependencies), reverseDependencies) + removeAll(productMap, sourceMap) + removeAll(forward(reverseUses), reverseUses) + } + protected final def forward(map: DMap[T]): DMap[T] = + { + val f = newMap[T] + for( (key, values) <- map; value <- values) f.add(value, key) + f + } +} \ No newline at end of file diff --git a/cache/TrackingFormat.scala b/cache/TrackingFormat.scala new file mode 100644 index 000000000..1b6c463a0 --- /dev/null +++ b/cache/TrackingFormat.scala @@ -0,0 +1,52 @@ +package xsbt + +import java.io.File +import scala.collection.mutable.{HashMap, Map, MultiMap, Set} +import sbinary.{DefaultProtocol, Format, Operations} +import DefaultProtocol._ +import TrackingFormat._ +import DependencyTracking.{DependencyMap => DMap, newMap} + +private class TrackingFormat[T](directory: File, translateProducts: Boolean)(implicit tFormat: Format[T]) extends NotNull +{ + + val indexFile = new File(directory, "index") + val dependencyFile = new File(directory, "dependencies") + def read(): DependencyTracking[T] = + { + val indexMap = Operations.fromFile[Map[Int,T]](indexFile) + val indexedFormat = wrap[T,Int](ignore => error("Read-only"), indexMap.apply) + Operations.fromFile(dependencyFile)(trackingFormat(translateProducts)(indexedFormat)) + } + def write(tracking: DependencyTracking[T]) + { + val index = new IndexMap[T] + val indexedFormat = wrap[T,Int](t => index(t), ignore => error("Write-only")) + + Operations.toFile(tracking)(dependencyFile)(trackingFormat(translateProducts)(indexedFormat)) + Operations.toFile(index.indices)(indexFile) + } +} +private object TrackingFormat +{ + implicit def mutableMapFormat[S, T](implicit binS : Format[S], binT : Format[T]) : Format[Map[S, T]] = + viaArray( (x : Array[(S, T)]) => Map(x :_*)); + implicit def depMapFormat[T](implicit bin: Format[T]) : Format[DMap[T]] = + { + viaArray { (x : Array[(T, Set[T])]) => + val map = newMap[T] + map ++= x + map + } + } + def trackingFormat[T](translateProducts: Boolean)(implicit tFormat: Format[T]): Format[DependencyTracking[T]] = + asProduct3((a: DMap[T],b: DMap[T],c: DMap[T]) => new DefaultTracking(translateProducts)(a,b,c) : DependencyTracking[T])(dt => Some(dt.reverseDependencies, dt.reverseUses, dt.sourceMap)) +} + +private final class IndexMap[T] extends NotNull +{ + private[this] var lastIndex = 0 + private[this] val map = new HashMap[T, Int] + def indices = map.toArray.map( (_: (T,Int)).swap ) + def apply(t: T) = map.getOrElseUpdate(t, { lastIndex += 1; lastIndex }) +} \ No newline at end of file diff --git a/cache/src/test/scala/Tracking.scala b/cache/src/test/scala/Tracking.scala new file mode 100644 index 000000000..ff952556b --- /dev/null +++ b/cache/src/test/scala/Tracking.scala @@ -0,0 +1,45 @@ +package xsbt + +import java.io.File + +trait examples +{ + def classpathTask: Task[Set[File]] + def sourcesTask: Task[Set[File]] + import DependencyTracking._ + lazy val compile = + changed(classpathTask, FilesInfo.lastModified, new File("cache/compile/classpath/")) { classpathChanges => + changed(sourcesTask, FilesInfo.hash, new File("cache/compile/sources/")) { sourceChanges => + invalidate(classpathChanges +++ sourceChanges, new File("cache/compile/dependencies/'")) { (report, tracking) => + val recompileSources = report.invalid ** sourceChanges.allInputs + val classpath = classpathChanges.allInputs + Task() + } + } + } + + trait sync + { + def sources: Task[Set[File]] = Task(Set.empty[File]) + def mapper: Task[FileMapper] = outputDirectory map(FileMapper.basic) + def outputDirectory: Task[File] = Task(new File("test")) + + import Task._ + lazy val task = syncTask + def syncTask = + (sources, mapper) bind { (srcs,mp) => + DependencyTracking.trackBasic(sources, FilesInfo.hash, new File("cache/sync/")) { (sourceChanges, report, tracking) => + Task + { + for(src <- report.invalid ** sourceChanges.allInputs) yield + { + val target = mp(src) + FileUtilities.copyFile(src, target) + tracking.product(src, target) + target + } + } + } + } + } +} \ No newline at end of file diff --git a/project/build/XSbt.scala b/project/build/XSbt.scala index 212e721d0..7e387e868 100644 --- a/project/build/XSbt.scala +++ b/project/build/XSbt.scala @@ -2,6 +2,8 @@ import sbt._ class XSbt(info: ProjectInfo) extends ParentProject(info) { + /* Subproject declarations*/ + val launchInterfaceSub = project(launchPath / "interface", "Launcher Interface", new InterfaceProject(_)) val launchSub = project(launchPath, "Launcher", new LaunchProject(_), launchInterfaceSub) @@ -22,10 +24,16 @@ class XSbt(info: ProjectInfo) extends ParentProject(info) val compilerSub = project(compilePath, "Compile", new CompileProject(_), launchInterfaceSub, interfaceSub, ivySub, ioSub, classpathSub, compileInterfaceSub) + /* Multi-subproject paths */ + def launchPath = path("launch") def utilPath = path("util") def compilePath = path("compile") + //run in parallel + override def parallelExecution = true + + /* Subproject configurations*/ class LaunchProject(info: ProjectInfo) extends Base(info) with TestWithIO with TestDependencies { val ivy = "org.apache.ivy" % "ivy" % "2.0.0" @@ -45,11 +53,11 @@ class XSbt(info: ProjectInfo) extends ParentProject(info) val ju = "junit" % "junit" % "4.5" % "test->default" // required by specs to compile properly } - override def parallelExecution = true class IOProject(info: ProjectInfo) extends Base(info) with TestDependencies class TaskProject(info: ProjectInfo) extends Base(info) with TestDependencies class CacheProject(info: ProjectInfo) extends Base(info) { + // these compilation options are useful for debugging caches and task composition //override def compileOptions = super.compileOptions ++ List(Unchecked,ExplainTypes, CompileOption("-Xlog-implicits")) } class Base(info: ProjectInfo) extends DefaultProject(info) with ManagedBase @@ -59,9 +67,10 @@ class XSbt(info: ProjectInfo) extends ParentProject(info) class CompileProject(info: ProjectInfo) extends Base(info) { override def testCompileAction = super.testCompileAction dependsOn(launchSub.testCompile, compileInterfaceSub.`package`, interfaceSub.`package`) + // don't include launch interface in published dependencies because it will be provided by launcher override def deliverProjectDependencies = Set(super.deliverProjectDependencies.toSeq : _*) - launchInterfaceSub.projectID override def testClasspath = super.testClasspath +++ launchSub.testClasspath +++ compileInterfaceSub.jarPath +++ interfaceSub.jarPath - override def compileOptions = super.compileOptions ++ Seq(CompileOption("-Xno-varargs-conversion")) + override def compileOptions = super.compileOptions ++ Seq(CompileOption("-Xno-varargs-conversion")) //needed for invoking nsc.scala.tools.Main.process(Array[String]) } class IvyProject(info: ProjectInfo) extends Base(info) with TestWithIO { @@ -69,6 +78,7 @@ class XSbt(info: ProjectInfo) extends ParentProject(info) } class InterfaceProject(info: ProjectInfo) extends DefaultProject(info) with ManagedBase { + // ensure that interfaces are only Java sources and that they cannot reference Scala classes override def mainSources = descendents(mainSourceRoots, "*.java") override def compileOrder = CompileOrder.JavaThenScala } @@ -90,14 +100,15 @@ class XSbt(info: ProjectInfo) extends ParentProject(info) override def testClasspath = super.testClasspath +++ ioSub.testClasspath } } + trait SourceProject extends BasicScalaProject { - override final def crossScalaVersions = Set.empty - override def packagePaths = mainResources +++ mainSources + override final def crossScalaVersions = Set.empty // don't need to cross-build a source package + override def packagePaths = mainResources +++ mainSources // the default artifact is a jar of the main sources and resources } trait ManagedBase extends BasicScalaProject { - override def deliverScalaDependencies = Nil + override def deliverScalaDependencies = Nil // override def crossScalaVersions = Set("2.7.5") override def managedStyle = ManagedStyle.Ivy override def useDefaultConfigurations = false diff --git a/util/io/PathMapper.scala b/util/io/PathMapper.scala index 2f8afdc09..eea4c37c1 100644 --- a/util/io/PathMapper.scala +++ b/util/io/PathMapper.scala @@ -9,15 +9,31 @@ trait PathMapper extends NotNull { def apply(file: File): String } - -object PathMapper -{ - val basic = new FMapper(_.getPath) - def relativeTo(base: File) = new FMapper(file => FileUtilities.relativize(base, file).getOrElse(file.getPath)) - val flat = new FMapper(_.getName) - def apply(f: File => String) = new FMapper(f) -} -class FMapper(f: File => String) extends PathMapper +class PMapper(f: File => String) extends PathMapper { def apply(file: File) = f(file) +} +object PathMapper +{ + val basic = new PMapper(_.getPath) + def relativeTo(base: File) = new PMapper(file => FileUtilities.relativize(base, file).getOrElse(file.getPath)) + val flat = new PMapper(_.getName) + def apply(f: File => String) = new PMapper(f) +} + +trait FileMapper extends NotNull +{ + def apply(file: File): File +} +class FMapper(f: File => File) extends FileMapper +{ + def apply(file: File) = f(file) +} +object FileMapper +{ + def basic(newDirectory: File) = new FMapper(file => new File(newDirectory, file.getPath)) + def rebase(oldBase: File, newBase: File) = + new FMapper(file => new File(newBase, FileUtilities.relativize(oldBase, file).getOrElse(error(file + " not a descendent of " + oldBase)))) + def flat(newDirectory: File) = new FMapper(file => new File(newDirectory, file.getName)) + def apply(f: File => File) = new FMapper(f) } \ No newline at end of file