From 0d5814e2b36ebe551e9cdc5ed84df2af14b92c94 Mon Sep 17 00:00:00 2001 From: Mark Harrah Date: Fri, 17 Sep 2010 21:38:40 -0400 Subject: [PATCH] Rework external dependency tracking and multi-projects Reduce AnalysisCallback interface: remove discovery simplify dependency notification methods Use map of classpath entry to Analysis for locating source API for external dependencies Handle classpath changes by locating class on classpath and either locating Analysis/Source as above or comparing Stamp. This requires storing the class name of a binary dependency now. Make this process aware of full classpath, including boot classpath --- compile/inc/Analysis.scala | 6 +- compile/inc/Compile.scala | 76 ++++++---- compile/inc/Incremental.scala | 38 ++++- compile/inc/Locate.scala | 9 ++ compile/inc/Stamp.scala | 26 ++-- compile/interface/Analyzer.scala | 27 +--- compile/persist/AnalysisFormats.scala | 4 +- .../src/main/java/xsbti/AnalysisCallback.java | 28 +--- main/ClasspathProject.scala | 132 ++++++++++++++---- main/Compile.scala | 10 +- main/build/AggressiveCompile.scala | 13 +- util/classfile/Analyze.scala | 7 +- 12 files changed, 242 insertions(+), 134 deletions(-) diff --git a/compile/inc/Analysis.scala b/compile/inc/Analysis.scala index d53d4ccdc..795358888 100644 --- a/compile/inc/Analysis.scala +++ b/compile/inc/Analysis.scala @@ -18,7 +18,7 @@ trait Analysis def copy(stamps: Stamps = stamps, apis: APIs = apis, relations: Relations = relations): Analysis def addSource(src: File, api: Source, stamp: Stamp, internalDeps: Iterable[File]): Analysis - def addBinaryDep(src: File, dep: File, stamp: Stamp): Analysis + def addBinaryDep(src: File, dep: File, className: String, stamp: Stamp): Analysis def addExternalDep(src: File, dep: String, api: Source): Analysis def addProduct(src: File, product: File, stamp: Stamp): Analysis } @@ -44,8 +44,8 @@ private class MAnalysis(val stamps: Stamps, val apis: APIs, val relations: Relat def addSource(src: File, api: Source, stamp: Stamp, internalDeps: Iterable[File]): Analysis = copy( stamps.markInternalSource(src, stamp), apis.markInternalSource(src, api), relations.addInternalSrcDeps(src, internalDeps) ) - def addBinaryDep(src: File, dep: File, stamp: Stamp): Analysis = - copy( stamps.markBinary(dep, stamp), apis, relations.addBinaryDep(src, dep) ) + def addBinaryDep(src: File, dep: File, className: String, stamp: Stamp): Analysis = + copy( stamps.markBinary(dep, className, stamp), apis, relations.addBinaryDep(src, dep) ) def addExternalDep(src: File, dep: String, depAPI: Source): Analysis = copy( stamps, apis.markExternalAPI(dep, depAPI), relations.addExternalDep(src, dep) ) diff --git a/compile/inc/Compile.scala b/compile/inc/Compile.scala index a2e626a8c..42f3239e9 100644 --- a/compile/inc/Compile.scala +++ b/compile/inc/Compile.scala @@ -9,42 +9,74 @@ import java.io.File object IncrementalCompile { - def apply(sources: Set[File], compile: (Set[File], xsbti.AnalysisCallback) => Unit, previous: Analysis, externalAPI: String => Source): Analysis = + def apply(sources: Set[File], entry: String => Option[File], compile: (Set[File], xsbti.AnalysisCallback) => Unit, previous: Analysis, forEntry: File => Option[Analysis], outputPath: File): Analysis = { val current = Stamps.initial(Stamp.exists, Stamp.hash, Stamp.lastModified) val internalMap = (f: File) => previous.relations.produced(f).headOption - Incremental.compile(sources, previous, current, externalAPI, doCompile(compile, internalMap, current)) + val externalAPI = getExternalAPI(entry, forEntry) + Incremental.compile(sources, entry, previous, current, forEntry, doCompile(compile, internalMap, externalAPI, current, outputPath)) } - def doCompile(compile: (Set[File], xsbti.AnalysisCallback) => Unit, internalMap: File => Option[File], current: ReadStamps) = (srcs: Set[File]) => { - val callback = new AnalysisCallback(internalMap, current) + def doCompile(compile: (Set[File], xsbti.AnalysisCallback) => Unit, internalMap: File => Option[File], externalAPI: (File, String) => Option[Source], current: ReadStamps, outputPath: File) = (srcs: Set[File]) => { + val callback = new AnalysisCallback(internalMap, externalAPI, current, outputPath) compile(srcs, callback) callback.get } + def getExternalAPI(entry: String => Option[File], forEntry: File => Option[Analysis]): (File, String) => Option[Source] = + (file: File,className: String) => + entry(className) flatMap { defines => + if(file != Locate.resolve(defines, className) ) + None + else + forEntry(defines) flatMap { analysis => + analysis.relations.produced(file).headOption flatMap { src => + analysis.apis.internal get src + } + } + } } -private final class AnalysisCallback(internalMap: File => Option[File], current: ReadStamps) extends xsbti.AnalysisCallback +private final class AnalysisCallback(internalMap: File => Option[File], externalAPI: (File, String) => Option[Source], current: ReadStamps, outputPath: File) extends xsbti.AnalysisCallback { override def toString = ( List("APIs", "Binary deps", "Products", "Source deps") zip List(apis, binaryDeps, classes, sourceDeps)).map { case (label, map) => label + "\n\t" + map.mkString("\n\t") }.mkString("\n") - import collection.mutable.{HashMap, HashSet, Map, Set} + import collection.mutable.{HashMap, HashSet, ListBuffer, Map, Set} private val apis = new HashMap[File, Source] private val binaryDeps = new HashMap[File, Set[File]] private val classes = new HashMap[File, Set[File]] private val sourceDeps = new HashMap[File, Set[File]] + private val extSrcDeps = new ListBuffer[(File, String, Source)] + private val binaryClassName = new HashMap[File, String] private def add[A,B](map: Map[A,Set[B]], a: A, b: B): Unit = map.getOrElseUpdate(a, new HashSet[B]) += b - def sourceDependency(dependsOn: File, source: File) = add(sourceDeps, source, dependsOn) - - def jarDependency(jar: File, source: File) = add(binaryDeps, source, jar) - def classDependency(clazz: File, source: File) = add(binaryDeps, source, clazz) - - def productDependency(classFile: File, sourcePath: File) = - internalMap(classFile) match { - case Some(dependsOn) => sourceDependency(dependsOn, sourcePath) - case None => classDependency(classFile, sourcePath) + def sourceDependency(dependsOn: File, source: File) = if(source != dependsOn) add(sourceDeps, source, dependsOn) + def externalBinaryDependency(binary: File, className: String, source: File) + { + binaryClassName.put(binary, className) + add(binaryDeps, source, binary) + } + def externalSourceDependency(triple: (File, String, Source)) = extSrcDeps += triple + + def binaryDependency(classFile: File, name: String, source: File) = + { + internalMap(classFile) match + { + case Some(dependsOn) => + // dependency is a product of a source not included in this compilation + sourceDependency(dependsOn, source) + case None => + externalAPI(classFile, name) match + { + case Some(api) => + // dependency is a product of a source in another project + externalSourceDependency( (source, name, api) ) + case None => + // dependency is some other binary on the classpath + externalBinaryDependency(classFile, name, source) + } } + } def generatedClass(source: File, module: File) = add(classes, source, module) @@ -52,26 +84,20 @@ private final class AnalysisCallback(internalMap: File => Option[File], current: def endSource(sourcePath: File): Unit = assert(apis.contains(sourcePath)) - def get: Analysis = addBinaries( addProducts( addSources(Analysis.Empty) ) ) + def get: Analysis = addExternals( addBinaries( addProducts( addSources(Analysis.Empty) ) ) ) def addProducts(base: Analysis): Analysis = addAll(base, classes)( (a, src, prod) => a.addProduct(src, prod, current product prod ) ) - def addBinaries(base: Analysis): Analysis = addAll(base, binaryDeps)( (a, src, bin) => a.addBinaryDep(src, bin, current binary bin) ) + def addBinaries(base: Analysis): Analysis = addAll(base, binaryDeps)( (a, src, bin) => a.addBinaryDep(src, bin, binaryClassName(bin), current binary bin) ) def addSources(base: Analysis): Analysis = (base /: apis) { case (a, (src, api) ) => a.addSource(src, api, current.internalSource(src), sourceDeps.getOrElse(src, Nil: Iterable[File])) } + def addExternals(base: Analysis): Analysis = (base /: extSrcDeps) { case (a, (source, name, api)) => a.addExternalDep(source, name, api) } def addAll[A,B](base: Analysis, m: Map[A, Set[B]])( f: (Analysis, A, B) => Analysis): Analysis = (base /: m) { case (outer, (a, bs)) => (outer /: bs) { (inner, b) => f(inner, a, b) } } - - private def emptyS = new Array[String](0) - def superclassNames = emptyS - def annotationNames = emptyS - def superclassNotFound(superclassName: String) {} - def foundSubclass(source: File, subclassName: String, superclassName: String, isModule: Boolean) {} - def foundAnnotated(source: File, className: String, annotationName: String, isModule: Boolean) {} - def foundApplication(source: File, className: String) {} + def beginSource(source: File) {} } \ No newline at end of file diff --git a/compile/inc/Incremental.scala b/compile/inc/Incremental.scala index 5e3a7ecb4..e84deddea 100644 --- a/compile/inc/Incremental.scala +++ b/compile/inc/Incremental.scala @@ -12,9 +12,9 @@ import java.io.File object Incremental { def println(s: String) = if(java.lang.Boolean.getBoolean("xsbt.inc.debug")) System.out.println(s) else () - def compile(sources: Set[File], previous: Analysis, current: ReadStamps, externalAPI: String => Source, doCompile: Set[File] => Analysis)(implicit equivS: Equiv[Stamp]): Analysis = + def compile(sources: Set[File], entry: String => Option[File], previous: Analysis, current: ReadStamps, forEntry: File => Option[Analysis], doCompile: Set[File] => Analysis)(implicit equivS: Equiv[Stamp]): Analysis = { - val initialChanges = changedInitial(sources, previous.stamps, previous.apis, current, externalAPI) + val initialChanges = changedInitial(entry, sources, previous, current, forEntry) val initialInv = invalidateInitial(previous.relations, initialChanges) println("Initially invalidated: " + initialInv) cycle(initialInv, previous, doCompile) @@ -60,12 +60,15 @@ object Incremental new APIChanges(modifiedAPIs, changedNames) } - def changedInitial(sources: Set[File], previous: Stamps, previousAPIs: APIs, current: ReadStamps, externalAPI: String => Source)(implicit equivS: Equiv[Stamp]): InitialChanges = + def changedInitial(entry: String => Option[File], sources: Set[File], previousAnalysis: Analysis, current: ReadStamps, forEntry: File => Option[Analysis])(implicit equivS: Equiv[Stamp]): InitialChanges = { + val previous = previousAnalysis.stamps + val previousAPIs = previousAnalysis.apis + val srcChanges = changes(previous.allInternalSources.toSet, sources, f => !equivS.equiv( previous.internalSource(f), current.internalSource(f) ) ) val removedProducts = previous.allProducts.filter( p => !equivS.equiv( previous.product(p), current.product(p) ) ).toSet - val binaryDepChanges = previous.allBinaries.filter( f => !equivS.equiv( previous.binary(f), current.binary(f) ) ).toSet - val extChanges = changedIncremental(previousAPIs.allExternals, previousAPIs.externalAPI, externalAPI) + val binaryDepChanges = previous.allBinaries.filter( externalBinaryModified(entry, previous.className _, previous, current)).toSet + val extChanges = changedIncremental(previousAPIs.allExternals, previousAPIs.externalAPI, currentExternalAPI(entry, forEntry)) InitialChanges(srcChanges, removedProducts, binaryDepChanges, extChanges ) } @@ -121,6 +124,31 @@ object Incremental previous -- invalidatedSrcs } + def externalBinaryModified(entry: String => Option[File], className: File => Option[String], previous: Stamps, current: ReadStamps)(implicit equivS: Equiv[Stamp]): File => Boolean = + dependsOn => + orTrue( + for { + name <- className(dependsOn) + e <- entry(name) + } yield { + val resolved = Locate.resolve(e, name) + (resolved != dependsOn) || !equivS.equiv(previous.binary(dependsOn), current.binary(resolved)) + } + ) + + def currentExternalAPI(entry: String => Option[File], forEntry: File => Option[Analysis]): String => Source = + className => + orEmpty( + for { + e <- entry(className) + analysis <- forEntry(e) + src <- analysis.relations.produced(Locate.resolve(e, className)).headOption + } yield + analysis.apis.internalAPI(src) + ) + + def orEmpty(o: Option[Source]): Source = o getOrElse APIs.emptyAPI + def orTrue(o: Option[Boolean]): Boolean = o getOrElse true // unmodifiedSources should not contain any sources in the previous compilation run // (this may unnecessarily invalidate them otherwise) /*def scopeInvalidation(previous: Analysis, otherSources: Set[File], names: NameChanges): Set[File] = diff --git a/compile/inc/Locate.scala b/compile/inc/Locate.scala index 4a12425af..5a2396086 100644 --- a/compile/inc/Locate.scala +++ b/compile/inc/Locate.scala @@ -29,6 +29,15 @@ object Locate case x => x } + /** Returns a function that searches the provided class path for + * a class name and returns the entry that defines that class.*/ + def entry(classpath: Seq[File]): String => Option[File] = + { + val entries = classpath.toStream.map { entry => (entry, definesClass(entry)) } + className => entries collect { case (entry, defines) if defines(className) => entry } headOption; + } + def resolve(f: File, className: String): File = if(f.isDirectory) classFile(f, className) else f + def getValue[S](get: File => String => Option[S])(entry: File): String => Either[Boolean, S] = { val defClass = definesClass(entry) diff --git a/compile/inc/Stamp.scala b/compile/inc/Stamp.scala index 4b1c02b0a..b078c2318 100644 --- a/compile/inc/Stamp.scala +++ b/compile/inc/Stamp.scala @@ -27,9 +27,12 @@ trait Stamps extends ReadStamps def sources: Map[File, Stamp] def binaries: Map[File, Stamp] def products: Map[File, Stamp] + def classNames: Map[File, String] + + def className(bin: File): Option[String] def markInternalSource(src: File, s: Stamp): Stamps - def markBinary(bin: File, s: Stamp): Stamps + def markBinary(bin: File, className: String, s: Stamp): Stamps def markProduct(prod: File, s: Stamp): Stamps def filter(prod: File => Boolean, removeSources: Iterable[File], bin: File => Boolean): Stamps @@ -77,36 +80,37 @@ object Stamps def empty: Stamps = { val eSt = Map.empty[File, Stamp] - apply(eSt, eSt, eSt) + apply(eSt, eSt, eSt, Map.empty[File, String]) } - def apply(products: Map[File, Stamp], sources: Map[File, Stamp], binaries: Map[File, Stamp]): Stamps = - new MStamps(products, sources, binaries) + def apply(products: Map[File, Stamp], sources: Map[File, Stamp], binaries: Map[File, Stamp], binaryClassNames: Map[File, String]): Stamps = + new MStamps(products, sources, binaries, binaryClassNames) } -private class MStamps(val products: Map[File, Stamp], val sources: Map[File, Stamp], val binaries: Map[File, Stamp]) extends Stamps +private class MStamps(val products: Map[File, Stamp], val sources: Map[File, Stamp], val binaries: Map[File, Stamp], val classNames: Map[File, String]) extends Stamps { def allInternalSources: collection.Set[File] = sources.keySet def allBinaries: collection.Set[File] = binaries.keySet def allProducts: collection.Set[File] = products.keySet def ++ (o: Stamps): Stamps = - new MStamps(products ++ o.products, sources ++ o.sources, binaries ++ o.binaries) + new MStamps(products ++ o.products, sources ++ o.sources, binaries ++ o.binaries, classNames ++ o.classNames) def markInternalSource(src: File, s: Stamp): Stamps = - new MStamps(products, sources.updated(src, s), binaries) + new MStamps(products, sources.updated(src, s), binaries, classNames) - def markBinary(bin: File, s: Stamp): Stamps = - new MStamps(products, sources, binaries.updated(bin, s)) + def markBinary(bin: File, className: String, s: Stamp): Stamps = + new MStamps(products, sources, binaries.updated(bin, s), classNames.updated(bin, className)) def markProduct(prod: File, s: Stamp): Stamps = - new MStamps(products.updated(prod, s), sources, binaries) + new MStamps(products.updated(prod, s), sources, binaries, classNames) def filter(prod: File => Boolean, removeSources: Iterable[File], bin: File => Boolean): Stamps = - new MStamps(products.filterKeys(prod), sources -- removeSources, binaries.filterKeys(bin)) + new MStamps(products.filterKeys(prod), sources -- removeSources, binaries.filterKeys(bin), classNames.filterKeys(bin)) def product(prod: File) = getStamp(products, prod) def internalSource(src: File) = getStamp(sources, src) def binary(bin: File) = getStamp(binaries, bin) + def className(bin: File) = classNames get bin } diff --git a/compile/interface/Analyzer.scala b/compile/interface/Analyzer.scala index 809881b92..048a7c75d 100644 --- a/compile/interface/Analyzer.scala +++ b/compile/interface/Analyzer.scala @@ -37,20 +37,19 @@ final class Analyzer(val global: Global, val callback: AnalysisCallback) extends callback.beginSource(sourceFile) for(on <- unit.depends) { + def binaryDependency(file: File, className: String) = callback.binaryDependency(file, className, sourceFile) val onSource = on.sourceFile if(onSource == null) { classFile(on) match { - case Some(f) => - { + case Some((f,className)) => f match { - case ze: ZipArchive#Entry => callback.jarDependency(new File(archive(ze).getName), sourceFile) - case pf: PlainFile => callback.classDependency(pf.file, sourceFile) + case ze: ZipArchive#Entry => binaryDependency(new File(archive(ze).getName), className) + case pf: PlainFile => binaryDependency(pf.file, className) case _ => () } - } case None => () } } @@ -82,25 +81,11 @@ final class Analyzer(val global: Global, val callback: AnalysisCallback) extends } } - private def classForName(name: String) = - { - try - { - if(name.indexOf('.') < 0) - { - val sym = definitions.EmptyPackageClass.info.member(newTypeName(name)) - if(sym != NoSymbol) Some( sym ) else { callback.superclassNotFound(name); None } - } - else - Some( global.definitions.getClass(newTermName(name)) ) - } - catch { case fe: scala.tools.nsc.FatalError => callback.superclassNotFound(name); None } - } - private def classFile(sym: Symbol): Option[AbstractFile] = + private def classFile(sym: Symbol): Option[(AbstractFile, String)] = { import scala.tools.nsc.symtab.Flags val name = flatname(sym, finder.classSeparator) + moduleSuffix(sym) - finder.findClass(name) orElse { + finder.findClass(name).map(file => (file, name)) orElse { if(isTopLevelModule(sym)) { val linked = linkedClass(sym) diff --git a/compile/persist/AnalysisFormats.scala b/compile/persist/AnalysisFormats.scala index 13ac559ec..57cd5521a 100644 --- a/compile/persist/AnalysisFormats.scala +++ b/compile/persist/AnalysisFormats.scala @@ -21,8 +21,8 @@ object AnalysisFormats implicit def setupFormat(implicit outDirF: Format[File], optionF: Format[CompileOptions], compilerVersion: Format[String], orderF: Format[CompileOrder.Value]): Format[CompileSetup] = asProduct4[CompileSetup, File, CompileOptions, String, CompileOrder.Value]( (a,b,c,d) => new CompileSetup(a,b,c,d) )(s => (s.outputDirectory, s.options, s.compilerVersion, s.order))(outDirF, optionF, compilerVersion, orderF) - implicit def stampsFormat(implicit prodF: Format[Map[File, Stamp]], srcF: Format[Map[File, Stamp]], binF: Format[Map[File, Stamp]]): Format[Stamps] = - asProduct3( Stamps.apply _ )( s => (s.products, s.sources, s.binaries) )(prodF, srcF, binF) + implicit def stampsFormat(implicit prodF: Format[Map[File, Stamp]], srcF: Format[Map[File, Stamp]], binF: Format[Map[File, Stamp]], nameF: Format[Map[File, String]]): Format[Stamps] = + asProduct4( Stamps.apply _ )( s => (s.products, s.sources, s.binaries, s.classNames) )(prodF, srcF, binF, nameF) implicit def stampFormat(implicit hashF: Format[Hash], modF: Format[LastModified], existsF: Format[Exists]): Format[Stamp] = asUnion(hashF, modF, existsF) diff --git a/interface/src/main/java/xsbti/AnalysisCallback.java b/interface/src/main/java/xsbti/AnalysisCallback.java index 03c4798c9..d3eb2ab54 100644 --- a/interface/src/main/java/xsbti/AnalysisCallback.java +++ b/interface/src/main/java/xsbti/AnalysisCallback.java @@ -7,43 +7,21 @@ import java.io.File; public interface AnalysisCallback { - /** The names of classes that the analyzer should find subclasses of.*/ - public String[] superclassNames(); - /** The names of annotations that the analyzer should look for on methods and classes.*/ - public String[] annotationNames(); - /** Called when the the given superclass could not be found on the classpath by the compiler.*/ - public void superclassNotFound(String superclassName); /** Called before the source at the given location is processed. */ public void beginSource(File source); - /** Called when the a subclass of one of the classes given in superclassNames is - * discovered.*/ - public void foundSubclass(File source, String subclassName, String superclassName, boolean isModule); - /** Called when an annotation with name annotationName is found on a class or one of its methods.*/ - public void foundAnnotated(File source, String className, String annotationName, boolean isModule); /** Called to indicate that the source file source depends on the source file * dependsOn. Note that only source files included in the current compilation will * passed to this method. Dependencies on classes generated by sources not in the current compilation will * be passed as class dependencies to the classDependency method.*/ public void sourceDependency(File dependsOn, File source); - /** Called to indicate that the source file source depends on the jar - * jar.*/ - public void jarDependency(File jar, File source); - /** Called to indicate that the source file source depends on the class file - * clazz.*/ - public void classDependency(File clazz, File source); - /** Called to indicate that the source file sourcePath depends on the class file - * classFile that is a product of some source. This differs from classDependency - * because it is really a sourceDependency. The source corresponding to classFile - * was not incuded in the compilation so the plugin doesn't know what the source is though. It - * only knows that the class file came from the output directory.*/ - public void productDependency(File classFile, File sourcePath); + /** Called to indicate that the source file source depends on the top-level + * class named name from class or jar file binary. */ + public void binaryDependency(File binary, String name, File source); /** Called to indicate that the source file source produces a class file at * module.*/ public void generatedClass(File source, File module); /** Called after the source at the given location has been processed. */ public void endSource(File sourcePath); - /** Called when a module with a public 'main' method with the right signature is found.*/ - public void foundApplication(File source, String className); /** Called when the public API of a source file is extracted. */ public void api(File sourceFile, xsbti.api.Source source); } \ No newline at end of file diff --git a/main/ClasspathProject.scala b/main/ClasspathProject.scala index 8d8ac08e4..ae8e90185 100644 --- a/main/ClasspathProject.scala +++ b/main/ClasspathProject.scala @@ -1,24 +1,44 @@ +/* sbt -- Simple Build Tool + * Copyright 2010 Mark Harrah + */ package sbt import std._ + import inc.Analysis import TaskExtra._ import ClasspathProject._ import java.io.File import Path._ + import Types._ import scala.xml.NodeSeq trait ClasspathProject { def configurations: Seq[Configuration] - def products(configuration: Configuration, intermediate: Boolean): Task[Seq[File]] - def unmanagedClasspath(configuration: Configuration): Task[Seq[File]] - def managedClasspath(configuration: Configuration): Task[Seq[File]] + val products: Classpath + val unmanagedClasspath: Classpath + val managedClasspath: Classpath + val internalDependencyClasspath: Classpath - def dependencyClasspath(configuration: Configuration): Task[Seq[File]] = - (unmanagedClasspath(configuration), managedClasspath(configuration)) map concat[File] + lazy val externalDependencyClasspath: Classpath = + TaskMap { (configuration: Configuration) => + val un = unmanagedClasspath(configuration) + val m = managedClasspath(configuration) + (un, m) map concat[Attributed[File]] + } - def fullClasspath(configuration: Configuration, intermediate: Boolean): Task[Seq[File]] = - (dependencyClasspath(configuration), products(configuration, intermediate) ) map concat[File] + lazy val dependencyClasspath: Classpath = + TaskMap { (configuration: Configuration) => + val external = externalDependencyClasspath(configuration) + val internal = internalDependencyClasspath(configuration) + (external, internal) map concat[Attributed[File]] + } + lazy val fullClasspath: Classpath = + TaskMap { case (configuration: Configuration) => + val dep = dependencyClasspath(configuration) + val prod = products(configuration) + (dep, prod) map concat[Attributed[File]] + } } trait BasicClasspathProject extends ClasspathProject @@ -42,18 +62,25 @@ trait BasicClasspathProject extends ClasspathProject def classpathFilter: FileFilter = GlobFilter("*.jar") def defaultExcludeFilter: FileFilter = MultiProject.defaultExcludes - override def managedClasspath(configuration: Configuration) = - update map { _.getOrElse(configuration, error("No such configuration '" + configuration.toString + "'")) } - - def unmanagedClasspath(configuration: Configuration): Task[Seq[File]] = - unmanagedBase map { base => - (base * (classpathFilter -- defaultExcludeFilter) +++ - (base / configuration.toString).descendentsExcept(classpathFilter, defaultExcludeFilter)).getFiles.toSeq + override val managedClasspath: Classpath = + TaskMap { configuration => + update map { x => attributed(x.getOrElse(configuration, error("No such configuration '" + configuration.toString + "'")) ) } } + val unmanagedClasspath: Classpath = + TaskMap { configuration => + unmanagedBase map { base => + attributed( (base * (classpathFilter -- defaultExcludeFilter) +++ + (base / configuration.toString).descendentsExcept(classpathFilter, defaultExcludeFilter)).getFiles.toSeq ) + } + } + + lazy val configurationMap: Map[String, Configuration] = + configurations map { conf => (conf.name, conf) } toMap; + import Types._ lazy val update = (ivyModule, updateConfig) map { case module :+: config :+: HNil => - val confMap = configurations map { conf => (conf.name, conf) } toMap; + val confMap = configurationMap IvyActions.update(module, config) map { case (key, value) => (confMap(key), value) } toMap; } } @@ -61,7 +88,7 @@ trait BasicClasspathProject extends ClasspathProject trait DefaultClasspathProject extends BasicClasspathProject with Project { def projectID: ModuleID - def baseResolvers: Seq[Resolver] = Resolver.withDefaultResolvers(ReflectUtilities.allVals[Resolver](this).toSeq.map(_._2) ) + def baseResolvers: Seq[Resolver] lazy val resolvers: Task[Seq[Resolver]] = task { baseResolvers } def otherResolvers: Seq[Resolver] = Nil @@ -84,6 +111,20 @@ trait DefaultClasspathProject extends BasicClasspathProject with Project def ivyScala: Option[IvyScala] = None def ivyValidate: Boolean = false + //TODO: transitive dependencies + lazy val internalDependencyClasspath: Classpath = + TaskMap { (conf: Configuration) => + val confMap = configurationMap + val productsTasks = + for( (p: ClasspathProject, Some(confString)) <- ClasspathProject.resolvedDependencies(this)) yield + { + println("Project " + p.name + ", conf: " + confString) + val to = parseSimpleConfigurations(confString).getOrElse(conf.toString, missingMapping(this.name, p.name, conf.toString)) + p.products(confMap(to)) + } + (productsTasks.toSeq.join) named(name + "/join") map(_.flatten) named(name + "/int") + } + lazy val unmanagedBase = task { dependencyPath.asFile } lazy val moduleSettings: Task[ModuleSettings] = task { @@ -98,10 +139,10 @@ trait MultiClasspathProject extends DefaultClasspathProject def version: String def projectDependencies: Iterable[ModuleID] = - ClasspathProject.resolvedDependencies(this) collect { case (p: DefaultClasspathProject, conf) => p.projectID.copy(configurations = conf) } + resolvedDependencies(this) collect { case (p: DefaultClasspathProject, conf) => p.projectID.copy(configurations = conf) } lazy val projectResolver = - ClasspathProject.depMap(this) map { m => + depMap(this) map { m => new RawRepository(new ProjectResolver("inter-project", m)) } @@ -111,14 +152,42 @@ trait MultiClasspathProject extends DefaultClasspathProject override lazy val resolvers: Task[Seq[Resolver]] = projectResolver map { _ +: baseResolvers } } +trait ReflectiveClasspathProject extends DefaultClasspathProject +{ + private[this] def vals[T: Manifest] = ReflectUtilities.allVals[T](this).toSeq.map(_._2) + def configurations: Seq[Configuration] = vals[Configuration] + def baseResolvers: Seq[Resolver] = Resolver.withDefaultResolvers(vals[Resolver] ) +} + import org.apache.ivy.core.module import module.id.ModuleRevisionId import module.descriptor.ModuleDescriptor object ClasspathProject { + type Classpath = Configuration => Task[Seq[Attributed[File]]] + + val Analyzed = AttributeKey[Analysis]("analysis") + + def attributed[T](in: Seq[T]): Seq[Attributed[T]] = in map Attributed.blank + + def analyzed[T](data: T, analysis: Analysis) = Attributed.blank(data).put(Analyzed, analysis) + + def analyzed(compile: Task[Analysis], inputs: Task[Compile.Inputs]): Task[Attributed[File]] = + (compile, inputs) map { case analysis :+: i :+: HNil => + analyzed(i.config.classesDirectory, analysis) + } + def concat[A]: (Seq[A], Seq[A]) => Seq[A] = _ ++ _ + def extractAnalysis[T](a: Attributed[T]): (T, Analysis) = + (a.data, a.metadata get Analyzed getOrElse Analysis.Empty) + + def analysisMap[T](cp: Seq[Attributed[T]]): Map[T, Analysis] = + (cp map extractAnalysis).toMap + + def data[T](in: Seq[Attributed[T]]): Seq[T] = in.map(_.data) + def depMap(root: Project): Task[Map[ModuleRevisionId, ModuleDescriptor]] = depMap(MultiProject.topologicalSort(root).dropRight(1) collect { case cp: DefaultClasspathProject => cp }) @@ -131,20 +200,27 @@ object ClasspathProject } def resolvedDependencies(p: Project): Iterable[(Project, Option[String])] = - { - import ProjectDependency.Classpath - p.dependencies map { - case Classpath(Left(extPath), conf) => (p.info.externals(extPath), conf) - case Classpath(Right(proj), conf) => (proj, conf) + p.dependencies map { cp => + (resolveProject(cp.project, p), cp.configuration) } - } + + def resolveProject(e: Either[File, Project], context: Project): Project = + e match { + case Left(extPath) => context.info.externals(extPath) + case Right(proj) => proj + } + def parseSimpleConfigurations(confString: String): Map[String, String] = - confString.split(";").map( conf => - conf.split("->",2).toList.map(_.trim) match { - case x :: Nil => (x,x) - case x :: y :: Nil => (x,y) + confString.split(";").flatMap( conf => + trim(conf.split("->",2)) match { + case x :: Nil => (x,x) :: Nil + case x :: y :: Nil => trim(x.split(",")) map { a => (a,y) } case _ => error("Invalid configuration '" + conf + "'") // shouldn't get here } ).toMap + private def trim(a: Array[String]): List[String] = a.toList.map(_.trim) + + def missingMapping(from: String, to: String, conf: String) = + error("No configuration mapping defined from '" + from + "' to '" + to + "' for '" + conf + "'") } \ No newline at end of file diff --git a/main/Compile.scala b/main/Compile.scala index 018257fc3..8bbb7752c 100644 --- a/main/Compile.scala +++ b/main/Compile.scala @@ -14,7 +14,7 @@ object Compile final class Inputs(val compilers: Compilers, val config: Options, val incSetup: IncSetup, val log: Logger) final class Options(val classpath: Seq[File], val sources: Seq[File], val classesDirectory: File, val options: Seq[String], val javacOptions: Seq[String], val maxErrors: Int) - final class IncSetup(val javaSrcBases: Seq[File], val cacheDirectory: File) + final class IncSetup(val javaSrcBases: Seq[File], val analysisMap: Map[File, Analysis], val cacheDirectory: File) final class Compilers(val scalac: AnalyzingCompiler, val javac: JavaCompiler) def inputs(classpath: Seq[File], sources: Seq[File], outputDirectory: File, options: Seq[String], javacOptions: Seq[String], javaSrcBases: Seq[File], maxErrors: Int)(implicit compilers: Compilers, log: Logger): Inputs = @@ -23,13 +23,13 @@ object Compile val classesDirectory = outputDirectory / "classes" val cacheDirectory = outputDirectory / "cache" val augClasspath = classesDirectory.asFile +: classpath - inputs(augClasspath, sources, classesDirectory, options, javacOptions, javaSrcBases, cacheDirectory, maxErrors) + inputs(augClasspath, sources, classesDirectory, options, javacOptions, javaSrcBases, Map.empty, cacheDirectory, maxErrors) } - def inputs(classpath: Seq[File], sources: Seq[File], classesDirectory: File, options: Seq[String], javacOptions: Seq[String], javaSrcBases: Seq[File], cacheDirectory: File, maxErrors: Int)(implicit compilers: Compilers, log: Logger): Inputs = + def inputs(classpath: Seq[File], sources: Seq[File], classesDirectory: File, options: Seq[String], javacOptions: Seq[String], javaSrcBases: Seq[File], analysisMap: Map[File, Analysis], cacheDirectory: File, maxErrors: Int)(implicit compilers: Compilers, log: Logger): Inputs = new Inputs( compilers, new Options(classpath, sources, classesDirectory, options, javacOptions, maxErrors), - new IncSetup(javaSrcBases, cacheDirectory), + new IncSetup(javaSrcBases, analysisMap, cacheDirectory), log ) @@ -74,6 +74,6 @@ object Compile import in.incSetup._ val agg = new build.AggressiveCompile(cacheDirectory) - agg(scalac, javac, sources, classpath, classesDirectory, javaSrcBases, options, javacOptions, maxErrors)(in.log) + agg(scalac, javac, sources, classpath, classesDirectory, javaSrcBases, options, javacOptions, analysisMap, maxErrors)(in.log) } } \ No newline at end of file diff --git a/main/build/AggressiveCompile.scala b/main/build/AggressiveCompile.scala index 4d8fbc19a..ed684b29c 100644 --- a/main/build/AggressiveCompile.scala +++ b/main/build/AggressiveCompile.scala @@ -21,14 +21,14 @@ final class CompileConfiguration(val sources: Seq[File], val classpath: Seq[File class AggressiveCompile(cacheDirectory: File) { - def apply(compiler: AnalyzingCompiler, javac: JavaCompiler, sources: Seq[File], classpath: Seq[File], outputDirectory: File, javaSrcBases: Seq[File] = Nil, options: Seq[String] = Nil, javacOptions: Seq[String] = Nil, maxErrors: Int = 100)(implicit log: Logger): Analysis = + def apply(compiler: AnalyzingCompiler, javac: JavaCompiler, sources: Seq[File], classpath: Seq[File], outputDirectory: File, javaSrcBases: Seq[File] = Nil, options: Seq[String] = Nil, javacOptions: Seq[String] = Nil, analysisMap: Map[File, Analysis] = Map.empty, maxErrors: Int = 100)(implicit log: Logger): Analysis = { val setup = new CompileSetup(outputDirectory, new CompileOptions(options, javacOptions), compiler.scalaInstance.actualVersion, CompileOrder.Mixed) - compile1(sources, classpath, javaSrcBases, setup, store, Map.empty, compiler, javac, maxErrors) + compile1(sources, classpath, javaSrcBases, setup, store, analysisMap, compiler, javac, maxErrors) } def withBootclasspath(args: CompilerArguments, classpath: Seq[File]): Seq[File] = - args.bootClasspath ++ classpath + args.bootClasspath ++ args.finishClasspath(classpath) def compile1(sources: Seq[File], classpath: Seq[File], javaSrcBases: Seq[File], setup: CompileSetup, store: AnalysisStore, analysis: Map[File, Analysis], compiler: AnalyzingCompiler, javac: JavaCompiler, maxErrors: Int)(implicit log: Logger): Analysis = { @@ -46,9 +46,10 @@ class AggressiveCompile(cacheDirectory: File) val extApis = getAnalysis(f) match { case Some(a) => a.apis.external; case None => Map.empty[String, Source] } extApis.get _ } - val apiOrEmpty = (api: Either[Boolean, Source]) => api.right.toOption.getOrElse( APIs.emptyAPI ) + val apiOption= (api: Either[Boolean, Source]) => api.right.toOption val cArgs = new CompilerArguments(compiler.scalaInstance, compiler.cp) - val externalAPI = apiOrEmpty compose Locate.value(withBootclasspath(cArgs, classpath), getAPI) + val searchClasspath = withBootclasspath(cArgs, classpath) + val entry = Locate.entry(searchClasspath) val compile0 = (include: Set[File], callback: AnalysisCallback) => { IO.createDirectory(outputDirectory) @@ -72,7 +73,7 @@ class AggressiveCompile(cacheDirectory: File) case Some(previous) if equiv.equiv(previous, currentSetup) => previousAnalysis case _ => Incremental.prune(sourcesSet, previousAnalysis) } - IncrementalCompile(sourcesSet, compile0, analysis, externalAPI) + IncrementalCompile(sourcesSet, entry, compile0, analysis, getAnalysis, outputDirectory) } private def extract(previous: Option[(Analysis, CompileSetup)]): (Analysis, Option[CompileSetup]) = previous match diff --git a/util/classfile/Analyze.scala b/util/classfile/Analyze.scala index d9d803010..9a03361e0 100644 --- a/util/classfile/Analyze.scala +++ b/util/classfile/Analyze.scala @@ -56,6 +56,7 @@ private[sbt] object Analyze val loaded = load(tpe, Some("Problem processing dependencies of source " + source)) for(clazz <- loaded; file <- ErrorHandling.convert(IO.classLocationFile(clazz)).right) { + val name = clazz.getName if(file.isDirectory) { val resolved = resolveClassFile(file, tpe) @@ -66,14 +67,14 @@ private[sbt] object Analyze productToSource.get(resolvedPath) match { case Some(dependsOn) => analysis.sourceDependency(dependsOn, source) - case None => analysis.productDependency(resolvedPath, source) + case None => analysis.binaryDependency(resolved, clazz.getName, source) } } else - analysis.classDependency(resolved, source) + analysis.binaryDependency(resolved, name, source) } else - analysis.jarDependency(file, source) + analysis.binaryDependency(file, name, source) } } }