Track dependencies for Java sources using classfile parsing (with the usual limitations of this approach)

git-svn-id: https://simple-build-tool.googlecode.com/svn/trunk@886 d89573ee-9141-11dd-94d4-bdf5e562f29c

src/main/scala/sbt/BuilderProject.scala

@@ -47,6 +47,7 @@ private sealed abstract class BasicBuilderProject extends InternalProject with S
 		new BuilderCompileConfiguration
 		{
 			def label = "builder"
+			def sourceRoots = info.projectPath +++ path(DefaultSourceDirectoryName)
 			def sources = (info.projectPath * sourceFilter) +++ path(DefaultSourceDirectoryName).descendentsExcept(sourceFilter, defaultExcludes)
 			def outputDirectory = compilePath
 			def classpath = projectClasspath
@@ -188,7 +189,8 @@ private final class BuilderProject(val info: ProjectInfo, val pluginPath: Path,
 			new BuilderCompileConfiguration
 			{
 				def label = "plugin builder"
-				def sources = descendents(managedSourcePath, sourceFilter)
+				def sourceRoots = managedSourcePath
+				def sources = descendents(sourceRoots, sourceFilter)
 				def outputDirectory = outputPath / "plugin-classes"
 				def classpath: PathFinder = pluginClasspath +++ sbtJarPath
 				def analysisPath = outputPath / "plugin-analysis"

src/main/scala/sbt/Conditional.scala

@@ -52,7 +52,10 @@ trait Conditional[Source, Product, External] extends NotNull
 		removedSources --= sourcesSnapshot
 		val removedCount = removedSources.size
 		for(removed <- removedSources)
+		{
+			log.debug("Source " + removed + " removed.")
 			analysis.removeDependent(removed)
+		}
 		
 		val unmodified = new HashSet[Source]
 		val modified = new HashSet[Source]
@@ -186,6 +189,7 @@ trait Conditional[Source, Product, External] extends NotNull
 abstract class AbstractCompileConfiguration extends NotNull
 {
 	def label: String
+	def sourceRoots: PathFinder
 	def sources: PathFinder
 	def outputDirectory: Path
 	def classpath: PathFinder
@@ -321,7 +325,9 @@ abstract class AbstractCompileConditional(val config: AbstractCompileConfigurati
 		val id = AnalysisCallback.register(analysisCallback)
 		val allOptions = (("-Xplugin:" + FileUtilities.sbtJar.getAbsolutePath) ::
 			("-P:sbt-analyzer:callback:" + id.toString) :: Nil) ++ options
-		val r = (new Compile(config.maxErrors))(label, dirtySources, classpathString, outputDirectory, allOptions, javaOptions, compileOrder, log)
+		def run = (new Compile(config.maxErrors))(label, dirtySources, classpathString, outputDirectory, allOptions, javaOptions, compileOrder, log)
+		val loader = ClasspathUtilities.toLoader(cp)
+		val r = classfile.Analyze(projectPath, outputDirectory, dirtySources, sourceRoots.get, log)(analysis.allProducts, analysisCallback, loader)(run)
 		AnalysisCallback.unregister(id)
 		if(log.atLevel(Level.Debug))
 		{

src/main/scala/sbt/DefaultProject.scala

@@ -190,6 +190,7 @@ abstract class BasicScalaProject extends ScalaProject with BasicDependencyProjec
 	{
 		def baseCompileOptions = compileOptions
 		def label = mainLabel
+		def sourceRoots = mainSourceRoots
 		def sources = mainSources
 		def outputDirectory = mainCompilePath
 		def classpath = compileClasspath
@@ -201,6 +202,7 @@ abstract class BasicScalaProject extends ScalaProject with BasicDependencyProjec
 	{
 		def baseCompileOptions = testCompileOptions
 		def label = testLabel
+		def sourceRoots = testSourceRoots
 		def sources = testSources
 		def outputDirectory = testCompilePath
 		def classpath = testClasspath

src/main/scala/sbt/IntegrationTesting.scala

@@ -68,6 +68,7 @@ trait BasicIntegrationTesting extends ScalaIntegrationTesting with IntegrationTe
 	class IntegrationTestCompileConfig extends BaseCompileConfig
 	{
 		def label = integrationTestLabel
+		def sourceRoots = integrationTestScalaSourceRoots
 		def sources = integrationTestSources
 		def outputDirectory = integrationTestCompilePath
 		def classpath = integrationTestClasspath

src/main/scala/sbt/ProjectPaths.scala

@@ -19,6 +19,8 @@ trait ScalaPaths extends PackagePaths
 	def mainSources: PathFinder
 	/** A PathFinder that selects all test sources.*/
 	def testSources: PathFinder
+	def mainSourceRoots: PathFinder
+	def testSourceRoots: PathFinder
 	/** A PathFinder that selects all main resources.*/
 	def mainResources: PathFinder
 	/** A PathFinder that selects all test resources. */
@@ -62,8 +64,6 @@ trait ScalaPaths extends PackagePaths
 
 trait BasicScalaPaths extends Project with ScalaPaths
 {
-	def mainSourceRoots: PathFinder
-	def testSourceRoots: PathFinder
 	def mainResourcesPath: PathFinder
 	def testResourcesPath: PathFinder
 	def managedDependencyRootPath: Path
@@ -265,6 +265,7 @@ trait MavenStyleWebstartPaths extends WebstartPaths with MavenStyleScalaPaths
 trait IntegrationTestPaths extends NotNull
 {
 	def integrationTestSources: PathFinder
+	def integrationTestScalaSourceRoots: PathFinder
 	def integrationTestResourcesPath: Path
 
 	def integrationTestCompilePath: Path
@@ -273,8 +274,9 @@ trait IntegrationTestPaths extends NotNull
 trait BasicIntegrationTestPaths extends IntegrationTestPaths
 {
 	def integrationTestScalaSourcePath: Path
-	def integrationTestSources = sources(integrationTestScalaSourcePath)
+	def integrationTestScalaSourceRoots: PathFinder = integrationTestScalaSourcePath
-	protected def sources(base: Path): PathFinder
+	def integrationTestSources = sources(integrationTestScalaSourceRoots)
+	protected def sources(base: PathFinder): PathFinder
 }
 trait MavenStyleIntegrationTestPaths extends BasicIntegrationTestPaths with MavenStyleScalaPaths
 {

src/main/scala/sbt/classfile/Analyze.scala

@@ -0,0 +1,98 @@
+/* sbt -- Simple Build Tool
+ * Copyright 2009 Mark Harrah
+ */
+package sbt.classfile
+
+import scala.collection.mutable
+import mutable.{ArrayBuffer, Buffer}
+import java.io.File
+
+object Analyze
+{
+	def apply[T](basePath: Path, outputDirectory: Path, sources: Iterable[Path], roots: Iterable[Path], log: Logger)
+		(allProducts: => scala.collection.Set[Path], analysis: AnalysisCallback, loader: ClassLoader)
+		(compile: => Option[String]): Option[String] =
+	{
+		val sourceSet = Set(sources.toSeq : _*)
+		val classesFinder = outputDirectory ** GlobFilter("*.class")
+		val existingClasses = classesFinder.get
+		
+		// runs after compilation
+		def analyze()
+		{
+			val allClasses = Set(classesFinder.get.toSeq : _*)
+			val newClasses = allClasses -- existingClasses -- allProducts
+			
+			val productToSource = new mutable.HashMap[Path, Path]
+			val sourceToClassFiles = new mutable.HashMap[Path, Buffer[ClassFile]]
+			
+			// parse class files and assign classes to sources.  This must be done before dependencies, since the information comes
+			// as class->class dependencies that must be mapped back to source->class dependencies using the source+class assignment
+			for(newClass <- newClasses;
+				path <- Path.relativize(outputDirectory, newClass);
+				classFile = Parser(newClass.asFile, log);
+				sourceFile <- classFile.sourceFile;
+				source <- guessSourcePath(sourceSet, roots, classFile.className, log))
+			{
+				analysis.beginSource(source)
+				analysis.generatedClass(source, path)
+				productToSource(path) = source
+				sourceToClassFiles.getOrElseUpdate(source, new ArrayBuffer[ClassFile]) += classFile
+			}
+			
+			// get class to class dependencies and map back to source to class dependencies
+			for( (source, classFiles) <- sourceToClassFiles )
+			{
+				for(classFile <- classFiles if isTopLevel(classFile);
+					method <- classFile.methods; if method.isMain)
+						analysis.foundApplication(source, classFile.className)
+				def processDependency(tpeSlashed: String)
+				{
+					val tpe = tpeSlashed.replace('/','.')
+					Control.trapAndLog(log)
+					{
+						val clazz = Class.forName(tpe, false, loader)
+						val file = FileUtilities.classLocationFile(clazz)
+						if(file.isDirectory)
+						{
+							val resolved = resolveClassFile(file, tpeSlashed)
+							require(resolved.exists)
+							val resolvedPath = Path.fromFile(resolved)
+							if(Path.fromFile(file) == outputDirectory)
+							{
+								productToSource.get(resolvedPath) match
+								{
+									case Some(dependsOn) => analysis.sourceDependency(dependsOn, source)
+									case None => analysis.productDependency(resolvedPath, source)
+								}
+							}
+							else
+								analysis.classDependency(resolved, source)
+						}
+						else
+							analysis.jarDependency(file, source)
+					}
+				}
+				
+				classFiles.flatMap(_.types).foreach(processDependency)
+				analysis.endSource(source)
+			}
+		}
+		
+		compile orElse Control.convertErrorMessage(log)(analyze()).left.toOption
+	}
+	private def resolveClassFile(file: File, className: String): File = (file /: (className + ".class").split("""\\"""))(new File(_, _))
+	private def guessSourcePath(sources: scala.collection.Set[Path], roots: Iterable[Path], className: String, log: Logger) =
+	{
+		val relativeSourceFile = className.replace('.', '/') + ".java"
+		val candidates = roots.map(root => Path.fromString(root, relativeSourceFile)).filter(sources.contains).toList
+		candidates match
+		{
+			case Nil => log.warn("Could not determine source for class " + className)
+			case head :: Nil => ()
+			case _ =>log.warn("Multiple sources matched for class " + className + ": " + candidates.mkString(", "))
+		}
+		candidates
+	}
+	private def isTopLevel(classFile: ClassFile) = classFile.className.indexOf('$') < 0
+}

src/main/scala/sbt/classfile/ClassFile.scala

@@ -0,0 +1,65 @@
+/* sbt -- Simple Build Tool
+ * Copyright 2009 Mark Harrah
+ */
+package sbt.classfile
+
+import Constants._
+import java.io.File
+
+trait ClassFile
+{
+	val majorVersion: Int
+	val minorVersion: Int
+	val fileName: String
+	val className: String
+	val superClassName: String
+	val interfaceNames: RandomAccessSeq[String]
+	val accessFlags: Int
+	val constantPool: RandomAccessSeq[Constant]
+	val fields: RandomAccessSeq[FieldOrMethodInfo]
+	val methods: RandomAccessSeq[FieldOrMethodInfo]
+	val attributes: RandomAccessSeq[AttributeInfo]
+	val sourceFile: Option[String]
+	def types: Set[String]
+	def stringValue(a: AttributeInfo): String
+}
+
+final case class Constant(tag: Byte, nameIndex: Int, typeIndex: Int, value: Option[AnyRef]) extends NotNull
+{
+	def this(tag: Byte, nameIndex: Int, typeIndex: Int) = this(tag, nameIndex, typeIndex, None)
+	def this(tag: Byte, nameIndex: Int) = this(tag, nameIndex, -1)
+	def this(tag: Byte, value: AnyRef) = this(tag, -1, -1, Some(value))
+	def wide = tag == ConstantLong || tag == ConstantDouble
+}
+final case class FieldOrMethodInfo(accessFlags: Int, name: Option[String], descriptor: Option[String], attributes: RandomAccessSeq[AttributeInfo]) extends NotNull
+{
+	def isStatic = (accessFlags&ACC_STATIC)== ACC_STATIC
+	def isPublic = (accessFlags&ACC_PUBLIC)==ACC_PUBLIC
+	def isMain = isPublic && isStatic && descriptor.filter(_ == "([Ljava/lang/String;)V").isDefined
+}
+final case class AttributeInfo(name: Option[String], value: Array[Byte]) extends NotNull
+{
+	def isNamed(s: String) = name.filter(s == _).isDefined
+	def isSignature = isNamed("Signature")
+	def isSourceFile = isNamed("SourceFile")
+}
+object Constants
+{
+	final val ACC_STATIC = 0x0008
+	final val ACC_PUBLIC = 0x0001
+
+	final val JavaMagic = 0xCAFEBABE
+	final val ConstantUTF8 = 1
+	final val ConstantUnicode = 2
+	final val ConstantInteger = 3
+	final val ConstantFloat = 4
+	final val ConstantLong = 5
+	final val ConstantDouble = 6
+	final val ConstantClass = 7
+	final val ConstantString = 8
+	final val ConstantField = 9
+	final val ConstantMethod = 10
+	final val ConstantInterfaceMethod = 11
+	final val ConstantNameAndType = 12
+	final val ClassDescriptor = 'L'
+}

src/main/scala/sbt/classfile/Parser.scala

@@ -0,0 +1,171 @@
+/* sbt -- Simple Build Tool
+ * Copyright 2009 Mark Harrah
+ */
+package sbt.classfile
+
+import java.io.{DataInputStream, File, InputStream}
+
+// Translation of jdepend.framework.ClassFileParser by Mike Clark, Clarkware Consulting, Inc.
+// BSD Licensed
+//
+// Note that unlike the rest of sbt, some things might be null.
+
+import Constants._
+
+object Parser
+{
+	def apply(file: File, log: Logger): ClassFile = FileUtilities.readStreamValue(file, log)(parse(file.getCanonicalPath, log)).right.get
+	private def parse(fileName: String, log: Logger)(is: InputStream): Either[String, ClassFile] = Right(parseImpl(fileName, is, log))
+	private def parseImpl(filename: String, is: InputStream, log: Logger): ClassFile =
+	{
+		val in = new DataInputStream(is)
+		new ClassFile
+		{
+			assume(in.readInt() == JavaMagic, "Invalid class file: " + fileName)
+			
+			val fileName = filename
+			val minorVersion: Int = in.readUnsignedShort()
+			val majorVersion: Int = in.readUnsignedShort()
+
+			val constantPool = parseConstantPool(in)
+			val accessFlags: Int = in.readUnsignedShort()
+
+			val className = getClassConstantName(in.readUnsignedShort())
+			val superClassName = getClassConstantName(in.readUnsignedShort())
+			val interfaceNames = array(in.readUnsignedShort())(getClassConstantName(in.readUnsignedShort()))
+
+			val fields = readFieldsOrMethods()
+			val methods = readFieldsOrMethods()
+
+			val attributes = array(in.readUnsignedShort())(parseAttribute())
+
+			lazy val sourceFile =
+				for(sourceFileAttribute <- attributes.find(_.isSourceFile)) yield
+					toUTF8(entryIndex(sourceFileAttribute))
+
+			def stringValue(a: AttributeInfo) = toUTF8(entryIndex(a))
+
+			private def readFieldsOrMethods() = array(in.readUnsignedShort())(parseFieldOrMethodInfo())
+			private def toUTF8(entryIndex: Int) =
+			{
+				val entry = constantPool(entryIndex)
+				assume(entry.tag == ConstantUTF8, "Constant pool entry is not a UTF8 type: " + entryIndex)
+				entry.value.get.asInstanceOf[String]
+			}
+			private def getClassConstantName(entryIndex: Int) =
+			{
+				val entry = constantPool(entryIndex)
+				if(entry == null) ""
+				else slashesToDots(toUTF8(entry.nameIndex))
+			}
+			private def toString(index: Int) =
+			{
+				if(index <= 0) None
+				else Some(toUTF8(index))
+			}
+			private def parseFieldOrMethodInfo() =
+				new FieldOrMethodInfo(in.readUnsignedShort(), toString(in.readUnsignedShort()), toString(in.readUnsignedShort()),
+					array(in.readUnsignedShort())(parseAttribute()) )
+			private def parseAttribute() =
+			{
+				val nameIndex = in.readUnsignedShort()
+				val name = if(nameIndex == -1) None else Some(toUTF8(nameIndex))
+				val value = array(in.readInt())(in.readByte())
+				new AttributeInfo(name, value)
+			}
+
+			def types = Set((fieldTypes ++ methodTypes ++ classConstantReferences) : _*)
+
+			private def getTypes(fieldsOrMethods: Array[FieldOrMethodInfo]) =
+				fieldsOrMethods.flatMap { fieldOrMethod =>
+					descriptorToTypes(fieldOrMethod.descriptor)
+				}
+			
+			private def fieldTypes = getTypes(fields)
+			private def methodTypes = getTypes(methods)
+			
+			private def classConstantReferences =
+				constants.flatMap { constant =>
+					constant.tag match
+					{
+						case ConstantClass => toUTF8(constant.nameIndex) :: Nil
+						case _ => Nil
+					}
+				}
+			private def constants =
+			{
+				def next(i: Int, list: List[Constant]): List[Constant] =
+				{
+					if(i < constantPool.length)
+					{
+						val constant = constantPool(i)
+						next(if(constant.wide) i+2 else i+1, constant :: list)
+					}
+					else
+						list
+				}
+				next(1, Nil)
+			}
+		}
+    }
+    private def array[T](size: Int)(f: => T) = Array.fromFunction(i => f)(size)
+	private def parseConstantPool(in: DataInputStream) =
+	{
+		val constantPoolSize = in.readUnsignedShort()
+		val pool = new Array[Constant](constantPoolSize)
+		
+		def parse(i: Int): Unit =
+			if(i < constantPoolSize)
+			{
+				val constant = getConstant(in)
+				pool(i) = constant
+				parse( if(constant.wide) i+2 else i+1 )
+			}
+		
+		parse(1) // to lookup: why 1?
+		pool
+	}
+
+	private def getConstant(in: DataInputStream) =
+	{
+		val tag = in.readByte()
+		tag match
+		{
+			case ConstantClass | ConstantString => new Constant(tag, in.readUnsignedShort())
+			case ConstantField | ConstantMethod | ConstantInterfaceMethod | ConstantNameAndType => 
+				new Constant(tag, in.readUnsignedShort(), in.readUnsignedShort())
+			case ConstantInteger => new Constant(tag, new java.lang.Integer(in.readInt()))
+			case ConstantFloat => new Constant(tag, new java.lang.Float(in.readFloat()))
+			case ConstantLong => new Constant(tag, new java.lang.Long(in.readLong()))
+			case ConstantDouble => new Constant(tag, new java.lang.Double(in.readDouble()))
+			case ConstantUTF8 => new Constant(tag, in.readUTF())
+			case _ => error("Unknown constant: " + tag)
+		}
+	}
+
+	private def toInt(v: Byte) = if(v < 0) v + 256 else v.toInt
+	private def entryIndex(a: AttributeInfo) =
+	{
+		val Array(v0, v1) = a.value
+		toInt(v0) * 256 + toInt(v1)
+	}
+
+	private def slashesToDots(s: String) = s.replace('/', '.')
+
+	private def descriptorToTypes(descriptor: Option[String]) =
+	{
+		def toTypes(descriptor: String, types: List[String]): List[String] =
+		{
+			val startIndex = descriptor.indexOf(ClassDescriptor)
+			if(startIndex < 0)
+				types
+			else
+			{
+				val endIndex = descriptor.indexOf(';', startIndex+1)
+				val tpe = slashesToDots(descriptor.substring(startIndex + 1, endIndex))
+				toTypes(descriptor.substring(endIndex), tpe :: types)
+			}
+		}
+		toTypes(descriptor.getOrElse(""), Nil)
+    }
+}

src/sbt-test/java/separate/project/build.properties

@@ -0,0 +1,2 @@
+project.name=Test
+project.version=1.0

src/sbt-test/java/separate/project/build/src/TestProject.scala

@@ -0,0 +1,6 @@
+import sbt._
+
+class TestProject(info: ProjectInfo) extends DefaultProject(info)
+{
+	override def compileOrder = CompileOrder.JavaThenScala
+}

src/sbt-test/java/separate/src/main/java/F.java

@@ -0,0 +1,5 @@
+package fj;
+
+public interface F<A, B> {
+  B f(A a);
+}

src/sbt-test/java/separate/src/main/scala/Dual.scala

@@ -0,0 +1,11 @@
+package scalaz
+
+trait Dual[A] {
+  val value : A
+}
+
+object Dual {
+  implicit def dual[A](a: A) = new Dual[A] {
+    val value = a
+  }
+}

src/sbt-test/java/separate/src/main/scala/FImpl.scala

@@ -0,0 +1,3 @@
+package test
+
+trait FImpl[A,B] extends fj.F[A,B]

src/sbt-test/java/separate/test

@@ -0,0 +1,9 @@
+> clean
+[success]
+
+> test
+[success]
+
+# this will fail if the sources are recompiled again (see the project definition)
+> test
+[success]

src/sbt-test/source-dependencies/java-basic/changes/A.java

@@ -0,0 +1,6 @@
+package a;
+
+public class A
+{
+	public static int x() { return 3; }
+}

src/sbt-test/source-dependencies/java-basic/changes/A2.java

@@ -0,0 +1,6 @@
+package a.b;
+
+public class A
+{
+	public static int x() { return 3; }
+}

src/sbt-test/source-dependencies/java-basic/changes/B1.java

@@ -0,0 +1,6 @@
+package a.b;
+
+public class B
+{
+	public int y() { return 3; }
+}

src/sbt-test/source-dependencies/java-basic/changes/B2.java

@@ -0,0 +1,6 @@
+package a.b;
+
+public class B
+{
+	public int y() { return a.A.x(); }
+}

src/sbt-test/source-dependencies/java-basic/changes/B3.java

@@ -0,0 +1,6 @@
+package a.b;
+
+public class B
+{
+	public static void main(String[] args) {}
+}

src/sbt-test/source-dependencies/java-basic/project/build.properties

@@ -0,0 +1,2 @@
+project.name=Basic Java Dependency Test
+project.version=1.0

src/sbt-test/source-dependencies/java-basic/test

@@ -0,0 +1,61 @@
+# Basic test for Java dependency tracking
+
+# A is a basic Java file with no dependencies.  Just a basic check for Java compilation
+$ copy-file changes/A.java src/main/java/a/A.java
+[success]
+> compile
+[success]
+
+# A2 is a basic Java file with no dependencies.  This is added to verify
+# that classes are properly mapped back to their source.
+# (There are two files named A.java now, one in a/ and one in a/b)
+$ copy-file changes/A2.java src/main/java/a/b/A.java
+[success]
+> compile
+[success]
+
+# This adds B, another basic Java file with no dependencies
+$ copy-file changes/B1.java src/main/java/a/b/B.java
+[success]
+> compile
+[success]
+
+# Now, modify B so that it depends on a.A
+# This ensures that dependencies on a source not included in the compilation
+# (a/A.java has not changed) are tracked
+$ copy-file changes/B2.java src/main/java/a/b/B.java
+[success]
+> compile
+[success]
+
+# Remove a.b.A and there should be no problem compiling, since B should
+# have recorded a dependency on a.A and not a.b.A
+$ delete src/main/java/a/b/A.java
+[success]
+> compile
+[success]
+
+# Remove a.A and B should be recompiled if the dependency on a.A was properly
+# recorded.  This should be a compile error, since we haven't updated B to not
+# depend on A
+$ delete src/main/java/a/A.java
+[success]
+> compile
+[failure]
+
+# Replace B with a new B that doesn't depend on a.A and so it should compile
+# It shouldn't run though, because it doesn't have a main method
+$ copy-file changes/B1.java src/main/java/a/b/B.java
+[success]
+> compile
+[success]
+> run
+[failure]
+
+
+# Replace B with a new B that has a main method and should therefore run
+# if the main method was properly detected
+$ copy-file changes/B3.java src/main/java/a/b/B.java
+[success]
+> run
+[success]