From 3c4bc23cdb491b79abbebc4ebc9294a55e949648 Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Wed, 5 Nov 2014 14:31:26 -0500 Subject: [PATCH] First set of refactorings from review. * Split Java analyzing compile into its own class. * MixedAnalyzingCompiler now only does the mixing * Start moving methods around to more-final locations * Static analyzingCompile method now constructs a MixedAnalyzingCOmpiler and delegates to incremental compile. --- .../src/main/scala/sbt/inc/Incremental.scala | 44 ++-- .../sbt/compiler/IncrementalCompiler.scala | 15 +- .../sbt/compiler/MixedAnalyzingCompiler.scala | 190 ++++++++---------- .../javac/AnalyzingJavaCompiler.scala | 108 ++++++++++ .../java/xsbti/compile/CompileProgress.java | 5 + .../xsbti/compile/IncrementalCompiler.java | 10 + project/Sbt.scala | 4 +- 7 files changed, 247 insertions(+), 129 deletions(-) create mode 100644 compile/src/main/scala/sbt/compiler/javac/AnalyzingJavaCompiler.scala diff --git a/compile/inc/src/main/scala/sbt/inc/Incremental.scala b/compile/inc/src/main/scala/sbt/inc/Incremental.scala index 9dfa04a64..fa0484206 100644 --- a/compile/inc/src/main/scala/sbt/inc/Incremental.scala +++ b/compile/inc/src/main/scala/sbt/inc/Incremental.scala @@ -10,28 +10,30 @@ import xsbti.api.{ Compilation, Source } import xsbti.compile.DependencyChanges import java.io.File -/** Helper class to run incremental compilation algorithm. - * - * - * This class delegates down to - * - IncrementalNameHashing - * - IncrementalDefault - * - IncrementalAnyStyle - */ +/** + * Helper class to run incremental compilation algorithm. + * + * + * This class delegates down to + * - IncrementalNameHashing + * - IncrementalDefault + * - IncrementalAnyStyle + */ object Incremental { - /** Runs the incremental compiler algorithm. - * - * @param sources The sources to compile - * @param entry The means of looking up a class on the classpath. - * @param previous The previously detected source dependencies. - * @param current A mechanism for generating stamps (timestamps, hashes, etc). - * @param doCompile The function which can run one level of compile. - * @param log The log where we write debugging information - * @param options Incremental compilation options - * @param equivS The means of testing whether two "Stamps" are the same. - * @return - * A flag of whether or not compilation completed succesfully, and the resulting dependency analysis object. - */ + /** + * Runs the incremental compiler algorithm. + * + * @param sources The sources to compile + * @param entry The means of looking up a class on the classpath. + * @param previous The previously detected source dependencies. + * @param current A mechanism for generating stamps (timestamps, hashes, etc). + * @param doCompile The function which can run one level of compile. + * @param log The log where we write debugging information + * @param options Incremental compilation options + * @param equivS The means of testing whether two "Stamps" are the same. + * @return + * A flag of whether or not compilation completed succesfully, and the resulting dependency analysis object. + */ def compile(sources: Set[File], entry: String => Option[File], previous: Analysis, diff --git a/compile/integration/src/main/scala/sbt/compiler/IncrementalCompiler.scala b/compile/integration/src/main/scala/sbt/compiler/IncrementalCompiler.scala index 845b0f45e..2100c4333 100644 --- a/compile/integration/src/main/scala/sbt/compiler/IncrementalCompiler.scala +++ b/compile/integration/src/main/scala/sbt/compiler/IncrementalCompiler.scala @@ -6,11 +6,18 @@ import sbt.inc.{ Analysis, IncOptions, TextAnalysisFormat } import xsbti.{ Logger, Maybe } import xsbti.compile._ +// TODO - +// 1. Move analyzingCompile from MixedAnalyzingCompiler into here +// 2. Create AnalyzingJavaComiler class +// 3. MixedAnalyzingCompiler should just provide the raw 'compile' method used in incremental compiler (and +// by this class. + /** - * An implementation of the incremetnal compiler that can compile inputs and dump out source dependency analysis. + * An implementation of the incremental compiler that can compile inputs and dump out source dependency analysis. */ object IC extends IncrementalCompiler[Analysis, AnalyzingCompiler] { - def compile(in: Inputs[Analysis, AnalyzingCompiler], log: Logger): Analysis = + + override def compile(in: Inputs[Analysis, AnalyzingCompiler], log: Logger): Analysis = { val setup = in.setup; import setup._ val options = in.options; import options.{ options => scalacOptions, _ } @@ -32,10 +39,10 @@ object IC extends IncrementalCompiler[Analysis, AnalyzingCompiler] { private[this] def m2o[S](opt: Maybe[S]): Option[S] = if (opt.isEmpty) None else Some(opt.get) @deprecated("0.13.8", "A logger is no longer needed.") - def newScalaCompiler(instance: ScalaInstance, interfaceJar: File, options: ClasspathOptions, log: Logger): AnalyzingCompiler = + override def newScalaCompiler(instance: ScalaInstance, interfaceJar: File, options: ClasspathOptions, log: Logger): AnalyzingCompiler = new AnalyzingCompiler(instance, CompilerInterfaceProvider.constant(interfaceJar), options) - def newScalaCompiler(instance: ScalaInstance, interfaceJar: File, options: ClasspathOptions): AnalyzingCompiler = + override def newScalaCompiler(instance: ScalaInstance, interfaceJar: File, options: ClasspathOptions): AnalyzingCompiler = new AnalyzingCompiler(instance, CompilerInterfaceProvider.constant(interfaceJar), options) def compileInterfaceJar(label: String, sourceJar: File, targetJar: File, interfaceJar: File, instance: ScalaInstance, log: Logger) { diff --git a/compile/integration/src/main/scala/sbt/compiler/MixedAnalyzingCompiler.scala b/compile/integration/src/main/scala/sbt/compiler/MixedAnalyzingCompiler.scala index cfe98b6c7..9b7aeb20a 100644 --- a/compile/integration/src/main/scala/sbt/compiler/MixedAnalyzingCompiler.scala +++ b/compile/integration/src/main/scala/sbt/compiler/MixedAnalyzingCompiler.scala @@ -5,6 +5,7 @@ import java.lang.ref.{ SoftReference, Reference } import sbt.classfile.Analyze import sbt.classpath.ClasspathUtilities +import sbt.compiler.javac.AnalyzingJavaCompiler import sbt.inc.Locate.DefinesClass import sbt._ import sbt.inc._ @@ -14,6 +15,80 @@ import xsbti.api.Source import xsbti.compile.CompileOrder._ import xsbti.compile._ +/** An instance of an analyzing compiler that can run both javac + scalac. */ +final class MixedAnalyzingCompiler( + val scalac: AnalyzingCompiler, + val javac: AnalyzingJavaCompiler, + val config: CompileConfiguration, + val log: Logger) { + import config._ + import currentSetup._ + + private[this] val absClasspath = classpath.map(_.getAbsoluteFile) + /** Mechanism to work with compiler arguments. */ + private[this] val cArgs = new CompilerArguments(compiler.scalaInstance, compiler.cp) + + /** + * Compiles the given Java/Scala files. + * + * @param include The files to compile right now + * @param changes A list of dependency changes. + * @param callback The callback where we report dependency issues. + */ + def compile(include: Set[File], changes: DependencyChanges, callback: AnalysisCallback): Unit = { + val outputDirs = outputDirectories(output) + outputDirs foreach (IO.createDirectory) + val incSrc = sources.filter(include) + val (javaSrcs, scalaSrcs) = incSrc partition javaOnly + logInputs(log, javaSrcs.size, scalaSrcs.size, outputDirs) + /** compiles the scala code necessary using the analyzing compiler. */ + def compileScala(): Unit = + if (!scalaSrcs.isEmpty) { + val sources = if (order == Mixed) incSrc else scalaSrcs + val arguments = cArgs(Nil, absClasspath, None, options.options) + timed("Scala compilation", log) { + compiler.compile(sources, changes, arguments, output, callback, reporter, config.cache, log, progress) + } + } + /** + * Compiles the Java code necessary. All analysis code is included in this method. + */ + def compileJava(): Unit = + if (!javaSrcs.isEmpty) { + // Runs the analysis portion of Javac. + timed("Java compile + analysis", log) { + javac.compile(javaSrcs, options.javacOptions.toArray[String], output, callback, reporter, log, progress) + } + } + // TODO - Maybe on "Mixed" we should try to compile both Scala + Java. + if (order == JavaThenScala) { compileJava(); compileScala() } else { compileScala(); compileJava() } + } + + private[this] def outputDirectories(output: Output): Seq[File] = output match { + case single: SingleOutput => List(single.outputDirectory) + case mult: MultipleOutput => mult.outputGroups map (_.outputDirectory) + } + /** Debugging method to time how long it takes to run various compilation tasks. */ + private[this] def timed[T](label: String, log: Logger)(t: => T): T = { + val start = System.nanoTime + val result = t + val elapsed = System.nanoTime - start + log.debug(label + " took " + (elapsed / 1e9) + " s") + result + } + + private[this] def logInputs(log: Logger, javaCount: Int, scalaCount: Int, outputDirs: Seq[File]) { + val scalaMsg = Analysis.counted("Scala source", "", "s", scalaCount) + val javaMsg = Analysis.counted("Java source", "", "s", javaCount) + val combined = scalaMsg ++ javaMsg + if (!combined.isEmpty) + log.info(combined.mkString("Compiling ", " and ", " to " + outputDirs.map(_.getAbsolutePath).mkString(",") + "...")) + } + + /** Returns true if the file is java. */ + private[this] def javaOnly(f: File) = f.getName.endsWith(".java") +} + /** * This is a compiler that mixes the `sbt.compiler.AnalyzingCompiler` for Scala incremental compilation * with a `xsbti.JavaCompiler`, allowing cross-compilation of mixed Java/Scala projects with analysis output. @@ -113,91 +188,21 @@ object MixedAnalyzingCompiler { { import config._ import currentSetup._ + // TODO - most of this around classpath-ness sohuld move to AnalyzingJavaCompiler constructor val absClasspath = classpath.map(_.getAbsoluteFile) val apiOption = (api: Either[Boolean, Source]) => api.right.toOption val cArgs = new CompilerArguments(compiler.scalaInstance, compiler.cp) val searchClasspath = explicitBootClasspath(options.options) ++ withBootclasspath(cArgs, absClasspath) val entry = Locate.entry(searchClasspath, definesClass) - - /** Here we set up a function that can run *one* iteration of the incremental compiler. - * - * Given: - * A set of files to compile - * A set of dependency changes - * A callback for analysis results - * - * Run: - * Scalac + Javac on the set of files, the order of which is determined by the current CompileConfiguration. - */ - val compile0 = (include: Set[File], changes: DependencyChanges, callback: AnalysisCallback) => { - val outputDirs = outputDirectories(output) - outputDirs foreach (IO.createDirectory) - val incSrc = sources.filter(include) - val (javaSrcs, scalaSrcs) = incSrc partition javaOnly - logInputs(log, javaSrcs.size, scalaSrcs.size, outputDirs) - /** compiles the scala code necessary using the analyzing compiler. */ - def compileScala(): Unit = - if (!scalaSrcs.isEmpty) { - val sources = if (order == Mixed) incSrc else scalaSrcs - val arguments = cArgs(Nil, absClasspath, None, options.options) - timed("Scala compilation", log) { - compiler.compile(sources, changes, arguments, output, callback, reporter, config.cache, log, progress) - } - } - /** Compiles the Java code necessary. All analysis code is included in this method. - * - * TODO - much of this logic should be extracted into an "AnalyzingJavaCompiler" class. - */ - def compileJava(): Unit = - if (!javaSrcs.isEmpty) { - import Path._ - @annotation.tailrec def ancestor(f1: File, f2: File): Boolean = - if (f2 eq null) false else if (f1 == f2) true else ancestor(f1, f2.getParentFile) - // Here we outline "chunks" of compiles we need to run so that the .class files end up in the right - // location for Java. - val chunks: Map[Option[File], Seq[File]] = output match { - case single: SingleOutput => Map(Some(single.outputDirectory) -> javaSrcs) - case multi: MultipleOutput => - javaSrcs groupBy { src => - multi.outputGroups find { out => ancestor(out.sourceDirectory, src) } map (_.outputDirectory) - } - } - // Report warnings about source files that have no output directory. - chunks.get(None) foreach { srcs => - log.error("No output directory mapped for: " + srcs.map(_.getAbsolutePath).mkString(",")) - } - // Here we try to memoize (cache) the known class files in the output directory. - val memo = for ((Some(outputDirectory), srcs) <- chunks) yield { - val classesFinder = PathFinder(outputDirectory) ** "*.class" - (classesFinder, classesFinder.get, srcs) - } - // Here we construct a class-loader we'll use to load + analyze the - val loader = ClasspathUtilities.toLoader(searchClasspath) - timed("Java compilation", log) { - try javac.compileWithReporter(javaSrcs.toArray, absClasspath.toArray, output, options.javacOptions.toArray, reporter, log) - catch { - // Handle older APIs - case _: NoSuchMethodError => - javac.compile(javaSrcs.toArray, absClasspath.toArray, output, options.javacOptions.toArray, log) - } - } - /** Reads the API information directly from the Class[_] object. Used when Analyzing dependencies. */ - def readAPI(source: File, classes: Seq[Class[_]]): Set[String] = { - val (api, inherits) = ClassToAPI.process(classes) - callback.api(source, api) - inherits.map(_.getName) - } - // Runs the analysis portion of Javac. - timed("Java analysis", log) { - for ((classesFinder, oldClasses, srcs) <- memo) { - val newClasses = Set(classesFinder.get: _*) -- oldClasses - Analyze(newClasses.toSeq, srcs, log)(callback, loader, readAPI) - } - } - } - // TODO - Maybe on "Mixed" we should try to compile both Scala + Java. - if (order == JavaThenScala) { compileJava(); compileScala() } else { compileScala(); compileJava() } - } + // Construct a compiler which can handle both java and scala sources. + val mixedCompiler = + new MixedAnalyzingCompiler( + compiler, + // TODO - Construction of analyzing Java compiler MAYBE should be earlier... + new AnalyzingJavaCompiler(javac, classpath, compiler.scalaInstance, entry, searchClasspath), + config, + log + ) // Here we check to see if any previous compilation results are invalid, based on altered // javac/scalac options. @@ -212,29 +217,10 @@ object MixedAnalyzingCompiler { case Some(previous) if equiv.equiv(previous, currentSetup) => previousAnalysis case _ => Incremental.prune(sourcesSet, previousAnalysis) } - // Run the incremental compiler using the large compile0 "compile step" we've defined. - IncrementalCompile(sourcesSet, entry, compile0, analysis, getAnalysis, output, log, incOptions) + // Run the incremental compiler using the mixed compiler we've defined. + IncrementalCompile(sourcesSet, entry, mixedCompiler.compile, analysis, getAnalysis, output, log, incOptions) } - private[this] def outputDirectories(output: Output): Seq[File] = output match { - case single: SingleOutput => List(single.outputDirectory) - case mult: MultipleOutput => mult.outputGroups map (_.outputDirectory) - } - /** Debugging method to time how long it takes to run various compilation tasks. */ - private[this] def timed[T](label: String, log: Logger)(t: => T): T = - { - val start = System.nanoTime - val result = t - val elapsed = System.nanoTime - start - log.debug(label + " took " + (elapsed / 1e9) + " s") - result - } - private[this] def logInputs(log: Logger, javaCount: Int, scalaCount: Int, outputDirs: Seq[File]) { - val scalaMsg = Analysis.counted("Scala source", "", "s", scalaCount) - val javaMsg = Analysis.counted("Java source", "", "s", javaCount) - val combined = scalaMsg ++ javaMsg - if (!combined.isEmpty) - log.info(combined.mkString("Compiling ", " and ", " to " + outputDirs.map(_.getAbsolutePath).mkString(",") + "...")) - } + /** Returns true if the file is java. */ def javaOnly(f: File) = f.getName.endsWith(".java") diff --git a/compile/src/main/scala/sbt/compiler/javac/AnalyzingJavaCompiler.scala b/compile/src/main/scala/sbt/compiler/javac/AnalyzingJavaCompiler.scala new file mode 100644 index 000000000..4bcce6e7c --- /dev/null +++ b/compile/src/main/scala/sbt/compiler/javac/AnalyzingJavaCompiler.scala @@ -0,0 +1,108 @@ +package sbt.compiler.javac + +import java.io.File + +import sbt.classfile.Analyze +import sbt.classpath.ClasspathUtilities +import sbt.compiler.CompilerArguments +import sbt._ +import xsbti.api.Source +import xsbti.{ Reporter, AnalysisCallback } +import xsbti.compile._ + +object AnalyzingJavaCompiler { + type ClasspathLookup = String => Option[File] +} + +/** + * This is a java compiler which will also report any discovered source dependencies/apis out via + * an analysis callback. + * + * @param searchClasspath Differes from classpath in that we look up binary dependencies via this classpath. + * @param classLookup A mechanism by which we can figure out if a JAR contains a classfile. + */ +final class AnalyzingJavaCompiler private[sbt] ( + val javac: xsbti.compile.JavaCompiler, + val classpath: Seq[File], + val scalaInstance: xsbti.compile.ScalaInstance, + val classLookup: AnalyzingJavaCompiler.ClasspathLookup, + val searchClasspath: Seq[File]) { + + /** + * Compile some java code using the current configured compiler. + * + * @param sources The sources to compile + * @param options The options for the Java compiler + * @param output The output configuration for this compiler + * @param callback A callback to report discovered source/binary dependencies on. + * @param reporter A reporter where semantic compiler failures can be reported. + * @param log A place where we can log debugging/error messages. + * @param progressOpt An optional compilation progress reporter. Where we can report back what files we're currently compiling. + */ + def compile(sources: Seq[File], options: Seq[String], output: Output, callback: AnalysisCallback, reporter: Reporter, log: Logger, progressOpt: Option[CompileProgress]): Unit = { + if (!sources.isEmpty) { + val absClasspath = classpath.map(_.getAbsoluteFile) + import Path._ + @annotation.tailrec def ancestor(f1: File, f2: File): Boolean = + if (f2 eq null) false else if (f1 == f2) true else ancestor(f1, f2.getParentFile) + // Here we outline "chunks" of compiles we need to run so that the .class files end up in the right + // location for Java. + val chunks: Map[Option[File], Seq[File]] = output match { + case single: SingleOutput => Map(Some(single.outputDirectory) -> sources) + case multi: MultipleOutput => + sources groupBy { src => + multi.outputGroups find { out => ancestor(out.sourceDirectory, src) } map (_.outputDirectory) + } + } + // Report warnings about source files that have no output directory. + chunks.get(None) foreach { srcs => + log.error("No output directory mapped for: " + srcs.map(_.getAbsolutePath).mkString(",")) + } + // Here we try to memoize (cache) the known class files in the output directory. + val memo = for ((Some(outputDirectory), srcs) <- chunks) yield { + val classesFinder = PathFinder(outputDirectory) ** "*.class" + (classesFinder, classesFinder.get, srcs) + } + // Here we construct a class-loader we'll use to load + analyze the + val loader = ClasspathUtilities.toLoader(searchClasspath) + // TODO - Perhaps we just record task 0/2 here + timed("Java compilation", log) { + try javac.compileWithReporter(sources.toArray, absClasspath.toArray, output, options.toArray, reporter, log) + catch { + // Handle older APIs + case _: NoSuchMethodError => + javac.compile(sources.toArray, absClasspath.toArray, output, options.toArray, log) + } + } + // TODO - Perhaps we just record task 1/2 here + + /** Reads the API information directly from the Class[_] object. Used when Analyzing dependencies. */ + def readAPI(source: File, classes: Seq[Class[_]]): Set[String] = { + val (api, inherits) = ClassToAPI.process(classes) + callback.api(source, api) + inherits.map(_.getName) + } + // Runs the analysis portion of Javac. + timed("Java analysis", log) { + for ((classesFinder, oldClasses, srcs) <- memo) { + val newClasses = Set(classesFinder.get: _*) -- oldClasses + Analyze(newClasses.toSeq, srcs, log)(callback, loader, readAPI) + } + } + // TODO - Perhaps we just record task 2/2 here + } + } + + // TODO - This code is duplciated later in MixedAnalyzing compiler. It should probably just live in this class. + private[this] def explicitBootClasspath(options: Seq[String]): Seq[File] = options.dropWhile(_ != CompilerArguments.BootClasspathOption).drop(1).take(1).headOption.toList.flatMap(IO.parseClasspath) + private[this] def withBootclasspath(args: CompilerArguments, classpath: Seq[File]): Seq[File] = + args.bootClasspathFor(classpath) ++ args.extClasspath ++ args.finishClasspath(classpath) + /** Debugging method to time how long it takes to run various compilation tasks. */ + private[this] def timed[T](label: String, log: Logger)(t: => T): T = { + val start = System.nanoTime + val result = t + val elapsed = System.nanoTime - start + log.debug(label + " took " + (elapsed / 1e9) + " s") + result + } +} diff --git a/interface/src/main/java/xsbti/compile/CompileProgress.java b/interface/src/main/java/xsbti/compile/CompileProgress.java index 902a50018..17174ff6a 100755 --- a/interface/src/main/java/xsbti/compile/CompileProgress.java +++ b/interface/src/main/java/xsbti/compile/CompileProgress.java @@ -1,5 +1,10 @@ package xsbti.compile; +/** + * An API for reporting when files are being compiled. + * + * Note; This is tied VERY SPECIFICALLY to scala. + */ public interface CompileProgress { void startUnit(String phase, String unitPath); diff --git a/interface/src/main/java/xsbti/compile/IncrementalCompiler.java b/interface/src/main/java/xsbti/compile/IncrementalCompiler.java index f2323111d..c98263e7f 100644 --- a/interface/src/main/java/xsbti/compile/IncrementalCompiler.java +++ b/interface/src/main/java/xsbti/compile/IncrementalCompiler.java @@ -44,8 +44,18 @@ public interface IncrementalCompiler * @param instance The Scala version to use * @param interfaceJar The compiler interface jar compiled for the Scala version being used * @param options Configures how arguments to the underlying Scala compiler will be built. + * */ + @Deprecated ScalaCompiler newScalaCompiler(ScalaInstance instance, File interfaceJar, ClasspathOptions options, Logger log); + /** + * Creates a compiler instance that can be used by the `compile` method. + * + * @param instance The Scala version to use + * @param interfaceJar The compiler interface jar compiled for the Scala version being used + * @param options Configures how arguments to the underlying Scala compiler will be built. + */ + ScalaCompiler newScalaCompiler(ScalaInstance instance, File interfaceJar, ClasspathOptions options); /** * Compiles the source interface for a Scala version. The resulting jar can then be used by the `newScalaCompiler` method diff --git a/project/Sbt.scala b/project/Sbt.scala index d5c1f0539..77712268a 100644 --- a/project/Sbt.scala +++ b/project/Sbt.scala @@ -156,8 +156,8 @@ object Sbt extends Build { // Persists the incremental data structures using SBinary lazy val compilePersistSub = testedBaseProject(compilePath / "persist", "Persist") dependsOn (compileIncrementalSub, apiSub, compileIncrementalSub % "test->test") settings (sbinary) // sbt-side interface to compiler. Calls compiler-side interface reflectively - lazy val compilerSub = testedBaseProject(compilePath, "Compile") dependsOn (launchInterfaceSub, interfaceSub % "compile;test->test", logSub, ioSub, classpathSub, - logSub % "test->test", launchSub % "test->test", apiSub % "test") settings (compilerSettings: _*) + lazy val compilerSub = testedBaseProject(compilePath, "Compile") dependsOn (launchInterfaceSub, interfaceSub % "compile;test->test", logSub, ioSub, classpathSub, apiSub, classfileSub, + logSub % "test->test", launchSub % "test->test") settings (compilerSettings: _*) lazy val compilerIntegrationSub = baseProject(compilePath / "integration", "Compiler Integration") dependsOn ( compileIncrementalSub, compilerSub, compilePersistSub, apiSub, classfileSub) lazy val compilerIvySub = baseProject(compilePath / "ivy", "Compiler Ivy Integration") dependsOn (ivySub, compilerSub)