diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index d24010467..940a0673e 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -371,6 +371,7 @@ object Defaults extends BuildCommon Seq(ScalaCheck, Specs2, Specs, ScalaTest, JUnit) }, testListeners :== Nil, + testReportJUnitXml :== false, testOptions :== Nil, testResultLogger :== TestResultLogger.Default, testFilter in testOnly :== (selectedFilter _) @@ -389,7 +390,8 @@ object Defaults extends BuildCommon trl.run(streams.value.log, executeTests.value, taskName) }, testOnly <<= inputTests(testOnly), - testQuick <<= inputTests(testQuick) + testQuick <<= inputTests(testQuick), + testListeners ++= (if( testReportJUnitXml.value ) Seq(new JUnitXmlTestsListener(target.value.getAbsolutePath)) else Nil) ) lazy val TaskGlobal: Scope = ThisScope.copy(task = Global) lazy val ConfigGlobal: Scope = ThisScope.copy(config = Global) diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index 3ea64983b..0758ce5b7 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -195,6 +195,7 @@ object Keys val testFrameworks = SettingKey[Seq[TestFramework]]("test-frameworks", "Registered, although not necessarily present, test frameworks.", CTask) val testListeners = TaskKey[Seq[TestReportListener]]("test-listeners", "Defines test listeners.", DTask) val testForkedParallel = SettingKey[Boolean]("test-forked-parallel", "Whether forked tests should be executed in parallel", CTask) + val testReportJUnitXml = SettingKey[Boolean]("test-report-junit-xml", "Produce JUnit XML test reports", BPlusTask) val testExecution = TaskKey[Tests.Execution]("test-execution", "Settings controlling test execution", DTask) val testFilter = TaskKey[Seq[String] => Seq[String => Boolean]]("test-filter", "Filter controlling whether the test is executed", DTask) val testResultLogger = SettingKey[TestResultLogger]("test-result-logger", "Logs results after a test task completes.", DTask) diff --git a/sbt/src/sbt-test/tests/junit-xml-report/project/JUnitXmlReportTest.scala b/sbt/src/sbt-test/tests/junit-xml-report/project/JUnitXmlReportTest.scala new file mode 100644 index 000000000..885be62a3 --- /dev/null +++ b/sbt/src/sbt-test/tests/junit-xml-report/project/JUnitXmlReportTest.scala @@ -0,0 +1,43 @@ +import sbt._ +import Keys._ +import scala.xml.XML +import Tests._ +import Defaults._ + +object JUnitXmlReportTest extends Build { + val checkReport = taskKey[Unit]("Check the test reports") + val checkNoReport = taskKey[Unit]("Check that no reports are present") + + private val oneSecondReportFile = "target/test-reports/a.pkg.OneSecondTest.xml" + private val failingReportFile = "target/test-reports/another.pkg.FailingTest.xml" + + lazy val root = Project("root", file("."), settings = defaultSettings ++ Seq( + scalaVersion := "2.9.2", + libraryDependencies += "com.novocode" % "junit-interface" % "0.10" % "test", + + testReportJUnitXml := true, + + // TODO use matchers instead of sys.error + checkReport := { + val oneSecondReport = XML.loadFile(oneSecondReportFile) + if( oneSecondReport.label != "testsuite" ) sys.error("Report should have a root element.") + // somehow the 'success' event doesn't go through... TODO investigate +// if( (oneSecondReport \ "@time").text.toFloat < 1f ) sys.error("expected test to take at least 1 sec") + if( (oneSecondReport \ "@name").text != "a.pkg.OneSecondTest" ) sys.error("wrong test name: " + (oneSecondReport \ "@name").text) + // TODO more checks + + val failingReport = XML.loadFile(failingReportFile) + if( failingReport.label != "testsuite" ) sys.error("Report should have a root element.") + if( (failingReport \ "@failures").text != "2" ) sys.error("expected 2 failures") + if( (failingReport \ "@name").text != "another.pkg.FailingTest" ) sys.error("wrong test name: " + (failingReport \ "@name").text) + // TODO more checks -> the two test cases with time etc.. + + // TODO check console output is in the report + }, + + checkNoReport := { + if( file(oneSecondReportFile).exists() ) sys.error(oneSecondReportFile + " should not exist") + if( file(failingReportFile).exists() ) sys.error(failingReportFile + " should not exist") + } + )) +} \ No newline at end of file diff --git a/sbt/src/sbt-test/tests/junit-xml-report/src/test/scala/tests.scala b/sbt/src/sbt-test/tests/junit-xml-report/src/test/scala/tests.scala new file mode 100644 index 000000000..38c636456 --- /dev/null +++ b/sbt/src/sbt-test/tests/junit-xml-report/src/test/scala/tests.scala @@ -0,0 +1,49 @@ +import org.junit.Test + +package a.pkg { + class OneSecondTest { + @Test + def oneSecond() { + Thread.sleep(1000) + } + } +} + +package another.pkg { + class FailingTest { + @Test + def failure1_OneSecond() { + Thread.sleep(1000) + sys.error("fail1") + } + + @Test + def failure2_HalfSecond() { + Thread.sleep(500) + sys.error("fail2") + } + } +} + +package console.test.pkg { + // we won't check console output in the report + // until SBT supports that + class ConsoleTests { + @Test + def sayHello() { + println("Hello") + System.out.println("World!") + } + + @Test + def multiThreadedHello() { + for( i <- 1 to 5 ) { + new Thread("t-" + i) { + override def run() { + println("Hello from thread " + i) + } + }.start() + } + } + } +} \ No newline at end of file diff --git a/sbt/src/sbt-test/tests/junit-xml-report/test b/sbt/src/sbt-test/tests/junit-xml-report/test new file mode 100644 index 000000000..f76a142c9 --- /dev/null +++ b/sbt/src/sbt-test/tests/junit-xml-report/test @@ -0,0 +1,11 @@ +-> test +> checkReport + +# there might be discrepancies between the 'normal' and the 'forked' mode + +> clean +> checkNoReport + +> set fork in Test := true +-> test +> checkReport \ No newline at end of file diff --git a/testing/src/main/scala/sbt/JUnitXmlTestsListener.scala b/testing/src/main/scala/sbt/JUnitXmlTestsListener.scala new file mode 100644 index 000000000..f3a0bdae6 --- /dev/null +++ b/testing/src/main/scala/sbt/JUnitXmlTestsListener.scala @@ -0,0 +1,168 @@ +package sbt + +import java.io.{StringWriter, PrintWriter, File} +import java.net.InetAddress +import scala.collection.mutable.ListBuffer +import scala.util.DynamicVariable +import scala.xml.{Elem, Node, XML} +import testing.{Event => TEvent, Status => TStatus, OptionalThrowable, TestSelector} + +/** + * A tests listener that outputs the results it receives in junit xml + * report format. + * @param outputDir path to the dir in which a folder with results is generated + */ +class JUnitXmlTestsListener(val outputDir:String) extends TestsListener +{ + /**Current hostname so we know which machine executed the tests*/ + val hostname = InetAddress.getLocalHost.getHostName + /**The dir in which we put all result files. Is equal to the given dir + "/test-reports"*/ + val targetDir = new File(outputDir + "/test-reports/") + + /**all system properties as XML*/ + val properties = + { + val iter = System.getProperties.entrySet.iterator + val props:ListBuffer[Node] = new ListBuffer() + while (iter.hasNext) { + val next = iter.next + props += + } + props + } + + + /** Gathers data for one Test Suite. We map test groups to TestSuites. + * Each TestSuite gets its own output file. + */ + class TestSuite(val name:String) { + val events:ListBuffer[TEvent] = new ListBuffer() + + /**Adds one test result to this suite.*/ + def addEvent(e:TEvent) = events += e + + /** Returns the number of tests of each state for the specified. */ + def count(status: TStatus) = events.count(_.status == status) + + /** Stops the time measuring and emits the XML for + * All tests collected so far. + */ + def stop():Elem = { + val duration = events.map(_.duration()).sum + + val (errors, failures, tests) = (count(TStatus.Error), count(TStatus.Failure), events.size) + + val result = + {properties} + { + for (e <- events) yield + selector.testName.split('.').last + case _ => "(It is not a test)" + } + } + time={(e.duration() / 1000.0).toString}> { + var trace:String = if (e.throwable.isDefined) { + val stringWriter = new StringWriter() + val writer = new PrintWriter(stringWriter) + e.throwable.get.printStackTrace(writer) + writer.flush() + stringWriter.toString + } + else { + "" + } + e.status match { + case TStatus.Error if (e.throwable.isDefined) => {trace} + case TStatus.Error => + case TStatus.Failure if (e.throwable.isDefined) => {trace} + case TStatus.Failure => + case TStatus.Skipped => + case _ => {} + } + } + + + } + + + + + result + } + } + + /**The currently running test suite*/ + var testSuite = new DynamicVariable(null: TestSuite) + + /**Creates the output Dir*/ + override def doInit() = {targetDir.mkdirs()} + + /** Starts a new, initially empty Suite with the given name. + */ + override def startGroup(name: String) {testSuite.value_=(new TestSuite(name))} + + /** Adds all details for the given even to the current suite. + */ + override def testEvent(event: TestEvent): Unit = for (e <- event.detail) {testSuite.value.addEvent(e)} + + /** called for each class or equivalent grouping + * We map one group to one Testsuite, so for each Group + * we create an XML like this: + * + * + * + * + * ... + * + * + * ... stack ... + * + * + * + * ...stack... + * + * + * + * + */ + override def endGroup(name: String, t: Throwable) = { + // create our own event to record the error + val event = new TEvent { + def fullyQualifiedName= name + //def description = + //"Throwable escaped the test run of '%s'".format(name) + def duration = -1 + def status = TStatus.Error + def fingerprint = null + def selector = null + def throwable = new OptionalThrowable(t) + } + testSuite.value.addEvent(event) + writeSuite() + } + + /** Ends the current suite, wraps up the result and writes it to an XML file + * in the output folder that is named after the suite. + */ + override def endGroup(name: String, result: TestResult.Value) = { + writeSuite() + } + + private def writeSuite() = { + val file = new File(targetDir, testSuite.value.name + ".xml").getAbsolutePath + // TODO would be nice to have a logger and log this with level debug + // System.err.println("Writing JUnit XML test report: " + file) + XML.save (file, testSuite.value.stop(), "UTF-8", true, null) + } + + /**Does nothing, as we write each file after a suite is done.*/ + override def doComplete(finalResult: TestResult.Value): Unit = {} + + /**Returns None*/ + override def contentLogger(test: TestDefinition): Option[ContentLogger] = None +}