mirror of https://github.com/sbt/sbt.git
Added a test reporter for JUnit XML
Taken from https://github.com/chenkelmann/junit_xml_listener and slightly improved. Activating is now just a matter of setting `testReportJUnitXml` to true. This not only brings a very handy feature (hudson/jenkins support that syntax), but gives more options for testing the SBT test infrastructure. For instance checking that the test output is well segregated during a parallel test run is just a matter of checking the report output. Conflicts: main/src/main/scala/sbt/Keys.scala
This commit is contained in:
Bruno Bieth
Josh Suereth
parent
3aa70b914a
commit
b38b86add5
|
|
@ -371,6 +371,7 @@ object Defaults extends BuildCommon
|
|||
Seq(ScalaCheck, Specs2, Specs, ScalaTest, JUnit)
|
||||
},
|
||||
testListeners :== Nil,
|
||||
testReportJUnitXml :== false,
|
||||
testOptions :== Nil,
|
||||
testResultLogger :== TestResultLogger.Default,
|
||||
testFilter in testOnly :== (selectedFilter _)
|
||||
|
|
@ -389,7 +390,8 @@ object Defaults extends BuildCommon
|
|||
trl.run(streams.value.log, executeTests.value, taskName)
|
||||
},
|
||||
testOnly <<= inputTests(testOnly),
|
||||
testQuick <<= inputTests(testQuick)
|
||||
testQuick <<= inputTests(testQuick),
|
||||
testListeners ++= (if( testReportJUnitXml.value ) Seq(new JUnitXmlTestsListener(target.value.getAbsolutePath)) else Nil)
|
||||
)
|
||||
lazy val TaskGlobal: Scope = ThisScope.copy(task = Global)
|
||||
lazy val ConfigGlobal: Scope = ThisScope.copy(config = Global)
|
||||
|
|
|
|||
|
|
@ -195,6 +195,7 @@ object Keys
|
|||
val testFrameworks = SettingKey[Seq[TestFramework]]("test-frameworks", "Registered, although not necessarily present, test frameworks.", CTask)
|
||||
val testListeners = TaskKey[Seq[TestReportListener]]("test-listeners", "Defines test listeners.", DTask)
|
||||
val testForkedParallel = SettingKey[Boolean]("test-forked-parallel", "Whether forked tests should be executed in parallel", CTask)
|
||||
val testReportJUnitXml = SettingKey[Boolean]("test-report-junit-xml", "Produce JUnit XML test reports", BPlusTask)
|
||||
val testExecution = TaskKey[Tests.Execution]("test-execution", "Settings controlling test execution", DTask)
|
||||
val testFilter = TaskKey[Seq[String] => Seq[String => Boolean]]("test-filter", "Filter controlling whether the test is executed", DTask)
|
||||
val testResultLogger = SettingKey[TestResultLogger]("test-result-logger", "Logs results after a test task completes.", DTask)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,43 @@
|
|||
import sbt._
|
||||
import Keys._
|
||||
import scala.xml.XML
|
||||
import Tests._
|
||||
import Defaults._
|
||||
|
||||
object JUnitXmlReportTest extends Build {
|
||||
val checkReport = taskKey[Unit]("Check the test reports")
|
||||
val checkNoReport = taskKey[Unit]("Check that no reports are present")
|
||||
|
||||
private val oneSecondReportFile = "target/test-reports/a.pkg.OneSecondTest.xml"
|
||||
private val failingReportFile = "target/test-reports/another.pkg.FailingTest.xml"
|
||||
|
||||
lazy val root = Project("root", file("."), settings = defaultSettings ++ Seq(
|
||||
scalaVersion := "2.9.2",
|
||||
libraryDependencies += "com.novocode" % "junit-interface" % "0.10" % "test",
|
||||
|
||||
testReportJUnitXml := true,
|
||||
|
||||
// TODO use matchers instead of sys.error
|
||||
checkReport := {
|
||||
val oneSecondReport = XML.loadFile(oneSecondReportFile)
|
||||
if( oneSecondReport.label != "testsuite" ) sys.error("Report should have a root <testsuite> element.")
|
||||
// somehow the 'success' event doesn't go through... TODO investigate
|
||||
// if( (oneSecondReport \ "@time").text.toFloat < 1f ) sys.error("expected test to take at least 1 sec")
|
||||
if( (oneSecondReport \ "@name").text != "a.pkg.OneSecondTest" ) sys.error("wrong test name: " + (oneSecondReport \ "@name").text)
|
||||
// TODO more checks
|
||||
|
||||
val failingReport = XML.loadFile(failingReportFile)
|
||||
if( failingReport.label != "testsuite" ) sys.error("Report should have a root <testsuite> element.")
|
||||
if( (failingReport \ "@failures").text != "2" ) sys.error("expected 2 failures")
|
||||
if( (failingReport \ "@name").text != "another.pkg.FailingTest" ) sys.error("wrong test name: " + (failingReport \ "@name").text)
|
||||
// TODO more checks -> the two test cases with time etc..
|
||||
|
||||
// TODO check console output is in the report
|
||||
},
|
||||
|
||||
checkNoReport := {
|
||||
if( file(oneSecondReportFile).exists() ) sys.error(oneSecondReportFile + " should not exist")
|
||||
if( file(failingReportFile).exists() ) sys.error(failingReportFile + " should not exist")
|
||||
}
|
||||
))
|
||||
}
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
import org.junit.Test
|
||||
|
||||
package a.pkg {
|
||||
class OneSecondTest {
|
||||
@Test
|
||||
def oneSecond() {
|
||||
Thread.sleep(1000)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
package another.pkg {
|
||||
class FailingTest {
|
||||
@Test
|
||||
def failure1_OneSecond() {
|
||||
Thread.sleep(1000)
|
||||
sys.error("fail1")
|
||||
}
|
||||
|
||||
@Test
|
||||
def failure2_HalfSecond() {
|
||||
Thread.sleep(500)
|
||||
sys.error("fail2")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
package console.test.pkg {
|
||||
// we won't check console output in the report
|
||||
// until SBT supports that
|
||||
class ConsoleTests {
|
||||
@Test
|
||||
def sayHello() {
|
||||
println("Hello")
|
||||
System.out.println("World!")
|
||||
}
|
||||
|
||||
@Test
|
||||
def multiThreadedHello() {
|
||||
for( i <- 1 to 5 ) {
|
||||
new Thread("t-" + i) {
|
||||
override def run() {
|
||||
println("Hello from thread " + i)
|
||||
}
|
||||
}.start()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
-> test
|
||||
> checkReport
|
||||
|
||||
# there might be discrepancies between the 'normal' and the 'forked' mode
|
||||
|
||||
> clean
|
||||
> checkNoReport
|
||||
|
||||
> set fork in Test := true
|
||||
-> test
|
||||
> checkReport
|
||||
|
|
@ -0,0 +1,168 @@
|
|||
package sbt
|
||||
|
||||
import java.io.{StringWriter, PrintWriter, File}
|
||||
import java.net.InetAddress
|
||||
import scala.collection.mutable.ListBuffer
|
||||
import scala.util.DynamicVariable
|
||||
import scala.xml.{Elem, Node, XML}
|
||||
import testing.{Event => TEvent, Status => TStatus, OptionalThrowable, TestSelector}
|
||||
|
||||
/**
|
||||
* A tests listener that outputs the results it receives in junit xml
|
||||
* report format.
|
||||
* @param outputDir path to the dir in which a folder with results is generated
|
||||
*/
|
||||
class JUnitXmlTestsListener(val outputDir:String) extends TestsListener
|
||||
{
|
||||
/**Current hostname so we know which machine executed the tests*/
|
||||
val hostname = InetAddress.getLocalHost.getHostName
|
||||
/**The dir in which we put all result files. Is equal to the given dir + "/test-reports"*/
|
||||
val targetDir = new File(outputDir + "/test-reports/")
|
||||
|
||||
/**all system properties as XML*/
|
||||
val properties =
|
||||
<properties> {
|
||||
val iter = System.getProperties.entrySet.iterator
|
||||
val props:ListBuffer[Node] = new ListBuffer()
|
||||
while (iter.hasNext) {
|
||||
val next = iter.next
|
||||
props += <property name={next.getKey.toString} value={next.getValue.toString} />
|
||||
}
|
||||
props
|
||||
}
|
||||
</properties>
|
||||
|
||||
/** Gathers data for one Test Suite. We map test groups to TestSuites.
|
||||
* Each TestSuite gets its own output file.
|
||||
*/
|
||||
class TestSuite(val name:String) {
|
||||
val events:ListBuffer[TEvent] = new ListBuffer()
|
||||
|
||||
/**Adds one test result to this suite.*/
|
||||
def addEvent(e:TEvent) = events += e
|
||||
|
||||
/** Returns the number of tests of each state for the specified. */
|
||||
def count(status: TStatus) = events.count(_.status == status)
|
||||
|
||||
/** Stops the time measuring and emits the XML for
|
||||
* All tests collected so far.
|
||||
*/
|
||||
def stop():Elem = {
|
||||
val duration = events.map(_.duration()).sum
|
||||
|
||||
val (errors, failures, tests) = (count(TStatus.Error), count(TStatus.Failure), events.size)
|
||||
|
||||
val result = <testsuite hostname={hostname} name={name}
|
||||
tests={tests + ""} errors={errors + ""} failures={failures + ""}
|
||||
time={(duration/1000.0).toString} >
|
||||
{properties}
|
||||
{
|
||||
for (e <- events) yield
|
||||
<testcase classname={name}
|
||||
name={
|
||||
e.selector match {
|
||||
case selector: TestSelector => selector.testName.split('.').last
|
||||
case _ => "(It is not a test)"
|
||||
}
|
||||
}
|
||||
time={(e.duration() / 1000.0).toString}> {
|
||||
var trace:String = if (e.throwable.isDefined) {
|
||||
val stringWriter = new StringWriter()
|
||||
val writer = new PrintWriter(stringWriter)
|
||||
e.throwable.get.printStackTrace(writer)
|
||||
writer.flush()
|
||||
stringWriter.toString
|
||||
}
|
||||
else {
|
||||
""
|
||||
}
|
||||
e.status match {
|
||||
case TStatus.Error if (e.throwable.isDefined) => <error message={e.throwable.get.getMessage} type={e.throwable.get.getClass.getName}>{trace}</error>
|
||||
case TStatus.Error => <error message={"No Exception or message provided"} />
|
||||
case TStatus.Failure if (e.throwable.isDefined) => <failure message={e.throwable.get.getMessage} type={e.throwable.get.getClass.getName}>{trace}</failure>
|
||||
case TStatus.Failure => <failure message={"No Exception or message provided"} />
|
||||
case TStatus.Skipped => <skipped />
|
||||
case _ => {}
|
||||
}
|
||||
}
|
||||
</testcase>
|
||||
|
||||
}
|
||||
<system-out><![CDATA[]]></system-out>
|
||||
<system-err><![CDATA[]]></system-err>
|
||||
</testsuite>
|
||||
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
/**The currently running test suite*/
|
||||
var testSuite = new DynamicVariable(null: TestSuite)
|
||||
|
||||
/**Creates the output Dir*/
|
||||
override def doInit() = {targetDir.mkdirs()}
|
||||
|
||||
/** Starts a new, initially empty Suite with the given name.
|
||||
*/
|
||||
override def startGroup(name: String) {testSuite.value_=(new TestSuite(name))}
|
||||
|
||||
/** Adds all details for the given even to the current suite.
|
||||
*/
|
||||
override def testEvent(event: TestEvent): Unit = for (e <- event.detail) {testSuite.value.addEvent(e)}
|
||||
|
||||
/** called for each class or equivalent grouping
|
||||
* We map one group to one Testsuite, so for each Group
|
||||
* we create an XML like this:
|
||||
* <?xml version="1.0" encoding="UTF-8" ?>
|
||||
* <testsuite errors="x" failures="y" tests="z" hostname="example.com" name="eu.henkelmann.bla.SomeTest" time="0.23">
|
||||
* <properties>
|
||||
* <property name="os.name" value="Linux" />
|
||||
* ...
|
||||
* </properties>
|
||||
* <testcase classname="eu.henkelmann.bla.SomeTest" name="testFooWorks" time="0.0" >
|
||||
* <error message="the foo did not work" type="java.lang.NullPointerException">... stack ...</error>
|
||||
* </testcase>
|
||||
* <testcase classname="eu.henkelmann.bla.SomeTest" name="testBarThrowsException" time="0.0" />
|
||||
* <testcase classname="eu.henkelmann.bla.SomeTest" name="testBaz" time="0.0">
|
||||
* <failure message="the baz was no bar" type="junit.framework.AssertionFailedError">...stack...</failure>
|
||||
* </testcase>
|
||||
* <system-out><![CDATA[]]></system-out>
|
||||
* <system-err><![CDATA[]]></system-err>
|
||||
* </testsuite>
|
||||
*/
|
||||
override def endGroup(name: String, t: Throwable) = {
|
||||
// create our own event to record the error
|
||||
val event = new TEvent {
|
||||
def fullyQualifiedName= name
|
||||
//def description =
|
||||
//"Throwable escaped the test run of '%s'".format(name)
|
||||
def duration = -1
|
||||
def status = TStatus.Error
|
||||
def fingerprint = null
|
||||
def selector = null
|
||||
def throwable = new OptionalThrowable(t)
|
||||
}
|
||||
testSuite.value.addEvent(event)
|
||||
writeSuite()
|
||||
}
|
||||
|
||||
/** Ends the current suite, wraps up the result and writes it to an XML file
|
||||
* in the output folder that is named after the suite.
|
||||
*/
|
||||
override def endGroup(name: String, result: TestResult.Value) = {
|
||||
writeSuite()
|
||||
}
|
||||
|
||||
private def writeSuite() = {
|
||||
val file = new File(targetDir, testSuite.value.name + ".xml").getAbsolutePath
|
||||
// TODO would be nice to have a logger and log this with level debug
|
||||
// System.err.println("Writing JUnit XML test report: " + file)
|
||||
XML.save (file, testSuite.value.stop(), "UTF-8", true, null)
|
||||
}
|
||||
|
||||
/**Does nothing, as we write each file after a suite is done.*/
|
||||
override def doComplete(finalResult: TestResult.Value): Unit = {}
|
||||
|
||||
/**Returns None*/
|
||||
override def contentLogger(test: TestDefinition): Option[ContentLogger] = None
|
||||
}
|
||||
Loading…
Reference in New Issue