When serially executing tests*, order tests by testOnly filter order

* parallelExecution in test := false

cc @bantonsson

main/actions/src/main/scala/sbt/Tests.scala

@@ -30,6 +30,8 @@ object Tests
 	final case class Exclude(tests: Iterable[String]) extends TestOption
 	final case class Listeners(listeners: Iterable[TestReportListener]) extends TestOption
 	final case class Filter(filterTest: String => Boolean) extends TestOption
+	/** Test execution will be ordered by the position of the matching filter. */
+	final case class Filters(filterTest: Seq[String => Boolean]) extends TestOption
 
 	// args for all frameworks
 	def Argument(args: String*): Argument = Argument(None, args.toList)
@@ -45,6 +47,7 @@ object Tests
 	{
 			import collection.mutable.{HashSet, ListBuffer, Map, Set}
 		val testFilters = new ListBuffer[String => Boolean]
+		var orderedFilters = Seq[String => Boolean]()
 		val excludeTestsSet = new HashSet[String]
 		val setup, cleanup = new ListBuffer[ClassLoader => Unit]
 		val testListeners = new ListBuffer[TestReportListener]
@@ -63,6 +66,7 @@ object Tests
 			option match
 			{
 				case Filter(include) => testFilters += include
+				case Filters(includes) => if(!orderedFilters.isEmpty) error("Cannot define multiple ordered test filters.") else orderedFilters = includes
 				case Exclude(exclude) => excludeTestsSet ++= exclude
 				case Listeners(listeners) => testListeners ++= listeners
 				case Setup(setupFunction) => setup += setupFunction
@@ -87,7 +91,8 @@ object Tests
 			log.warn("Arguments defined for test frameworks that are not present:\n\t" + undefinedFrameworks.mkString("\n\t"))
 
 		def includeTest(test: TestDefinition) = !excludeTestsSet.contains(test.name) && testFilters.forall(filter => filter(test.name))
-		val tests = discovered.filter(includeTest).toSet.toSeq
+		val filtered0 = discovered.filter(includeTest).distinct
+		val tests = if(orderedFilters.isEmpty) filtered0 else orderedFilters.flatMap(f => filtered0.filter(d => f(d.name))).distinct
 		val arguments = testArgsByFramework.map { case (k,v) => (k, v.toList) } toMap;
 		testTask(frameworks.values.toSeq, testLoader, tests, setup.readOnly, cleanup.readOnly, log, testListeners.readOnly, arguments, config)
 	}
@@ -117,7 +122,7 @@ object Tests
 	type TestRunnable = (String, () => TestResult.Value)
 	def makeParallel(runnables: Iterable[TestRunnable], setupTasks: Task[Unit], tags: Seq[(Tag,Int)]) =
 		runnables map { case (name, test) => task { (name, test()) } dependsOn setupTasks named name tagw(tags : _*) }
-	def makeSerial(runnables: Iterable[TestRunnable], setupTasks: Task[Unit], tags: Seq[(Tag,Int)]) =
+	def makeSerial(runnables: Seq[TestRunnable], setupTasks: Task[Unit], tags: Seq[(Tag,Int)]) =
 		task { runnables map { case (name, test) => (name, test()) } } dependsOn(setupTasks)
 
 	def processResults(results: Iterable[(String, TestResult.Value)]): (TestResult.Value, Map[String, TestResult.Value]) =

main/src/main/scala/sbt/Defaults.scala

@@ -388,7 +388,7 @@ object Defaults extends BuildCommon
 					new Tests.Execution(opts, par, ts)
 			}
 
-	def testQuickFilter: Initialize[Task[Seq[String] => String => Boolean]] =
+	def testQuickFilter: Initialize[Task[Seq[String] => Seq[String => Boolean]]] =
 	  (fullClasspath in test, streams in test) map {
 			(cp, s) =>
 				val ans = cp.flatMap(_.metadata get Keys.analysis)
@@ -407,12 +407,13 @@ object Defaults extends BuildCommon
 							apis.internal(f).compilation.startTime
 						}.max)
 				}
-				args => test => selectedFilter(args)(test) && {
-					succeeded.get(test) match {
-						case None => true
-						case Some(ts) => stamp(test) > ts
-					}
+				def noSuccessYet(test: String) = succeeded.get(test) match {
+					case None => true
+					case Some(ts) => stamp(test) > ts
 				}
+
+				args => for(filter <- selectedFilter(args)) yield
+					(test: String) => filter(test) && noSuccessYet(test)
 		}
 	def succeededFile(dir: File) = dir / "succeeded_tests"
 
@@ -429,7 +430,7 @@ object Defaults extends BuildCommon
 			val config = testExecution.value
 
 			implicit val display = Project.showContextKey(state.value)
-			val modifiedOpts = Tests.Filter(filter(selected)) +: Tests.Argument(frameworkOptions : _*) +: config.options
+			val modifiedOpts = Tests.Filters(filter(selected)) +: Tests.Argument(frameworkOptions : _*) +: config.options
 			val newConfig = config.copy(options = modifiedOpts)
 			val groupsTask = allTestGroupsTask(s, loadedTestFrameworks.value, testLoader.value, testGrouping.value, newConfig, fullClasspath.value, javaHome.value)
 			val processed =
@@ -452,10 +453,13 @@ object Defaults extends BuildCommon
 		Tests.foldTasks(groupTasks, config.parallel)
 	}
 
-	def selectedFilter(args: Seq[String]): String => Boolean =
+	def selectedFilter(args: Seq[String]): Seq[String => Boolean] =
 	{
 		val filters = args map GlobFilter.apply
-		s => filters.isEmpty || filters.exists { _ accept s }
+		if(filters.isEmpty)
+			Seq(const(true))
+		else
+			filters.map { f => (s: String) => f accept s }
 	}
 	def detectTests: Initialize[Task[Seq[TestDefinition]]] = (loadedTestFrameworks, compile, streams) map { (frameworkMap, analysis, s) =>
 		Tests.discover(frameworkMap.values.toSeq, analysis, s.log)._1

main/src/main/scala/sbt/Keys.scala

@@ -200,7 +200,7 @@ object Keys
 	val testFrameworks = SettingKey[Seq[TestFramework]]("test-frameworks", "Registered, although not necessarily present, test frameworks.", CTask)
 	val testListeners = TaskKey[Seq[TestReportListener]]("test-listeners", "Defines test listeners.", DTask)
 	val testExecution = TaskKey[Tests.Execution]("test-execution", "Settings controlling test execution", DTask)
-	val testFilter = TaskKey[Seq[String] => String => Boolean]("test-filter", "Filter controlling whether the test is executed", DTask)
+	val testFilter = TaskKey[Seq[String] => Seq[String => Boolean]]("test-filter", "Filter controlling whether the test is executed", DTask)
 	val testGrouping = TaskKey[Seq[Tests.Group]]("test-grouping", "Collects discovered tests into groups. Whether to fork and the options for forking are configurable on a per-group basis.", BMinusTask)
 	val isModule = AttributeKey[Boolean]("is-module", "True if the target is a module.", DSetting)
 

sbt/src/sbt-test/tests/order/build.sbt

@@ -0,0 +1,3 @@
+parallelExecution in test := false
+
+libraryDependencies += "org.scalacheck" %% "scalacheck" % "1.10.0" % "test"

sbt/src/sbt-test/tests/order/src/test/scala/A.scala

@@ -0,0 +1,5 @@
+import org.scalacheck._
+
+object A extends Properties("A") {
+	property("Ran second") = Prop.secure(Counter.i == B.value)
+}

sbt/src/sbt-test/tests/order/src/test/scala/B.scala

@@ -0,0 +1,9 @@
+import org.scalacheck._
+
+object B extends Properties("B") {
+	val value = 3
+	property("Succeed") = Prop.secure {
+		Counter.i = value
+		true
+	}
+}

sbt/src/sbt-test/tests/order/src/test/scala/Counter.scala

@@ -0,0 +1,3 @@
+object Counter {
+  var i = 0
+}

sbt/src/sbt-test/tests/order/test

@@ -0,0 +1,2 @@
+-> test-only A B
+> test-only B A

testing/src/main/scala/sbt/TestFramework.scala

@@ -132,17 +132,20 @@ object TestFramework
 		log: Logger,
 		listeners: Seq[TestReportListener],
 		testArgsByFramework: Map[Framework, Seq[String]]):
-			(() => Unit, Iterable[(String, () => TestResult.Value)], TestResult.Value => () => Unit) =
+			(() => Unit, Seq[(String, () => TestResult.Value)], TestResult.Value => () => Unit) =
 	{
 		val arguments = testArgsByFramework withDefaultValue Nil
 		val mappedTests = testMap(frameworks, tests, arguments)
 		if(mappedTests.isEmpty)
 			(() => (), Nil, _ => () => () )
 		else
-			createTestTasks(testLoader, mappedTests, log, listeners)
+			createTestTasks(testLoader, mappedTests, tests, log, listeners)
 	}
 
-	private def testMap(frameworks: Seq[Framework], tests: Seq[TestDefinition], args: Map[Framework, Seq[String]]):
+	private[this] def order(mapped: Map[String, () => TestResult.Value], inputs: Seq[TestDefinition]): Seq[(String, () => TestResult.Value)] =
+		for( d <- inputs; act <- mapped.get(d.name) ) yield (d.name, act)
+
+	private[this] def testMap(frameworks: Seq[Framework], tests: Seq[TestDefinition], args: Map[Framework, Seq[String]]):
 		Map[Framework, (Set[TestDefinition], Seq[String])] =
 	{
 		import scala.collection.mutable.{HashMap, HashSet, Set}
@@ -171,7 +174,7 @@ object TestFramework
 		uniqueDefs.toSet
 	}
 		
-	private def createTestTasks(loader: ClassLoader, tests: Map[Framework, (Set[TestDefinition], Seq[String])], log: Logger, listeners: Seq[TestReportListener]) =
+	private def createTestTasks(loader: ClassLoader, tests: Map[Framework, (Set[TestDefinition], Seq[String])], ordered: Seq[TestDefinition], log: Logger, listeners: Seq[TestReportListener]) =
 	{
 		val testsListeners = listeners collect { case tl: TestsListener => tl }
 		def foreachListenerSafe(f: TestsListener => Unit): () => Unit = () => safeForeach(testsListeners, log)(f)
@@ -180,7 +183,7 @@ object TestFramework
 
 		val startTask = foreachListenerSafe(_.doInit)
 		val testTasks =
-			tests.view flatMap { case (framework, (testDefinitions, testArgs)) =>
+			tests flatMap { case (framework, (testDefinitions, testArgs)) =>
 			
 					val runner = new TestRunner(framework, loader, listeners, log)
 					for(testDefinition <- testDefinitions) yield
@@ -191,7 +194,7 @@ object TestFramework
 			}
 
 		val endTask = (result: TestResult.Value) => foreachListenerSafe(_.doComplete(result))
-		(startTask, testTasks.toList, endTask)
+		(startTask, order(testTasks, ordered), endTask)
 	}
 	private[this] def withContextLoader[T](loader: ClassLoader)(eval: => T): T =
 	{