diff --git a/build.sbt b/build.sbt index 02b37203c..175279969 100644 --- a/build.sbt +++ b/build.sbt @@ -35,7 +35,8 @@ def commonSettings: Seq[Setting[_]] = Seq( lazy val utilRoot: Project = (project in file(".")). aggregate( utilInterface, utilControl, utilCollection, utilApplyMacro, utilComplete, - utilLogging, utilRelation, utilLogic, utilCache, utilTracking, utilTesting + utilLogging, utilRelation, utilLogic, utilCache, utilTracking, utilTesting, + utilScripted ). settings( inThisBuild(Seq( @@ -149,6 +150,18 @@ lazy val utilTesting = (project in internalPath / "util-testing"). libraryDependencies ++= Seq(scalaCheck, scalatest) ) +lazy val utilScripted = (project in internalPath / "util-scripted"). + dependsOn(utilLogging). + settings( + commonSettings, + name := "Util Scripted", + libraryDependencies += sbtIO, + libraryDependencies ++= { + if (scalaVersion.value startsWith "2.11") Seq(parserCombinator211) + else Seq() + } + ) + def customCommands: Seq[Setting[_]] = Seq( commands += Command.command("release") { state => // "clean" :: diff --git a/internal/util-scripted/src/main/java/sbt/internal/scripted/ScriptConfig.java b/internal/util-scripted/src/main/java/sbt/internal/scripted/ScriptConfig.java new file mode 100644 index 000000000..52cb52c4e --- /dev/null +++ b/internal/util-scripted/src/main/java/sbt/internal/scripted/ScriptConfig.java @@ -0,0 +1,31 @@ +package sbt.internal.scripted; + +import java.io.File; + +import xsbti.Logger; + +public class ScriptConfig { + + private String label; + private File testDirectory; + private Logger logger; + + public ScriptConfig(String label, File testDirectory, Logger logger) { + this.label = label; + this.testDirectory = testDirectory; + this.logger = logger; + } + + public String label() { + return this.label; + } + + public File testDirectory() { + return this.testDirectory; + } + + public Logger logger() { + return this.logger; + } + +} \ No newline at end of file diff --git a/internal/util-scripted/src/main/scala/sbt/internal/scripted/CommentHandler.scala b/internal/util-scripted/src/main/scala/sbt/internal/scripted/CommentHandler.scala new file mode 100644 index 000000000..370ae0005 --- /dev/null +++ b/internal/util-scripted/src/main/scala/sbt/internal/scripted/CommentHandler.scala @@ -0,0 +1,10 @@ +/* sbt -- Simple Build Tool + * Copyright 2009 Mark Harrah + */ +package sbt +package internal +package scripted + +object CommentHandler extends BasicStatementHandler { + def apply(command: String, args: List[String]) = () +} \ No newline at end of file diff --git a/internal/util-scripted/src/main/scala/sbt/internal/scripted/FileCommands.scala b/internal/util-scripted/src/main/scala/sbt/internal/scripted/FileCommands.scala new file mode 100644 index 000000000..ea5fc7559 --- /dev/null +++ b/internal/util-scripted/src/main/scala/sbt/internal/scripted/FileCommands.scala @@ -0,0 +1,134 @@ +/* sbt -- Simple Build Tool + * Copyright 2009 Mark Harrah + */ +package sbt +package internal +package scripted + +import java.io.File +import sbt.io.{ IO, Path } +import Path._ + +class FileCommands(baseDirectory: File) extends BasicStatementHandler { + lazy val commands = commandMap + def commandMap = + Map( + "touch" nonEmpty touch _, + "delete" nonEmpty delete _, + "exists" nonEmpty exists _, + "mkdir" nonEmpty makeDirectories _, + "absent" nonEmpty absent _, + // "sync" twoArg("Two directory paths", sync _), + "newer" twoArg ("Two paths", newer _), + "pause" noArg { + println("Pausing in " + baseDirectory) + /*readLine("Press enter to continue. ") */ + print("Press enter to continue. ") + System.console.readLine + println() + }, + "sleep" oneArg ("Time in milliseconds", time => Thread.sleep(time.toLong)), + "exec" nonEmpty (execute _), + "copy" copy (to => rebase(baseDirectory, to)), + "copy-file" twoArg ("Two paths", copyFile _), + "must-mirror" twoArg ("Two paths", diffFiles _), + "copy-flat" copy flat + ) + + def apply(command: String, arguments: List[String]): Unit = + commands.get(command).map(_(arguments)) match { + case Some(_) => () + case _ => scriptError("Unknown command " + command); () + } + + def scriptError(message: String): Unit = sys.error("Test script error: " + message) + def spaced[T](l: Seq[T]) = l.mkString(" ") + def fromStrings(paths: List[String]) = paths.map(fromString) + def fromString(path: String) = new File(baseDirectory, path) + def touch(paths: List[String]): Unit = IO.touch(fromStrings(paths)) + def delete(paths: List[String]): Unit = IO.delete(fromStrings(paths)) + /*def sync(from: String, to: String) = + IO.sync(fromString(from), fromString(to), log)*/ + def copyFile(from: String, to: String): Unit = + IO.copyFile(fromString(from), fromString(to)) + def makeDirectories(paths: List[String]) = + IO.createDirectories(fromStrings(paths)) + def diffFiles(file1: String, file2: String): Unit = { + val lines1 = IO.readLines(fromString(file1)) + val lines2 = IO.readLines(fromString(file2)) + if (lines1 != lines2) + scriptError("File contents are different:\n" + lines1.mkString("\n") + "\nAnd:\n" + lines2.mkString("\n")) + } + + def newer(a: String, b: String): Unit = + { + val pathA = fromString(a) + val pathB = fromString(b) + val isNewer = pathA.exists && (!pathB.exists || pathA.lastModified > pathB.lastModified) + if (!isNewer) { + scriptError(s"$pathA is not newer than $pathB") + } + } + def exists(paths: List[String]): Unit = { + val notPresent = fromStrings(paths).filter(!_.exists) + if (notPresent.nonEmpty) + scriptError("File(s) did not exist: " + notPresent.mkString("[ ", " , ", " ]")) + } + def absent(paths: List[String]): Unit = { + val present = fromStrings(paths).filter(_.exists) + if (present.nonEmpty) + scriptError("File(s) existed: " + present.mkString("[ ", " , ", " ]")) + } + def execute(command: List[String]): Unit = execute0(command.head, command.tail) + def execute0(command: String, args: List[String]): Unit = { + if (command.trim.isEmpty) + scriptError("Command was empty.") + else { + val exitValue = sys.process.Process(command :: args, baseDirectory).! + if (exitValue != 0) + sys.error("Nonzero exit value (" + exitValue + ")") + } + } + + // these are for readability of the command list + implicit def commandBuilder(s: String): CommandBuilder = new CommandBuilder(s) + final class CommandBuilder(commandName: String) { + type NamedCommand = (String, List[String] => Unit) + def nonEmpty(action: List[String] => Unit): NamedCommand = + commandName -> { paths => + if (paths.isEmpty) + scriptError("No arguments specified for " + commandName + " command.") + else + action(paths) + } + def twoArg(requiredArgs: String, action: (String, String) => Unit): NamedCommand = + commandName -> { + case List(from, to) => action(from, to) + case other => wrongArguments(requiredArgs, other) + } + def noArg(action: => Unit): NamedCommand = + commandName -> { + case Nil => action + case other => wrongArguments(other) + } + def oneArg(requiredArgs: String, action: String => Unit): NamedCommand = + commandName -> { + case List(single) => action(single) + case other => wrongArguments(requiredArgs, other) + } + def copy(mapper: File => FileMap): NamedCommand = + commandName -> { + case Nil => scriptError("No paths specified for " + commandName + " command.") + case path :: Nil => scriptError("No destination specified for " + commandName + " command.") + case paths => + val mapped = fromStrings(paths) + val map = mapper(mapped.last) + IO.copy(mapped.init pair map) + () + } + def wrongArguments(args: List[String]): Unit = + scriptError("Command '" + commandName + "' does not accept arguments (found '" + spaced(args) + "').") + def wrongArguments(requiredArgs: String, args: List[String]): Unit = + scriptError("Wrong number of arguments to " + commandName + " command. " + requiredArgs + " required, found: '" + spaced(args) + "'.") + } +} \ No newline at end of file diff --git a/internal/util-scripted/src/main/scala/sbt/internal/scripted/FilteredLoader.scala b/internal/util-scripted/src/main/scala/sbt/internal/scripted/FilteredLoader.scala new file mode 100644 index 000000000..cb2c3100d --- /dev/null +++ b/internal/util-scripted/src/main/scala/sbt/internal/scripted/FilteredLoader.scala @@ -0,0 +1,19 @@ +/* sbt -- Simple Build Tool + * Copyright 2009 Mark Harrah + */ +package sbt +package internal +package scripted + +final class FilteredLoader(parent: ClassLoader) extends ClassLoader(parent) { + @throws(classOf[ClassNotFoundException]) + override final def loadClass(className: String, resolve: Boolean): Class[_] = + { + if (className.startsWith("java.") || className.startsWith("javax.")) + super.loadClass(className, resolve) + else + throw new ClassNotFoundException(className) + } + override def getResources(name: String) = null + override def getResource(name: String) = null +} \ No newline at end of file diff --git a/internal/util-scripted/src/main/scala/sbt/internal/scripted/HandlersProvider.scala b/internal/util-scripted/src/main/scala/sbt/internal/scripted/HandlersProvider.scala new file mode 100644 index 000000000..3dcb4ef6d --- /dev/null +++ b/internal/util-scripted/src/main/scala/sbt/internal/scripted/HandlersProvider.scala @@ -0,0 +1,5 @@ +package sbt.internal.scripted + +trait HandlersProvider { + def getHandlers(config: ScriptConfig): Map[Char, StatementHandler] +} \ No newline at end of file diff --git a/internal/util-scripted/src/main/scala/sbt/internal/scripted/ScriptRunner.scala b/internal/util-scripted/src/main/scala/sbt/internal/scripted/ScriptRunner.scala new file mode 100644 index 000000000..f43b54f39 --- /dev/null +++ b/internal/util-scripted/src/main/scala/sbt/internal/scripted/ScriptRunner.scala @@ -0,0 +1,48 @@ +/* sbt -- Simple Build Tool + * Copyright 2009 Mark Harrah + */ +package sbt +package internal +package scripted + +final class TestException(statement: Statement, msg: String, exception: Throwable) + extends RuntimeException(statement.linePrefix + " " + msg, exception) + +class ScriptRunner { + import scala.collection.mutable.HashMap + def apply(statements: List[(StatementHandler, Statement)]): Unit = { + val states = new HashMap[StatementHandler, Any] + def processStatement(handler: StatementHandler, statement: Statement): Unit = { + val state = states(handler).asInstanceOf[handler.State] + val nextState = + try { Right(handler(statement.command, statement.arguments, state)) } + catch { case e: Exception => Left(e) } + nextState match { + case Left(err) => + if (statement.successExpected) { + err match { + case t: TestFailed => throw new TestException(statement, "Command failed: " + t.getMessage, null) + case _ => throw new TestException(statement, "Command failed", err) + } + } else + () + case Right(s) => + if (statement.successExpected) + states(handler) = s + else + throw new TestException(statement, "Command succeeded but failure was expected", null) + } + } + val handlers = Set() ++ statements.map(_._1) + + try { + handlers.foreach { handler => states(handler) = handler.initialState } + statements foreach (Function.tupled(processStatement)) + } finally { + for (handler <- handlers; state <- states.get(handler)) { + try { handler.finish(state.asInstanceOf[handler.State]) } + catch { case e: Exception => () } + } + } + } +} \ No newline at end of file diff --git a/internal/util-scripted/src/main/scala/sbt/internal/scripted/ScriptedTests.scala b/internal/util-scripted/src/main/scala/sbt/internal/scripted/ScriptedTests.scala new file mode 100644 index 000000000..dcfb50845 --- /dev/null +++ b/internal/util-scripted/src/main/scala/sbt/internal/scripted/ScriptedTests.scala @@ -0,0 +1,173 @@ +package sbt +package internal +package scripted + +import java.io.File +import sbt.util.Logger +import sbt.internal.util.{ ConsoleLogger, BufferedLogger, FullLogger } +import sbt.io.IO.wrapNull +import sbt.io.{ DirectoryFilter, HiddenFileFilter, Path, GlobFilter } +import sbt.internal.io.Resources + +object ScriptedRunnerImpl { + def run(resourceBaseDirectory: File, bufferLog: Boolean, tests: Array[String], handlersProvider: HandlersProvider): Unit = { + val runner = new ScriptedTests(resourceBaseDirectory, bufferLog, handlersProvider) + val logger = ConsoleLogger() + val allTests = get(tests, resourceBaseDirectory, logger) flatMap { + case ScriptedTest(group, name) => + runner.scriptedTest(group, name, logger) + } + runAll(allTests) + } + def runAll(tests: Seq[() => Option[String]]): Unit = { + val errors = for (test <- tests; err <- test()) yield err + if (errors.nonEmpty) + sys.error(errors.mkString("Failed tests:\n\t", "\n\t", "\n")) + } + def get(tests: Seq[String], baseDirectory: File, log: Logger): Seq[ScriptedTest] = + if (tests.isEmpty) listTests(baseDirectory, log) else parseTests(tests) + def listTests(baseDirectory: File, log: Logger): Seq[ScriptedTest] = + (new ListTests(baseDirectory, _ => true, log)).listTests + def parseTests(in: Seq[String]): Seq[ScriptedTest] = + for (testString <- in) yield { + val Array(group, name) = testString.split("/").map(_.trim) + ScriptedTest(group, name) + } +} + +final class ScriptedTests(resourceBaseDirectory: File, bufferLog: Boolean, handlersProvider: HandlersProvider) { + // import ScriptedTests._ + private val testResources = new Resources(resourceBaseDirectory) + + val ScriptFilename = "test" + val PendingScriptFilename = "pending" + + def scriptedTest(group: String, name: String, log: xsbti.Logger): Seq[() => Option[String]] = + scriptedTest(group, name, Logger.xlog2Log(log)) + def scriptedTest(group: String, name: String, log: Logger): Seq[() => Option[String]] = + scriptedTest(group, name, { _ => () }, log) + def scriptedTest(group: String, name: String, prescripted: File => Unit, log: Logger): Seq[() => Option[String]] = { + import Path._ + import GlobFilter._ + var failed = false + for (groupDir <- (resourceBaseDirectory * group).get; nme <- (groupDir * name).get) yield { + val g = groupDir.getName + val n = nme.getName + val str = s"$g / $n" + () => { + println("Running " + str) + testResources.readWriteResourceDirectory(g, n) { testDirectory => + val disabled = new File(testDirectory, "disabled").isFile + if (disabled) { + log.info("D " + str + " [DISABLED]") + None + } else { + try { scriptedTest(str, testDirectory, prescripted, log); None } + catch { case _: TestException | _: PendingTestSuccessException => Some(str) } + } + } + } + } + } + + private def scriptedTest(label: String, testDirectory: File, prescripted: File => Unit, log: Logger): Unit = + { + val buffered = new BufferedLogger(new FullLogger(log)) + if (bufferLog) + buffered.record() + + def createParser() = + { + // val fileHandler = new FileCommands(testDirectory) + // // val sbtHandler = new SbtHandler(testDirectory, launcher, buffered, launchOpts) + // new TestScriptParser(Map('$' -> fileHandler, /* '>' -> sbtHandler, */ '#' -> CommentHandler)) + val scriptConfig = new ScriptConfig(label, testDirectory, buffered) + new TestScriptParser(handlersProvider getHandlers scriptConfig) + } + val (file, pending) = { + val normal = new File(testDirectory, ScriptFilename) + val pending = new File(testDirectory, PendingScriptFilename) + if (pending.isFile) (pending, true) else (normal, false) + } + val pendingString = if (pending) " [PENDING]" else "" + + def runTest() = + { + val run = new ScriptRunner + val parser = createParser() + run(parser.parse(file)) + } + def testFailed(): Unit = { + if (pending) buffered.clear() else buffered.stop() + buffered.error("x " + label + pendingString) + } + + try { + prescripted(testDirectory) + runTest() + buffered.info("+ " + label + pendingString) + if (pending) throw new PendingTestSuccessException(label) + } catch { + case e: TestException => + testFailed() + e.getCause match { + case null | _: java.net.SocketException => buffered.error(" " + e.getMessage) + case _ => e.printStackTrace + } + if (!pending) throw e + case e: PendingTestSuccessException => + testFailed() + buffered.error(" Mark as passing to remove this failure.") + throw e + case e: Exception => + testFailed() + if (!pending) throw e + } finally { buffered.clear() } + } +} + +// object ScriptedTests extends ScriptedRunner { +// val emptyCallback: File => Unit = { _ => () } +// } + +final case class ScriptedTest(group: String, name: String) { + override def toString = group + "/" + name +} + +object ListTests { + def list(directory: File, filter: java.io.FileFilter) = wrapNull(directory.listFiles(filter)) +} +import ListTests._ +final class ListTests(baseDirectory: File, accept: ScriptedTest => Boolean, log: Logger) { + def filter = DirectoryFilter -- HiddenFileFilter + def listTests: Seq[ScriptedTest] = + { + list(baseDirectory, filter) flatMap { group => + val groupName = group.getName + listTests(group).map(ScriptedTest(groupName, _)) + } + } + private[this] def listTests(group: File): Set[String] = + { + val groupName = group.getName + val allTests = list(group, filter) + if (allTests.isEmpty) { + log.warn("No tests in test group " + groupName) + Set.empty + } else { + val (included, skipped) = allTests.toList.partition(test => accept(ScriptedTest(groupName, test.getName))) + if (included.isEmpty) + log.warn("Test group " + groupName + " skipped.") + else if (skipped.nonEmpty) { + log.warn("Tests skipped in group " + group.getName + ":") + skipped.foreach(testName => log.warn(" " + testName.getName)) + } + Set(included.map(_.getName): _*) + } + } +} + +class PendingTestSuccessException(label: String) extends Exception { + override def getMessage: String = + s"The pending test $label succeeded. Mark this test as passing to remove this failure." +} \ No newline at end of file diff --git a/internal/util-scripted/src/main/scala/sbt/internal/scripted/StatementHandler.scala b/internal/util-scripted/src/main/scala/sbt/internal/scripted/StatementHandler.scala new file mode 100644 index 000000000..15b7189ce --- /dev/null +++ b/internal/util-scripted/src/main/scala/sbt/internal/scripted/StatementHandler.scala @@ -0,0 +1,26 @@ +/* sbt -- Simple Build Tool + * Copyright 2009 Mark Harrah + */ +package sbt +package internal +package scripted + +trait StatementHandler { + type State + def initialState: State + def apply(command: String, arguments: List[String], state: State): State + def finish(state: State): Unit +} + +trait BasicStatementHandler extends StatementHandler { + final type State = Unit + final def initialState = () + final def apply(command: String, arguments: List[String], state: Unit): Unit = apply(command, arguments) + def apply(command: String, arguments: List[String]): Unit + def finish(state: Unit) = () +} + +/** Use when a stack trace is not useful */ +final class TestFailed(msg: String) extends RuntimeException(msg) { + override def fillInStackTrace = this +} \ No newline at end of file diff --git a/internal/util-scripted/src/main/scala/sbt/internal/scripted/TestScriptParser.scala b/internal/util-scripted/src/main/scala/sbt/internal/scripted/TestScriptParser.scala new file mode 100644 index 000000000..2e8b7f7f6 --- /dev/null +++ b/internal/util-scripted/src/main/scala/sbt/internal/scripted/TestScriptParser.scala @@ -0,0 +1,83 @@ +/* sbt -- Simple Build Tool + * Copyright 2009 Mark Harrah + */ +package sbt +package internal +package scripted + +import java.io.{ BufferedReader, File, InputStreamReader } +import scala.util.parsing.combinator._ +import scala.util.parsing.input.Positional +import Character.isWhitespace +import sbt.io.IO + +/* +statement* +statement ::= startChar successChar word+ nl +startChar ::= +successChar ::= '+' | '-' +word ::= [^ \[\]]+ +comment ::= '#' \S* nl +nl ::= '\r' \'n' | '\n' | '\r' | eof +*/ +final case class Statement(command: String, arguments: List[String], successExpected: Boolean, line: Int) { + def linePrefix = "{line " + line + "} " +} + +private object TestScriptParser { + val SuccessLiteral = "success" + val FailureLiteral = "failure" + val WordRegex = """[^ \[\]\s'\"][^ \[\]\s]*""".r +} + +import TestScriptParser._ +class TestScriptParser(handlers: Map[Char, StatementHandler]) extends RegexParsers { + require(handlers.nonEmpty) + override def skipWhitespace = false + + import IO.read + if (handlers.keys.exists(isWhitespace)) + sys.error("Start characters cannot be whitespace") + if (handlers.keys.exists(key => key == '+' || key == '-')) + sys.error("Start characters cannot be '+' or '-'") + + def parse(scriptFile: File): List[(StatementHandler, Statement)] = parse(read(scriptFile), Some(scriptFile.getAbsolutePath)) + def parse(script: String): List[(StatementHandler, Statement)] = parse(script, None) + private def parse(script: String, label: Option[String]): List[(StatementHandler, Statement)] = + { + parseAll(statements, script) match { + case Success(result, next) => result + case err: NoSuccess => + { + val labelString = label.map("'" + _ + "' ").getOrElse("") + sys.error("Could not parse test script, " + labelString + err.toString) + } + } + } + + lazy val statements = rep1(space ~> statement <~ newline) + def statement: Parser[(StatementHandler, Statement)] = + { + trait PositionalStatement extends Positional { + def tuple: (StatementHandler, Statement) + } + positioned { + val command = (word | err("expected command")) + val arguments = rep(space ~> (word | failure("expected argument"))) + (successParser ~ (space ~> startCharacterParser <~ space) ~! command ~! arguments) ^^ + { + case successExpected ~ start ~ command ~ arguments => + new PositionalStatement { + def tuple = (handlers(start), new Statement(command, arguments, successExpected, pos.line)) + } + } + } ^^ (_.tuple) + } + def successParser: Parser[Boolean] = ('+' ^^^ true) | ('-' ^^^ false) | success(true) + def space: Parser[String] = """[ \t]*""".r + lazy val word: Parser[String] = ("\'" ~> "[^'\n\r]*".r <~ "\'") | ("\"" ~> "[^\"\n\r]*".r <~ "\"") | WordRegex + def startCharacterParser: Parser[Char] = elem("start character", handlers.contains _) | + ((newline | err("expected start character " + handlers.keys.mkString("(", "", ")"))) ~> failure("end of input")) + + def newline = """\s*([\n\r]|$)""".r +} \ No newline at end of file diff --git a/project/Dependencies.scala b/project/Dependencies.scala index ac034d38a..2aa4efc99 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -25,4 +25,6 @@ object Dependencies { lazy val scalaCheck = "org.scalacheck" %% "scalacheck" % "1.12.4" lazy val scalatest = "org.scalatest" %% "scalatest" % "2.2.4" + + lazy val parserCombinator211 = "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.4" }