Merge pull request #26 from Duhemm/scripted-core

Add scripted-core

build.sbt

@@ -35,7 +35,8 @@ def commonSettings: Seq[Setting[_]] = Seq(
 lazy val utilRoot: Project = (project in file(".")).
   aggregate(
     utilInterface, utilControl, utilCollection, utilApplyMacro, utilComplete,
-    utilLogging, utilRelation, utilLogic, utilCache, utilTracking, utilTesting
+    utilLogging, utilRelation, utilLogic, utilCache, utilTracking, utilTesting,
+    utilScripted
   ).
   settings(
     inThisBuild(Seq(
@@ -149,6 +150,18 @@ lazy val utilTesting = (project in internalPath / "util-testing").
     libraryDependencies ++= Seq(scalaCheck, scalatest)
   )
 
+lazy val utilScripted = (project in internalPath / "util-scripted").
+  dependsOn(utilLogging).
+  settings(
+    commonSettings,
+    name := "Util Scripted",
+    libraryDependencies += sbtIO,
+    libraryDependencies ++= {
+      if (scalaVersion.value startsWith "2.11") Seq(parserCombinator211)
+      else Seq()
+    }
+  )
+
 def customCommands: Seq[Setting[_]] = Seq(
   commands += Command.command("release") { state =>
     // "clean" ::

internal/util-scripted/src/main/java/sbt/internal/scripted/ScriptConfig.java

@@ -0,0 +1,31 @@
+package sbt.internal.scripted;
+
+import java.io.File;
+
+import xsbti.Logger;
+
+public class ScriptConfig {
+
+	private String label;
+	private File testDirectory;
+	private Logger logger;
+
+	public ScriptConfig(String label, File testDirectory, Logger logger) {
+		this.label = label;
+		this.testDirectory = testDirectory;
+		this.logger = logger;
+	}
+
+	public String label() {
+		return this.label;
+	}
+
+	public File testDirectory() {
+		return this.testDirectory;
+	}
+
+	public Logger logger() {
+		return this.logger;
+	}
+
+}

internal/util-scripted/src/main/scala/sbt/internal/scripted/CommentHandler.scala

@@ -0,0 +1,10 @@
+/* sbt -- Simple Build Tool
+ * Copyright 2009  Mark Harrah
+ */
+package sbt
+package internal
+package scripted
+
+object CommentHandler extends BasicStatementHandler {
+  def apply(command: String, args: List[String]) = ()
+}

internal/util-scripted/src/main/scala/sbt/internal/scripted/FileCommands.scala

@@ -0,0 +1,134 @@
+/* sbt -- Simple Build Tool
+ * Copyright 2009  Mark Harrah
+ */
+package sbt
+package internal
+package scripted
+
+import java.io.File
+import sbt.io.{ IO, Path }
+import Path._
+
+class FileCommands(baseDirectory: File) extends BasicStatementHandler {
+  lazy val commands = commandMap
+  def commandMap =
+    Map(
+      "touch" nonEmpty touch _,
+      "delete" nonEmpty delete _,
+      "exists" nonEmpty exists _,
+      "mkdir" nonEmpty makeDirectories _,
+      "absent" nonEmpty absent _,
+      //			"sync" twoArg("Two directory paths", sync _),
+      "newer" twoArg ("Two paths", newer _),
+      "pause" noArg {
+        println("Pausing in " + baseDirectory)
+        /*readLine("Press enter to continue. ") */
+        print("Press enter to continue. ")
+        System.console.readLine
+        println()
+      },
+      "sleep" oneArg ("Time in milliseconds", time => Thread.sleep(time.toLong)),
+      "exec" nonEmpty (execute _),
+      "copy" copy (to => rebase(baseDirectory, to)),
+      "copy-file" twoArg ("Two paths", copyFile _),
+      "must-mirror" twoArg ("Two paths", diffFiles _),
+      "copy-flat" copy flat
+    )
+
+  def apply(command: String, arguments: List[String]): Unit =
+    commands.get(command).map(_(arguments)) match {
+      case Some(_) => ()
+      case _       => scriptError("Unknown command " + command); ()
+    }
+
+  def scriptError(message: String): Unit = sys.error("Test script error: " + message)
+  def spaced[T](l: Seq[T]) = l.mkString(" ")
+  def fromStrings(paths: List[String]) = paths.map(fromString)
+  def fromString(path: String) = new File(baseDirectory, path)
+  def touch(paths: List[String]): Unit = IO.touch(fromStrings(paths))
+  def delete(paths: List[String]): Unit = IO.delete(fromStrings(paths))
+  /*def sync(from: String, to: String) =
+		IO.sync(fromString(from), fromString(to), log)*/
+  def copyFile(from: String, to: String): Unit =
+    IO.copyFile(fromString(from), fromString(to))
+  def makeDirectories(paths: List[String]) =
+    IO.createDirectories(fromStrings(paths))
+  def diffFiles(file1: String, file2: String): Unit = {
+    val lines1 = IO.readLines(fromString(file1))
+    val lines2 = IO.readLines(fromString(file2))
+    if (lines1 != lines2)
+      scriptError("File contents are different:\n" + lines1.mkString("\n") + "\nAnd:\n" + lines2.mkString("\n"))
+  }
+
+  def newer(a: String, b: String): Unit =
+    {
+      val pathA = fromString(a)
+      val pathB = fromString(b)
+      val isNewer = pathA.exists && (!pathB.exists || pathA.lastModified > pathB.lastModified)
+      if (!isNewer) {
+        scriptError(s"$pathA is not newer than $pathB")
+      }
+    }
+  def exists(paths: List[String]): Unit = {
+    val notPresent = fromStrings(paths).filter(!_.exists)
+    if (notPresent.nonEmpty)
+      scriptError("File(s) did not exist: " + notPresent.mkString("[ ", " , ", " ]"))
+  }
+  def absent(paths: List[String]): Unit = {
+    val present = fromStrings(paths).filter(_.exists)
+    if (present.nonEmpty)
+      scriptError("File(s) existed: " + present.mkString("[ ", " , ", " ]"))
+  }
+  def execute(command: List[String]): Unit = execute0(command.head, command.tail)
+  def execute0(command: String, args: List[String]): Unit = {
+    if (command.trim.isEmpty)
+      scriptError("Command was empty.")
+    else {
+      val exitValue = sys.process.Process(command :: args, baseDirectory).!
+      if (exitValue != 0)
+        sys.error("Nonzero exit value (" + exitValue + ")")
+    }
+  }
+
+  // these are for readability of the command list
+  implicit def commandBuilder(s: String): CommandBuilder = new CommandBuilder(s)
+  final class CommandBuilder(commandName: String) {
+    type NamedCommand = (String, List[String] => Unit)
+    def nonEmpty(action: List[String] => Unit): NamedCommand =
+      commandName -> { paths =>
+        if (paths.isEmpty)
+          scriptError("No arguments specified for " + commandName + " command.")
+        else
+          action(paths)
+      }
+    def twoArg(requiredArgs: String, action: (String, String) => Unit): NamedCommand =
+      commandName -> {
+        case List(from, to) => action(from, to)
+        case other          => wrongArguments(requiredArgs, other)
+      }
+    def noArg(action: => Unit): NamedCommand =
+      commandName -> {
+        case Nil   => action
+        case other => wrongArguments(other)
+      }
+    def oneArg(requiredArgs: String, action: String => Unit): NamedCommand =
+      commandName -> {
+        case List(single) => action(single)
+        case other        => wrongArguments(requiredArgs, other)
+      }
+    def copy(mapper: File => FileMap): NamedCommand =
+      commandName -> {
+        case Nil         => scriptError("No paths specified for " + commandName + " command.")
+        case path :: Nil => scriptError("No destination specified for " + commandName + " command.")
+        case paths =>
+          val mapped = fromStrings(paths)
+          val map = mapper(mapped.last)
+          IO.copy(mapped.init pair map)
+          ()
+      }
+    def wrongArguments(args: List[String]): Unit =
+      scriptError("Command '" + commandName + "' does not accept arguments (found '" + spaced(args) + "').")
+    def wrongArguments(requiredArgs: String, args: List[String]): Unit =
+      scriptError("Wrong number of arguments to " + commandName + " command.  " + requiredArgs + " required, found: '" + spaced(args) + "'.")
+  }
+}

internal/util-scripted/src/main/scala/sbt/internal/scripted/FilteredLoader.scala

@@ -0,0 +1,19 @@
+/* sbt -- Simple Build Tool
+ * Copyright 2009 Mark Harrah
+ */
+package sbt
+package internal
+package scripted
+
+final class FilteredLoader(parent: ClassLoader) extends ClassLoader(parent) {
+  @throws(classOf[ClassNotFoundException])
+  override final def loadClass(className: String, resolve: Boolean): Class[_] =
+    {
+      if (className.startsWith("java.") || className.startsWith("javax."))
+        super.loadClass(className, resolve)
+      else
+        throw new ClassNotFoundException(className)
+    }
+  override def getResources(name: String) = null
+  override def getResource(name: String) = null
+}

internal/util-scripted/src/main/scala/sbt/internal/scripted/HandlersProvider.scala

@@ -0,0 +1,5 @@
+package sbt.internal.scripted
+
+trait HandlersProvider {
+  def getHandlers(config: ScriptConfig): Map[Char, StatementHandler]
+}

internal/util-scripted/src/main/scala/sbt/internal/scripted/ScriptRunner.scala

@@ -0,0 +1,48 @@
+/* sbt -- Simple Build Tool
+ * Copyright 2009  Mark Harrah
+ */
+package sbt
+package internal
+package scripted
+
+final class TestException(statement: Statement, msg: String, exception: Throwable)
+  extends RuntimeException(statement.linePrefix + " " + msg, exception)
+
+class ScriptRunner {
+  import scala.collection.mutable.HashMap
+  def apply(statements: List[(StatementHandler, Statement)]): Unit = {
+    val states = new HashMap[StatementHandler, Any]
+    def processStatement(handler: StatementHandler, statement: Statement): Unit = {
+      val state = states(handler).asInstanceOf[handler.State]
+      val nextState =
+        try { Right(handler(statement.command, statement.arguments, state)) }
+        catch { case e: Exception => Left(e) }
+      nextState match {
+        case Left(err) =>
+          if (statement.successExpected) {
+            err match {
+              case t: TestFailed => throw new TestException(statement, "Command failed: " + t.getMessage, null)
+              case _             => throw new TestException(statement, "Command failed", err)
+            }
+          } else
+            ()
+        case Right(s) =>
+          if (statement.successExpected)
+            states(handler) = s
+          else
+            throw new TestException(statement, "Command succeeded but failure was expected", null)
+      }
+    }
+    val handlers = Set() ++ statements.map(_._1)
+
+    try {
+      handlers.foreach { handler => states(handler) = handler.initialState }
+      statements foreach (Function.tupled(processStatement))
+    } finally {
+      for (handler <- handlers; state <- states.get(handler)) {
+        try { handler.finish(state.asInstanceOf[handler.State]) }
+        catch { case e: Exception => () }
+      }
+    }
+  }
+}

internal/util-scripted/src/main/scala/sbt/internal/scripted/ScriptedTests.scala

@@ -0,0 +1,173 @@
+package sbt
+package internal
+package scripted
+
+import java.io.File
+import sbt.util.Logger
+import sbt.internal.util.{ ConsoleLogger, BufferedLogger, FullLogger }
+import sbt.io.IO.wrapNull
+import sbt.io.{ DirectoryFilter, HiddenFileFilter, Path, GlobFilter }
+import sbt.internal.io.Resources
+
+object ScriptedRunnerImpl {
+  def run(resourceBaseDirectory: File, bufferLog: Boolean, tests: Array[String], handlersProvider: HandlersProvider): Unit = {
+    val runner = new ScriptedTests(resourceBaseDirectory, bufferLog, handlersProvider)
+    val logger = ConsoleLogger()
+    val allTests = get(tests, resourceBaseDirectory, logger) flatMap {
+      case ScriptedTest(group, name) =>
+        runner.scriptedTest(group, name, logger)
+    }
+    runAll(allTests)
+  }
+  def runAll(tests: Seq[() => Option[String]]): Unit = {
+    val errors = for (test <- tests; err <- test()) yield err
+    if (errors.nonEmpty)
+      sys.error(errors.mkString("Failed tests:\n\t", "\n\t", "\n"))
+  }
+  def get(tests: Seq[String], baseDirectory: File, log: Logger): Seq[ScriptedTest] =
+    if (tests.isEmpty) listTests(baseDirectory, log) else parseTests(tests)
+  def listTests(baseDirectory: File, log: Logger): Seq[ScriptedTest] =
+    (new ListTests(baseDirectory, _ => true, log)).listTests
+  def parseTests(in: Seq[String]): Seq[ScriptedTest] =
+    for (testString <- in) yield {
+      val Array(group, name) = testString.split("/").map(_.trim)
+      ScriptedTest(group, name)
+    }
+}
+
+final class ScriptedTests(resourceBaseDirectory: File, bufferLog: Boolean, handlersProvider: HandlersProvider) {
+  // import ScriptedTests._
+  private val testResources = new Resources(resourceBaseDirectory)
+
+  val ScriptFilename = "test"
+  val PendingScriptFilename = "pending"
+
+  def scriptedTest(group: String, name: String, log: xsbti.Logger): Seq[() => Option[String]] =
+    scriptedTest(group, name, Logger.xlog2Log(log))
+  def scriptedTest(group: String, name: String, log: Logger): Seq[() => Option[String]] =
+    scriptedTest(group, name, { _ => () }, log)
+  def scriptedTest(group: String, name: String, prescripted: File => Unit, log: Logger): Seq[() => Option[String]] = {
+    import Path._
+    import GlobFilter._
+    var failed = false
+    for (groupDir <- (resourceBaseDirectory * group).get; nme <- (groupDir * name).get) yield {
+      val g = groupDir.getName
+      val n = nme.getName
+      val str = s"$g / $n"
+      () => {
+        println("Running " + str)
+        testResources.readWriteResourceDirectory(g, n) { testDirectory =>
+          val disabled = new File(testDirectory, "disabled").isFile
+          if (disabled) {
+            log.info("D " + str + " [DISABLED]")
+            None
+          } else {
+            try { scriptedTest(str, testDirectory, prescripted, log); None }
+            catch { case _: TestException | _: PendingTestSuccessException => Some(str) }
+          }
+        }
+      }
+    }
+  }
+
+  private def scriptedTest(label: String, testDirectory: File, prescripted: File => Unit, log: Logger): Unit =
+    {
+      val buffered = new BufferedLogger(new FullLogger(log))
+      if (bufferLog)
+        buffered.record()
+
+      def createParser() =
+        {
+          // val fileHandler = new FileCommands(testDirectory)
+          // // val sbtHandler = new SbtHandler(testDirectory, launcher, buffered, launchOpts)
+          // new TestScriptParser(Map('$' -> fileHandler, /* '>' -> sbtHandler, */ '#' -> CommentHandler))
+          val scriptConfig = new ScriptConfig(label, testDirectory, buffered)
+          new TestScriptParser(handlersProvider getHandlers scriptConfig)
+        }
+      val (file, pending) = {
+        val normal = new File(testDirectory, ScriptFilename)
+        val pending = new File(testDirectory, PendingScriptFilename)
+        if (pending.isFile) (pending, true) else (normal, false)
+      }
+      val pendingString = if (pending) " [PENDING]" else ""
+
+      def runTest() =
+        {
+          val run = new ScriptRunner
+          val parser = createParser()
+          run(parser.parse(file))
+        }
+      def testFailed(): Unit = {
+        if (pending) buffered.clear() else buffered.stop()
+        buffered.error("x " + label + pendingString)
+      }
+
+      try {
+        prescripted(testDirectory)
+        runTest()
+        buffered.info("+ " + label + pendingString)
+        if (pending) throw new PendingTestSuccessException(label)
+      } catch {
+        case e: TestException =>
+          testFailed()
+          e.getCause match {
+            case null | _: java.net.SocketException => buffered.error("   " + e.getMessage)
+            case _                                  => e.printStackTrace
+          }
+          if (!pending) throw e
+        case e: PendingTestSuccessException =>
+          testFailed()
+          buffered.error("  Mark as passing to remove this failure.")
+          throw e
+        case e: Exception =>
+          testFailed()
+          if (!pending) throw e
+      } finally { buffered.clear() }
+    }
+}
+
+// object ScriptedTests extends ScriptedRunner {
+//   val emptyCallback: File => Unit = { _ => () }
+// }
+
+final case class ScriptedTest(group: String, name: String) {
+  override def toString = group + "/" + name
+}
+
+object ListTests {
+  def list(directory: File, filter: java.io.FileFilter) = wrapNull(directory.listFiles(filter))
+}
+import ListTests._
+final class ListTests(baseDirectory: File, accept: ScriptedTest => Boolean, log: Logger) {
+  def filter = DirectoryFilter -- HiddenFileFilter
+  def listTests: Seq[ScriptedTest] =
+    {
+      list(baseDirectory, filter) flatMap { group =>
+        val groupName = group.getName
+        listTests(group).map(ScriptedTest(groupName, _))
+      }
+    }
+  private[this] def listTests(group: File): Set[String] =
+    {
+      val groupName = group.getName
+      val allTests = list(group, filter)
+      if (allTests.isEmpty) {
+        log.warn("No tests in test group " + groupName)
+        Set.empty
+      } else {
+        val (included, skipped) = allTests.toList.partition(test => accept(ScriptedTest(groupName, test.getName)))
+        if (included.isEmpty)
+          log.warn("Test group " + groupName + " skipped.")
+        else if (skipped.nonEmpty) {
+          log.warn("Tests skipped in group " + group.getName + ":")
+          skipped.foreach(testName => log.warn(" " + testName.getName))
+        }
+        Set(included.map(_.getName): _*)
+      }
+    }
+}
+
+class PendingTestSuccessException(label: String) extends Exception {
+  override def getMessage: String =
+    s"The pending test $label succeeded. Mark this test as passing to remove this failure."
+}

internal/util-scripted/src/main/scala/sbt/internal/scripted/StatementHandler.scala

@@ -0,0 +1,26 @@
+/* sbt -- Simple Build Tool
+ * Copyright 2009  Mark Harrah
+ */
+package sbt
+package internal
+package scripted
+
+trait StatementHandler {
+  type State
+  def initialState: State
+  def apply(command: String, arguments: List[String], state: State): State
+  def finish(state: State): Unit
+}
+
+trait BasicStatementHandler extends StatementHandler {
+  final type State = Unit
+  final def initialState = ()
+  final def apply(command: String, arguments: List[String], state: Unit): Unit = apply(command, arguments)
+  def apply(command: String, arguments: List[String]): Unit
+  def finish(state: Unit) = ()
+}
+
+/** Use when a stack trace is not useful */
+final class TestFailed(msg: String) extends RuntimeException(msg) {
+  override def fillInStackTrace = this
+}

internal/util-scripted/src/main/scala/sbt/internal/scripted/TestScriptParser.scala

@@ -0,0 +1,83 @@
+/* sbt -- Simple Build Tool
+ * Copyright 2009  Mark Harrah
+ */
+package sbt
+package internal
+package scripted
+
+import java.io.{ BufferedReader, File, InputStreamReader }
+import scala.util.parsing.combinator._
+import scala.util.parsing.input.Positional
+import Character.isWhitespace
+import sbt.io.IO
+
+/*
+statement*
+statement ::= startChar successChar word+ nl
+startChar ::= <single character>
+successChar ::= '+' | '-'
+word ::= [^ \[\]]+
+comment ::= '#' \S* nl
+nl ::= '\r' \'n' | '\n' | '\r' | eof
+*/
+final case class Statement(command: String, arguments: List[String], successExpected: Boolean, line: Int) {
+  def linePrefix = "{line " + line + "} "
+}
+
+private object TestScriptParser {
+  val SuccessLiteral = "success"
+  val FailureLiteral = "failure"
+  val WordRegex = """[^ \[\]\s'\"][^ \[\]\s]*""".r
+}
+
+import TestScriptParser._
+class TestScriptParser(handlers: Map[Char, StatementHandler]) extends RegexParsers {
+  require(handlers.nonEmpty)
+  override def skipWhitespace = false
+
+  import IO.read
+  if (handlers.keys.exists(isWhitespace))
+    sys.error("Start characters cannot be whitespace")
+  if (handlers.keys.exists(key => key == '+' || key == '-'))
+    sys.error("Start characters cannot be '+' or '-'")
+
+  def parse(scriptFile: File): List[(StatementHandler, Statement)] = parse(read(scriptFile), Some(scriptFile.getAbsolutePath))
+  def parse(script: String): List[(StatementHandler, Statement)] = parse(script, None)
+  private def parse(script: String, label: Option[String]): List[(StatementHandler, Statement)] =
+    {
+      parseAll(statements, script) match {
+        case Success(result, next) => result
+        case err: NoSuccess =>
+          {
+            val labelString = label.map("'" + _ + "' ").getOrElse("")
+            sys.error("Could not parse test script, " + labelString + err.toString)
+          }
+      }
+    }
+
+  lazy val statements = rep1(space ~> statement <~ newline)
+  def statement: Parser[(StatementHandler, Statement)] =
+    {
+      trait PositionalStatement extends Positional {
+        def tuple: (StatementHandler, Statement)
+      }
+      positioned {
+        val command = (word | err("expected command"))
+        val arguments = rep(space ~> (word | failure("expected argument")))
+        (successParser ~ (space ~> startCharacterParser <~ space) ~! command ~! arguments) ^^
+          {
+            case successExpected ~ start ~ command ~ arguments =>
+              new PositionalStatement {
+                def tuple = (handlers(start), new Statement(command, arguments, successExpected, pos.line))
+              }
+          }
+      } ^^ (_.tuple)
+    }
+  def successParser: Parser[Boolean] = ('+' ^^^ true) | ('-' ^^^ false) | success(true)
+  def space: Parser[String] = """[ \t]*""".r
+  lazy val word: Parser[String] = ("\'" ~> "[^'\n\r]*".r <~ "\'") | ("\"" ~> "[^\"\n\r]*".r <~ "\"") | WordRegex
+  def startCharacterParser: Parser[Char] = elem("start character", handlers.contains _) |
+    ((newline | err("expected start character " + handlers.keys.mkString("(", "", ")"))) ~> failure("end of input"))
+
+  def newline = """\s*([\n\r]|$)""".r
+}

project/Dependencies.scala

@@ -25,4 +25,6 @@ object Dependencies {
 
   lazy val scalaCheck = "org.scalacheck" %% "scalacheck" % "1.12.4"
   lazy val scalatest = "org.scalatest" %% "scalatest" % "2.2.4"
+
+  lazy val parserCombinator211 = "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.4"
 }