Merge pull request #1236 from urbas/0.13

Implemented the fileParser and added DynamicExamples.

util/complete/src/main/scala/sbt/complete/ExampleSource.scala

@@ -0,0 +1,62 @@
+package sbt.complete
+
+import java.io.File
+import sbt.IO._
+
+/**
+ * These sources of examples are used in parsers for user input completion. An example of such a source is the
+ * [[sbt.complete.FileExamples]] class, which provides a list of suggested files to the user as they press the
+ * TAB key in the console.
+ */
+trait ExampleSource
+{
+	/**
+	 * @return a (possibly lazy) list of completion example strings. These strings are continuations of user's input. The
+	 *         user's input is incremented with calls to [[withAddedPrefix]].
+	 */
+	def apply(): Iterable[String]
+
+	/**
+	 * @param addedPrefix a string that just typed in by the user.
+	 * @return a new source of only those examples that start with the string typed by the user so far (with addition of
+	 *         the just added prefix).
+	 */
+	def withAddedPrefix(addedPrefix: String): ExampleSource
+}
+
+/**
+ * A convenience example source that wraps any collection of strings into a source of examples.
+ * @param examples the examples that will be displayed to the user when they press the TAB key.
+ */
+sealed case class FixedSetExamples(examples: Iterable[String]) extends ExampleSource
+{
+	override def withAddedPrefix(addedPrefix: String): ExampleSource = FixedSetExamples(examplesWithRemovedPrefix(addedPrefix))
+
+	override def apply(): Iterable[String] = examples
+
+	private def examplesWithRemovedPrefix(prefix: String) = examples.collect {
+		case example if example startsWith prefix => example substring prefix.length
+	}
+}
+
+/**
+ * Provides path completion examples based on files in the base directory.
+ * @param base the directory within which this class will search for completion examples.
+ * @param prefix the part of the path already written by the user.
+ */
+class FileExamples(base: File, prefix: String = "") extends ExampleSource
+{
+	override def apply(): Stream[String] = files(base).map(_ substring prefix.length)
+
+	override def withAddedPrefix(addedPrefix: String): FileExamples = new FileExamples(base, prefix + addedPrefix)
+
+	protected def files(directory: File): Stream[String] = {
+		val childPaths = directory.listFiles().toStream
+		val prefixedDirectChildPaths = childPaths.map(relativize(base, _).get).filter(_ startsWith prefix)
+		val dirsToRecurseInto = childPaths.filter(_.isDirectory).map(relativize(base, _).get).filter(dirStartsWithPrefix)
+		prefixedDirectChildPaths append dirsToRecurseInto.flatMap(dir => files(new File(base, dir)))
+	}
+
+	private def dirStartsWithPrefix(relativizedPath: String): Boolean =
+		(relativizedPath startsWith prefix) || (prefix startsWith relativizedPath)
+}

util/complete/src/main/scala/sbt/complete/Parser.scala

@@ -4,7 +4,7 @@
 package sbt.complete
 
 	import Parser._
-	import sbt.Types.{const, left, right, some}
+	import sbt.Types.{left, right, some}
 	import sbt.Util.{makeList,separate}
 
 /** A String parser that provides semi-automatic tab completion.
@@ -70,7 +70,7 @@ sealed trait RichParser[A]
 	/** If an exception is thrown by the original Parser,
 	* capture it and fail locally instead of allowing the exception to propagate up and terminate parsing.*/
 	def failOnException: Parser[A]
-	
+
 	@deprecated("Use `not` and explicitly provide the failure message", "0.12.2")
 	def unary_- : Parser[Unit]
 
@@ -87,6 +87,24 @@ sealed trait RichParser[A]
 	/** Explicitly defines the completions for the original Parser.*/
 	def examples(s: Set[String], check: Boolean = false): Parser[A]
 
+	/**
+	 * @param exampleSource the source of examples when displaying completions to the user.
+	 * @param maxNumberOfExamples limits the number of examples that the source of examples should return. This can
+	 *                            prevent lengthy pauses and avoids bad interactive user experience.
+	 * @param removeInvalidExamples indicates whether completion examples should be checked for validity (against the
+	 *                              given parser). Invalid examples will be filtered out and only valid suggestions will
+	 *                              be displayed.
+	 * @return a new parser with a new source of completions.
+	 */
+	def examples(exampleSource: ExampleSource, maxNumberOfExamples: Int, removeInvalidExamples: Boolean): Parser[A]
+
+	/**
+	 * @param exampleSource the source of examples when displaying completions to the user.
+	 * @return a new parser with a new source of completions. It displays at most 25 completion examples and does not
+	 *         remove invalid examples.
+	 */
+	def examples(exampleSource: ExampleSource): Parser[A] = examples(exampleSource, maxNumberOfExamples = 25, removeInvalidExamples = false)
+
 	/** Converts a Parser returning a Char sequence to a Parser returning a String.*/
 	def string(implicit ev: A <:< Seq[Char]): Parser[String]
 
@@ -156,6 +174,7 @@ object Parser extends ParserMain
 	def mkFailures(errors: => Seq[String], definitive: Boolean = false): Failure = new Failure(errors.distinct, definitive)
 	def mkFailure(error: => String, definitive: Boolean = false): Failure = new Failure(error :: Nil, definitive)
 
+	@deprecated("This method is deprecated and will be removed in the next major version. Use the parser directly to check for invalid completions.", since = "0.13.2")
 	def checkMatches(a: Parser[_], completions: Seq[String])
 	{
 		val bad = completions.filter( apply(a)(_).resultEmpty.isFailure)
@@ -165,7 +184,6 @@ object Parser extends ParserMain
 	def tuple[A,B](a: Option[A], b: Option[B]): Option[(A,B)] =
 		(a,b) match { case (Some(av), Some(bv)) => Some((av, bv)); case _ => None }
 
-
 	def mapParser[A,B](a: Parser[A], f: A => B): Parser[B] =
 		a.ifValid {
 			a.result match
@@ -239,7 +257,7 @@ object Parser extends ParserMain
 						case None => if(max.isZero) success(revAcc.reverse) else new Repeat(partial, repeated, min, max, revAcc)
 					}
 			}
-		
+
 		partial match
 		{
 			case Some(part) =>
@@ -284,7 +302,8 @@ trait ParserMain
 		def & (o: Parser[_]) = and(a, o)
 		def - (o: Parser[_]) = sub(a, o)
 		def examples(s: String*): Parser[A] = examples(s.toSet)
-		def examples(s: Set[String], check: Boolean = false): Parser[A] = Parser.examples(a, s, check)
+		def examples(s: Set[String], check: Boolean = false): Parser[A] = examples(new FixedSetExamples(s), s.size, check)
+		def examples(s: ExampleSource, maxNumberOfExamples: Int, removeInvalidExamples: Boolean): Parser[A] = Parser.examples(a, s, maxNumberOfExamples, removeInvalidExamples)
 		def filter(f: A => Boolean, msg: String => String): Parser[A] = filterParser(a, f, "", msg)
 		def string(implicit ev: A <:< Seq[Char]): Parser[String] = map(_.mkString)
 		def flatMap[B](f: A => Parser[B]) = bindParser(a, f)
@@ -295,7 +314,7 @@ trait ParserMain
 
 	/** Construct a parser that is valid, but has no valid result.  This is used as a way
 	* to provide a definitive Failure when a parser doesn't match empty input.  For example,
-	* in `softFailure(...) | p`, if `p` doesn't match the empty sequence, the failure will come 
+	* in `softFailure(...) | p`, if `p` doesn't match the empty sequence, the failure will come
 	* from the Parser constructed by the `softFailure` method. */
 	private[sbt] def softFailure(msg: => String, definitive: Boolean = false): Parser[Nothing] =
 		SoftInvalid( mkFailures(msg :: Nil, definitive) )
@@ -419,13 +438,26 @@ trait ParserMain
 		apply(p)(s).completions(level) x Completions.empty
 
 	def examples[A](a: Parser[A], completions: Set[String], check: Boolean = false): Parser[A] =
+		examples(a, new FixedSetExamples(completions), completions.size, check)
+
+	/**
+	 * @param a the parser to decorate with a source of examples. All validation and parsing is delegated to this parser,
+	 *          only [[Parser.completions]] is modified.
+	 * @param completions the source of examples when displaying completions to the user.
+	 * @param maxNumberOfExamples limits the number of examples that the source of examples should return. This can
+	 *                            prevent lengthy pauses and avoids bad interactive user experience.
+	 * @param removeInvalidExamples indicates whether completion examples should be checked for validity (against the given parser). An
+	 *                              exception is thrown if the example source contains no valid completion suggestions.
+	 * @tparam A the type of values that are returned by the parser.
+	 * @return
+	 */
+	def examples[A](a: Parser[A], completions: ExampleSource, maxNumberOfExamples: Int, removeInvalidExamples: Boolean): Parser[A] =
 		if(a.valid) {
 			a.result match
 			{
 				case Some(av) => success( av )
 				case None =>
-					if(check) checkMatches(a, completions.toSeq)
-					new Examples(a, completions)
+					new ParserWithExamples(a, completions, maxNumberOfExamples, removeInvalidExamples)
 			}
 		}
 		else a
@@ -442,7 +474,7 @@ trait ParserMain
 		}
 
 	/** Establishes delegate parser `t` as a single token of tab completion.
-	* When tab completion of part of this token is requested, the completions provided by the delegate `t` or a later derivative are appended to 
+	* When tab completion of part of this token is requested, the completions provided by the delegate `t` or a later derivative are appended to
 	* the prefix String already seen by this parser. */
 	def token[T](t: Parser[T]): Parser[T] = token(t, TokenCompletions.default)
 
@@ -691,17 +723,52 @@ private final class Not(delegate: Parser[_], failMessage: String) extends ValidP
 	}
 	override def toString = " -(%s)".format(delegate)
 }
-private final class Examples[T](delegate: Parser[T], fixed: Set[String]) extends ValidParser[T]
+
+/**
+ * This class wraps an existing parser (the delegate), and replaces the delegate's completions with examples from
+ * the given example source.
+ *
+ * This class asks the example source for a limited amount of examples (to prevent lengthy and expensive
+ * computations and large amounts of allocated data). It then passes these examples on to the UI.
+ *
+ * @param delegate the parser to decorate with completion examples (i.e., completion of user input).
+ * @param exampleSource the source from which this class will take examples (potentially filter them with the delegate
+ *                      parser), and pass them to the UI.
+ * @param maxNumberOfExamples the maximum number of completions to read from the example source and pass to the UI. This
+ *                            limit prevents lengthy example generation and allocation of large amounts of memory.
+ * @param removeInvalidExamples indicates whether to remove examples that are deemed invalid by the delegate parser.
+ * @tparam T the type of value produced by the parser.
+ */
+private final class ParserWithExamples[T](delegate: Parser[T], exampleSource: ExampleSource, maxNumberOfExamples: Int, removeInvalidExamples: Boolean) extends ValidParser[T]
 {
-	def derive(c: Char) = examples(delegate derive c, fixed.collect { case x if x.length > 0 && x(0) == c => x substring 1 })
+	def derive(c: Char) =
+		examples(delegate derive c, exampleSource.withAddedPrefix(c.toString), maxNumberOfExamples, removeInvalidExamples)
+
 	def result = delegate.result
+
 	lazy val resultEmpty = delegate.resultEmpty
-	def completions(level: Int) =
-		if(fixed.isEmpty)
+
+	def completions(level: Int) = {
+		if(exampleSource().isEmpty)
 			if(resultEmpty.isValid) Completions.nil else Completions.empty
+		else {
+			val examplesBasedOnTheResult = filteredExamples.take(maxNumberOfExamples).toSet
+			Completions(examplesBasedOnTheResult.map(ex => Completion.suggestion(ex)))
+		}
+	}
+
+	override def toString = "examples(" + delegate + ", " + exampleSource().take(2).toList + ")"
+
+	private def filteredExamples: Iterable[String] = {
+		if (removeInvalidExamples)
+			exampleSource().filter(isExampleValid)
 		else
-			Completions(fixed map(f => Completion.suggestion(f)) )
-	override def toString = "examples(" + delegate + ", " + fixed.take(2) + ")"
+			exampleSource()
+	}
+
+	private def isExampleValid(example: String): Boolean = {
+		apply(delegate)(example).resultEmpty.isValid
+	}
 }
 private final class StringLiteral(str: String, start: Int) extends ValidParser[String]
 {

util/complete/src/main/scala/sbt/complete/Parsers.scala

@@ -78,7 +78,7 @@ trait Parsers
 	def isScalaIDChar(c: Char) = c.isLetterOrDigit || c == '_'
 
 	def isDelimiter(c: Char) = c match { case '`' | '\'' | '\"' | /*';' | */',' | '.' => true ; case _ => false }
-	
+
 	/** Matches a single character that is not a whitespace character. */
 	lazy val NotSpaceClass = charClass(!_.isWhitespace, "non-whitespace character")
 
@@ -128,8 +128,15 @@ trait Parsers
 	/** Returns true if `c` is an ASCII letter or digit. */
 	def alphanum(c: Char) = ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || ('0' <= c && c <= '9')
 
-	// TODO: implement
-	def fileParser(base: File): Parser[File] = token(mapOrFail(NotSpace)(s => new File(s.mkString)), "<file>")
+	/**
+	 * @param base the directory used for completion proposals (when the user presses the TAB key). Only paths under this
+	 *             directory will be proposed.
+	 * @return the file that was parsed from the input string. The returned path may or may not exist.
+	 */
+	def fileParser(base: File): Parser[File] =
+		OptSpace ~> StringBasic
+			.examples(new FileExamples(base))
+			.map(new File(_))
 
 	/** Parses a port number.  Currently, this accepts any integer and presents a tab completion suggestion of `<port>`. */
 	lazy val Port = token(IntBasic, "<port>")
@@ -153,7 +160,7 @@ trait Parsers
 	/** Parses a verbatim quoted String value, discarding the quotes in the result.  This kind of quoted text starts with triple quotes `"""`
 	* and ends at the next triple quotes and may contain any character in between. */
 	lazy val StringVerbatim: Parser[String] = VerbatimDQuotes ~>
-		any.+.string.filter(!_.contains(VerbatimDQuotes), _ => "Invalid verbatim string") <~ 
+		any.+.string.filter(!_.contains(VerbatimDQuotes), _ => "Invalid verbatim string") <~
 		VerbatimDQuotes
 
 	/** Parses a string value, interpreting escapes and discarding the surrounding quotes in the result.
@@ -168,7 +175,7 @@ trait Parsers
 	  BackslashChar ~> ('b' ^^^ '\b' | 't' ^^^ '\t' | 'n' ^^^ '\n' | 'f' ^^^ '\f' | 'r' ^^^ '\r' |
 	  '\"' ^^^ '\"' | '\'' ^^^ '\'' | '\\' ^^^ '\\' | UnicodeEscape)
 
-	/** Parses a single unicode escape sequence into the represented Char. 
+	/** Parses a single unicode escape sequence into the represented Char.
 	* A unicode escape begins with a backslash, followed by a `u` and 4 hexadecimal digits representing the unicode value. */
 	lazy val UnicodeEscape: Parser[Char] =
 	  ("u" ~> repeat(HexDigit, 4, 4)) map { seq => Integer.parseInt(seq.mkString, 16).toChar }

util/complete/src/test/scala/ParserTest.scala

@@ -118,7 +118,8 @@ object ParserExample
 
 	val name = token("test")
 	val options = (ws ~> token("quick" | "failed" | "new") )*
-	val include = (ws ~> token(examples(notws.string, Set("am", "is", "are", "was", "were") )) )*
+	val exampleSet = Set("am", "is", "are", "was", "were")
+	val include = (ws ~> token(examples(notws.string, new FixedSetExamples(exampleSet), exampleSet.size, false )) )*
 
 	val t = name ~ options ~ include
 

util/complete/src/test/scala/sbt/complete/FileExamplesTest.scala

@@ -0,0 +1,92 @@
+package sbt.complete
+
+import org.specs2.mutable.Specification
+import org.specs2.specification.Scope
+import sbt.IO.withTemporaryDirectory
+import java.io.File
+import sbt.IO._
+
+class FileExamplesTest extends Specification
+{
+
+	"listing all files in an absolute base directory" should {
+		"produce the entire base directory's contents" in new directoryStructure {
+			fileExamples().toList should containTheSameElementsAs(allRelativizedPaths)
+		}
+	}
+
+	"listing files with a prefix that matches none" should {
+		"produce an empty list" in new directoryStructure(withCompletionPrefix = "z") {
+			fileExamples().toList should beEmpty
+		}
+	}
+
+	"listing single-character prefixed files" should {
+		"produce matching paths only" in new directoryStructure(withCompletionPrefix = "f") {
+			fileExamples().toList should containTheSameElementsAs(prefixedPathsOnly)
+		}
+	}
+
+	"listing directory-prefixed files" should {
+		"produce matching paths only" in new directoryStructure(withCompletionPrefix = "far") {
+			fileExamples().toList should containTheSameElementsAs(prefixedPathsOnly)
+		}
+
+		"produce sub-dir contents only when appending a file separator to the directory" in new directoryStructure(withCompletionPrefix = "far" + File.separator) {
+			fileExamples().toList should containTheSameElementsAs(prefixedPathsOnly)
+		}
+	}
+
+	"listing files with a sub-path prefix" should {
+		"produce matching paths only" in new directoryStructure(withCompletionPrefix = "far" + File.separator + "ba") {
+			fileExamples().toList should containTheSameElementsAs(prefixedPathsOnly)
+		}
+	}
+
+	"completing a full path" should {
+		"produce a list with an empty string" in new directoryStructure(withCompletionPrefix = "bazaar") {
+			fileExamples().toList shouldEqual List("")
+		}
+	}
+
+	class directoryStructure(withCompletionPrefix: String = "") extends Scope with DelayedInit
+	{
+		var fileExamples: FileExamples = _
+		var baseDir: File = _
+		var childFiles: List[File] = _
+		var childDirectories: List[File] = _
+		var nestedFiles: List[File] = _
+		var nestedDirectories: List[File] = _
+
+		def allRelativizedPaths: List[String] =
+			(childFiles ++ childDirectories ++ nestedFiles ++ nestedDirectories).map(relativize(baseDir, _).get)
+
+		def prefixedPathsOnly: List[String] =
+			allRelativizedPaths.filter(_ startsWith withCompletionPrefix).map(_ substring withCompletionPrefix.length)
+
+		override def delayedInit(testBody: => Unit): Unit = {
+			withTemporaryDirectory {
+				tempDir =>
+					createSampleDirStructure(tempDir)
+					fileExamples = new FileExamples(baseDir, withCompletionPrefix)
+					testBody
+			}
+		}
+
+		private def createSampleDirStructure(tempDir: File): Unit = {
+			childFiles = toChildFiles(tempDir, List("foo", "bar", "bazaar"))
+			childDirectories = toChildFiles(tempDir, List("moo", "far"))
+			nestedFiles = toChildFiles(childDirectories(1), List("farfile1", "barfile2"))
+			nestedDirectories = toChildFiles(childDirectories(1), List("fardir1", "bardir2"))
+
+			(childDirectories ++ nestedDirectories).map(_.mkdirs())
+			(childFiles ++ nestedFiles).map(_.createNewFile())
+
+			// NOTE: Creating a new file here because `tempDir.listFiles()` returned an empty list.
+			baseDir = new File(tempDir.getCanonicalPath)
+		}
+
+		private def toChildFiles(baseDir: File, files: List[String]): List[File] = files.map(new File(baseDir, _))
+	}
+
+}

util/complete/src/test/scala/sbt/complete/FixedSetExamplesTest.scala

@@ -0,0 +1,26 @@
+package sbt.complete
+
+import org.specs2.mutable.Specification
+import org.specs2.specification.Scope
+
+class FixedSetExamplesTest extends Specification {
+
+	"adding a prefix" should {
+		"produce a smaller set of examples with the prefix removed" in new examples {
+			fixedSetExamples.withAddedPrefix("f")() must containTheSameElementsAs(List("oo", "ool", "u"))
+			fixedSetExamples.withAddedPrefix("fo")() must containTheSameElementsAs(List("o", "ol"))
+			fixedSetExamples.withAddedPrefix("b")() must containTheSameElementsAs(List("ar"))
+		}
+	}
+
+	"without a prefix" should {
+		"produce the original set" in new examples {
+			fixedSetExamples() mustEqual exampleSet
+		}
+	}
+
+	trait examples extends Scope {
+		val exampleSet = List("foo", "bar", "fool", "fu")
+		val fixedSetExamples = FixedSetExamples(exampleSet)
+	}
+}

util/complete/src/test/scala/sbt/complete/ParserWithExamplesTest.scala

@@ -0,0 +1,93 @@
+package sbt.complete
+
+import org.specs2.mutable.Specification
+import org.specs2.specification.Scope
+import Completion._
+
+class ParserWithExamplesTest extends Specification {
+
+	"listing a limited number of completions" should {
+		"grab only the needed number of elements from the iterable source of examples" in new parserWithLazyExamples {
+			parserWithExamples.completions(0)
+			examples.size shouldEqual maxNumberOfExamples
+		}
+	}
+
+	"listing only valid completions" should {
+		"use the delegate parser to remove invalid examples" in new parserWithValidExamples {
+			val validCompletions = Completions(Set(
+				suggestion("blue"),
+				suggestion("red")
+			))
+			parserWithExamples.completions(0) shouldEqual validCompletions
+		}
+	}
+
+	"listing valid completions in a derived parser" should {
+		"produce only valid examples that start with the character of the derivation" in new parserWithValidExamples {
+			val derivedCompletions = Completions(Set(
+				suggestion("lue")
+			))
+			parserWithExamples.derive('b').completions(0) shouldEqual derivedCompletions
+		}
+	}
+
+	"listing valid and invalid completions" should {
+		"produce the entire source of examples" in new parserWithAllExamples {
+			val completions = Completions(examples.map(suggestion(_)).toSet)
+			parserWithExamples.completions(0) shouldEqual completions
+		}
+	}
+
+	"listing valid and invalid completions in a derived parser" should {
+		"produce only examples that start with the character of the derivation" in new parserWithAllExamples {
+			val derivedCompletions = Completions(Set(
+				suggestion("lue"),
+				suggestion("lock")
+			))
+			parserWithExamples.derive('b').completions(0) shouldEqual derivedCompletions
+		}
+	}
+
+	class parserWithLazyExamples extends parser(GrowableSourceOfExamples(), maxNumberOfExamples = 5, removeInvalidExamples = false)
+
+	class parserWithValidExamples extends parser(removeInvalidExamples = true)
+
+	class parserWithAllExamples extends parser(removeInvalidExamples = false)
+
+	case class parser(examples: Iterable[String] = Set("blue", "yellow", "greeen", "block", "red"),
+										maxNumberOfExamples: Int = 25,
+										removeInvalidExamples: Boolean) extends Scope {
+
+		import DefaultParsers._
+
+		val colorParser = "blue" | "green" | "black" | "red"
+		val parserWithExamples: Parser[String] = new ParserWithExamples[String](
+			colorParser,
+			FixedSetExamples(examples),
+			maxNumberOfExamples,
+			removeInvalidExamples
+		)
+	}
+
+	case class GrowableSourceOfExamples() extends Iterable[String] {
+		private var numberOfIteratedElements: Int = 0
+
+		override def iterator: Iterator[String] = {
+			new Iterator[String] {
+				var currentElement = 0
+
+				override def next(): String = {
+					currentElement += 1
+					numberOfIteratedElements = Math.max(currentElement, numberOfIteratedElements)
+					numberOfIteratedElements.toString
+				}
+
+				override def hasNext: Boolean = true
+			}
+		}
+
+		override def size: Int = numberOfIteratedElements
+	}
+
+}