From 7eafcaf544e90e19303519859ab7b407ec0d48d3 Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Sat, 24 Oct 2020 12:57:22 -0700 Subject: [PATCH] Strip ansi and color codes from terminal output It is possible for downstream dependencies to print or log messages containing ansi escape sequences and/or color codes. In older versions of sbt, these would be printed even if the user had disabled ansi codes or color via the sbt.log.noformat or sbt.color parameters. This commit adds a general api to EscHelpers that strips general ansi codes and color codes independently via flags. We can then use that api to ensure that all bytes written to System.out are stripped of ansi escape and color codes if the terminal properties demand this. The motivation was that JLine 3 will prepend the prompt string with \E[?2004h, which turns on bracketed paste mode (https://en.wikipedia.org/wiki/ANSI_escape_code). If the sbt shell is started with a terminal that doesn't support general ansi escape codes, such as the jEdit shell, ?2004h gets printed to the shell. To fix this, we can strip ansi codes from all output if the terminal doesn't support general ansic codes. This has the additional side effect of any ansi codes that appear in log messages or printlns that are added by non-sbt code will be stripped. It's unlikely that this is all that common. In addition to the JLine use case, I've noticed that utest prints colored output during test runs. Prior to this change, the colored output was present even when sbt was run with `-Dsbt.color=false` and after this change, the colors are correctly stripped. --- .../sbt/internal/util/ConsoleAppender.scala | 6 +- .../scala/sbt/internal/util/EscHelpers.scala | 76 +++++++++++++++---- .../main/scala/sbt/internal/util/JLine3.scala | 7 +- .../scala/sbt/internal/util/Terminal.scala | 9 ++- .../sbt/internal/util/CleanStringSpec.scala | 42 ++++++++-- 5 files changed, 113 insertions(+), 27 deletions(-) diff --git a/internal/util-logging/src/main/scala/sbt/internal/util/ConsoleAppender.scala b/internal/util-logging/src/main/scala/sbt/internal/util/ConsoleAppender.scala index c248eb484..55b0fe5d8 100644 --- a/internal/util-logging/src/main/scala/sbt/internal/util/ConsoleAppender.scala +++ b/internal/util-logging/src/main/scala/sbt/internal/util/ConsoleAppender.scala @@ -495,8 +495,10 @@ trait Appender extends AutoCloseable { // the output may have unwanted colors but it would still be legible. This should // only be relevant if the log message string itself contains ansi escape sequences // other than color codes which is very unlikely. - val toWrite = if (!ansiCodesSupported) { - if (useFormat) EscHelpers.stripMoves(msg) else EscHelpers.removeEscapeSequences(msg) + val toWrite = if (!ansiCodesSupported || !useFormat && msg.getBytes.contains(27.toByte)) { + val (bytes, len) = + EscHelpers.strip(msg.getBytes, stripAnsi = !ansiCodesSupported, stripColor = !useFormat) + new String(bytes, 0, len) } else msg out.println(toWrite) } diff --git a/internal/util-logging/src/main/scala/sbt/internal/util/EscHelpers.scala b/internal/util-logging/src/main/scala/sbt/internal/util/EscHelpers.scala index 19af0024a..ee84680c6 100644 --- a/internal/util-logging/src/main/scala/sbt/internal/util/EscHelpers.scala +++ b/internal/util-logging/src/main/scala/sbt/internal/util/EscHelpers.scala @@ -126,32 +126,76 @@ object EscHelpers { } index } - def stripMoves(s: String): String = { - val bytes = s.getBytes + + /** + * Strips ansi escape and color codes from an input string. + * + * @param bytes the input bytes + * @param stripAnsi toggles whether or not to remove general ansi escape codes + * @param stripColor toggles whether or not to remove ansi color codes + * @return a string with the escape and color codes removed depending on the input + * parameter along with the length of the output string (which may be smaller than + * the returned array) + */ + def strip(bytes: Array[Byte], stripAnsi: Boolean, stripColor: Boolean): (Array[Byte], Int) = { val res = Array.fill[Byte](bytes.length)(0) + var i = 0 var index = 0 - var lastEscapeIndex = -1 var state = 0 - def set(b: Byte) = { - res(index) = b - index += 1 - } + var limit = 0 + val digit = new ArrayBuffer[Byte] + var leftDigit = -1 + var escIndex = -1 bytes.foreach { b => - set(b) + if (index < res.length) res(index) = b + index += 1 + limit = math.max(limit, index) + if (state == 0) escIndex = -1 b match { case 27 => + escIndex = index - 1 state = esc - lastEscapeIndex = math.max(0, index) - case b if b == '[' && state == esc => state = csi - case 'm' => state = 0 - case b if state == csi && (b < 48 || b >= 58) && b != ';' => + case b if (state == esc || state == csi) && b >= 48 && b < 58 => + state = csi + digit += b + case '[' if state == esc => state = csi + case 8 => state = 0 - index = math.max(0, lastEscapeIndex - 1) - case b => + index = math.max(index - 1, 0) + case b if state == csi => + leftDigit = Try(new String(digit.toArray).toInt).getOrElse(0) + state = 0 + b.toChar match { + case 'h' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'J' | 'K' => + if (stripAnsi) index = math.max(escIndex, 0) + case 'm' => if (stripColor) index = escIndex + case ';' | 's' | 'u' | '?' => state = csi + case b => + } + digit.clear() + case b if state == esc => state = 0 + case b => } } - new String(res, 0, index) + (res, index) } + @deprecated("use EscHelpers.strip", "1.4.2") + def stripMoves(s: String): String = { + val (bytes, len) = strip(s.getBytes, stripAnsi = true, stripColor = false) + new String(bytes, 0, len) + } + + /** + * Removes the ansi escape sequences from a string and makes a best attempt at + * calculating any ansi moves by hand. For example, if the string contains + * a backspace character followed by a character, the output string would + * replace the character preceding the backspaces with the character proceding it. + * This is in contrast to `strip` which just removes all ansi codes entirely. + * + * @param s the input string + * @return a string containing the original characters of the input stream with + * the ansi escape codes removed. + */ def stripColorsAndMoves(s: String): String = { val bytes = s.getBytes val res = Array.fill[Byte](bytes.length)(0) @@ -174,6 +218,7 @@ object EscHelpers { leftDigit = Try(new String(digit.toArray).toInt).getOrElse(0) state = 0 b.toChar match { + case 'h' => index = math.max(index - 1, 0) case 'D' => index = math.max(index - leftDigit, 0) case 'C' => index = math.min(limit, math.min(index + leftDigit, res.length - 1)) case 'K' | 'J' => @@ -190,6 +235,7 @@ object EscHelpers { index += 1 limit = math.max(limit, index) } + (res, limit) new String(res, 0, limit) } diff --git a/internal/util-logging/src/main/scala/sbt/internal/util/JLine3.scala b/internal/util-logging/src/main/scala/sbt/internal/util/JLine3.scala index a80d81410..9de086fd3 100644 --- a/internal/util-logging/src/main/scala/sbt/internal/util/JLine3.scala +++ b/internal/util-logging/src/main/scala/sbt/internal/util/JLine3.scala @@ -124,7 +124,12 @@ private[sbt] object JLine3 { override val output: OutputStream = new OutputStream { override def write(b: Int): Unit = write(Array[Byte](b.toByte)) override def write(b: Array[Byte]): Unit = if (!closed.get) term.withPrintStream { ps => - ps.write(b) + val (toWrite, len) = if (b.contains(27.toByte)) { + if (!term.isAnsiSupported || !term.isColorEnabled) { + EscHelpers.strip(b, !term.isAnsiSupported, !term.isColorEnabled) + } else (b, b.length) + } else (b, b.length) + if (len == toWrite.length) ps.write(toWrite) else ps.write(toWrite, 0, len) term.prompt match { case a: Prompt.AskUser => a.write(b) case _ => diff --git a/internal/util-logging/src/main/scala/sbt/internal/util/Terminal.scala b/internal/util-logging/src/main/scala/sbt/internal/util/Terminal.scala index 6e1ddf277..7ff0c228a 100644 --- a/internal/util-logging/src/main/scala/sbt/internal/util/Terminal.scala +++ b/internal/util-logging/src/main/scala/sbt/internal/util/Terminal.scala @@ -930,7 +930,14 @@ object Terminal { } override def flush(): Unit = combinedOutputStream.flush() } - private def doWrite(bytes: Array[Byte]): Unit = withPrintStream { ps => + private def doWrite(rawBytes: Array[Byte]): Unit = withPrintStream { ps => + val (toWrite, len) = + if (rawBytes.contains(27.toByte)) { + if (!isAnsiSupported || !isColorEnabled) + EscHelpers.strip(rawBytes, stripAnsi = !isAnsiSupported, stripColor = !isColorEnabled) + else (rawBytes, rawBytes.length) + } else (rawBytes, rawBytes.length) + val bytes = if (len < toWrite.length) toWrite.take(len) else toWrite progressState.write(TerminalImpl.this, bytes, ps, hasProgress.get && !rawMode.get) } override private[sbt] val printStream: PrintStream = new LinePrintStream(outputStream) diff --git a/internal/util-logging/src/test/scala/sbt/internal/util/CleanStringSpec.scala b/internal/util-logging/src/test/scala/sbt/internal/util/CleanStringSpec.scala index 9939d668f..f734923a6 100644 --- a/internal/util-logging/src/test/scala/sbt/internal/util/CleanStringSpec.scala +++ b/internal/util-logging/src/test/scala/sbt/internal/util/CleanStringSpec.scala @@ -45,23 +45,25 @@ class CleanStringSpec extends FlatSpec { } it should "remove moves in string with only moves" in { val original = - new String(Array[Byte](27, 91, 50, 75, 27, 91, 51, 65, 27, 91, 49, 48, 48, 48, 68)) - assert(EscHelpers.stripMoves(original) == "") + Array[Byte](27, 91, 50, 75, 27, 91, 51, 65, 27, 91, 49, 48, 48, 48, 68) + val (bytes, len) = EscHelpers.strip(original, stripAnsi = true, stripColor = true) + assert(len == 0) } it should "remove moves in string with moves and letters" in { - val original = new String( + val original = Array[Byte](27, 91, 50, 75, 27, 91, 51, 65) ++ "foo".getBytes ++ Array[Byte](27, 91, 49, 48, 48, 48, 68) - ) - assert(EscHelpers.stripMoves(original) == "foo") + val (bytes, len) = EscHelpers.strip(original, stripAnsi = true, stripColor = true) + assert(new String(bytes, 0, len) == "foo") } it should "preserve colors" in { - val original = new String( + val original = Array[Byte](27, 91, 49, 48, 48, 48, 68, 27, 91, 48, 74, 102, 111, 111, 27, 91, 51, 54, 109, 62, 32, 27, 91, 48, 109) - ) // this is taken from an sbt prompt that looks like "foo> " with the > rendered blue + // this is taken from an sbt prompt that looks like "foo> " with the > rendered blue val colorArrow = new String(Array[Byte](27, 91, 51, 54, 109, 62)) - assert(EscHelpers.stripMoves(original) == "foo" + colorArrow + " " + scala.Console.RESET) + val (bytes, len) = EscHelpers.strip(original, stripAnsi = true, stripColor = false) + assert(new String(bytes, 0, len) == "foo" + colorArrow + " " + scala.Console.RESET) } it should "remove unusual escape characters" in { val original = new String( @@ -70,4 +72,28 @@ class CleanStringSpec extends FlatSpec { ) assert(EscHelpers.stripColorsAndMoves(original).isEmpty) } + it should "remove bracketed paste csi" in { + // taken from a test project prompt + val original = + Array[Byte](27, 91, 63, 50, 48, 48, 52, 104, 115, 98, 116, 58, 114, 101, 112, 114, 111, 62, + 32) + val (bytes, len) = EscHelpers.strip(original, stripAnsi = true, stripColor = false) + assert(new String(bytes, 0, len) == "sbt:repro> ") + } + it should "strip colors" in { + // taken from utest output + val original = + Array[Byte](91, 105, 110, 102, 111, 93, 32, 27, 91, 51, 50, 109, 43, 27, 91, 51, 57, 109, 32, + 99, 111, 109, 46, 97, 99, 109, 101, 46, 67, 111, 121, 111, 116, 101, 84, 101, 115, 116, 46, + 109, 97, 107, 101, 84, 114, 97, 112, 32, 27, 91, 50, 109, 57, 109, 115, 27, 91, 48, 109, 32, + 32, 27, 91, 48, 74, 10) + val (bytes, len) = EscHelpers.strip(original, stripAnsi = false, stripColor = true) + val expected = "[info] + com.acme.CoyoteTest.makeTrap 9ms " + + new String(Array[Byte](27, 91, 48, 74, 10)) + assert(new String(bytes, 0, len) == expected) + + val (bytes2, len2) = EscHelpers.strip(original, stripAnsi = true, stripColor = true) + val expected2 = "[info] + com.acme.CoyoteTest.makeTrap 9ms \n" + assert(new String(bytes2, 0, len2) == expected2) + } }