From e6c2ce084ac5f6a25ccab457ef2666307cf1877a Mon Sep 17 00:00:00 2001 From: Samuel CLARENC Date: Thu, 17 Jun 2021 09:00:15 +0200 Subject: [PATCH 1/4] Add the tag "integration-test" to the it configuration in BSP. --- protocol/src/main/scala/sbt/internal/bsp/BuildTargetTag.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/protocol/src/main/scala/sbt/internal/bsp/BuildTargetTag.scala b/protocol/src/main/scala/sbt/internal/bsp/BuildTargetTag.scala index e522fb482..fa1773f27 100644 --- a/protocol/src/main/scala/sbt/internal/bsp/BuildTargetTag.scala +++ b/protocol/src/main/scala/sbt/internal/bsp/BuildTargetTag.scala @@ -18,6 +18,7 @@ object BuildTargetTag { def fromConfig(config: String): Vector[String] = config match { case "test" => Vector(test) + case "it" => Vector(integrationTest) case "compile" => Vector(library) case _ => Vector.empty } From f82c0c4c5f5624c1e3375ac2821efad1266449aa Mon Sep 17 00:00:00 2001 From: Amina Adewusi Date: Fri, 18 Jun 2021 18:01:45 +0100 Subject: [PATCH 2/4] fixes remote caching not managing resource files What is the problem? When using remote caching, the resource files are not tracked so if they have changed, pullRemoteCache will deliver both the old resource file as well as the changed one. This is a problem, because it's not the behaviour that our users will expect and it's not in keeping with the contract of this feature. Why is this happening? Zinc, sbt's incremental compiler, keeps track of changes that have been made. It keeps this in what is called the Analysis file. However, resource files are not tracked in the Analysis file, so remote caching is not invalidating the unchanged resource file in place of the latest version. What is the solution? PullRemoteCache deletes all of the resources files. After this, copyResources is called by PackageBin, which puts the latest version of the resources back. --- .../main/scala/sbt/internal/RemoteCache.scala | 79 +++++++++++++++---- 1 file changed, 65 insertions(+), 14 deletions(-) diff --git a/main/src/main/scala/sbt/internal/RemoteCache.scala b/main/src/main/scala/sbt/internal/RemoteCache.scala index 6e46f607f..f116f5eb4 100644 --- a/main/src/main/scala/sbt/internal/RemoteCache.scala +++ b/main/src/main/scala/sbt/internal/RemoteCache.scala @@ -10,29 +10,34 @@ package internal import java.io.File import java.nio.file.Path -import Keys._ -import SlashSyntax0._ -import ScopeFilter.Make._ -import Project._ // for tag and inTask() -import org.apache.ivy.core.module.descriptor.{ Artifact => IArtifact, DefaultArtifact } -import org.apache.ivy.core.resolve.DownloadOptions +import org.apache.ivy.core.module.descriptor.{ DefaultArtifact, Artifact => IArtifact } import org.apache.ivy.core.report.DownloadStatus +import org.apache.ivy.core.resolve.DownloadOptions import org.apache.ivy.plugins.resolver.DependencyResolver -import std.TaskExtra._ // for join +import sbt.Defaults.prefix +import sbt.Keys._ +import sbt.Project._ +import sbt.ScopeFilter.Make._ +import sbt.SlashSyntax0._ import sbt.coursierint.LMCoursier +import sbt.internal.inc.{ HashUtil, JarUtils } +import sbt.internal.librarymanagement._ +import sbt.internal.remotecache._ +import sbt.io.IO +import sbt.io.Path.{ flat, rebase } +import sbt.io.syntax._ import sbt.librarymanagement._ import sbt.librarymanagement.ivy.{ Credentials, IvyPaths, UpdateOptions } import sbt.librarymanagement.syntax._ import sbt.nio.FileStamp import sbt.nio.Keys.{ inputFileStamps, outputFileStamps } -import sbt.internal.librarymanagement._ -import sbt.io.IO -import sbt.io.syntax._ -import sbt.internal.remotecache._ -import sbt.internal.inc.{ HashUtil, JarUtils } +import sbt.std.TaskExtra._ import sbt.util.InterfaceUtil.toOption import sbt.util.Logger +import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter } +import xsbti.FileConverter + import scala.annotation.nowarn object RemoteCache { @@ -152,6 +157,25 @@ object RemoteCache { ) ++ inConfig(Compile)(configCacheSettings(compileArtifact(Compile, cachedCompileClassifier))) ++ inConfig(Test)(configCacheSettings(testArtifact(Test, cachedTestClassifier)))) + def getResourceFilePaths() = Def.task { + import sbt.librarymanagement.LibraryManagementCodec._ + val t = classDirectory.value + val dirs = resourceDirectories.value.toSet + val flt: File => Option[File] = flat(t) + val cacheDirectory = crossTarget.value / (prefix(configuration.value.name) + "caches") + + val converter = fileConverter.value + val transform: File => Option[File] = (f: File) => rebase(dirs, t)(f).orElse(flt(f)) + val resourcesInClassesDir = resources.value + .flatMap(x => transform(x).toList) + .map(f => converter.toVirtualFile(f.toPath).toString) + val json = Converter.toJson[Seq[String]](resourcesInClassesDir).get + val tmp = CompactPrinter(json) + val file = cacheDirectory / "resources.json" + IO.write(file, tmp) + file + } + @nowarn def configCacheSettings[A <: RemoteCacheArtifact]( cacheArtifactTask: Def.Initialize[Task[A]] @@ -166,6 +190,10 @@ object RemoteCache { if (af.exists) { JarUtils.includeInJar(artp, Vector(af -> s"META-INF/inc_compile.zip")) } + val rf = getResourceFilePaths.value + if (rf.exists) { + JarUtils.includeInJar(artp, Vector(rf -> s"META-INF/resources.json")) + } // val testStream = (test / streams).?.value // testStream foreach { s => // val sf = Defaults.succeededFile(s.cacheDirectory) @@ -248,6 +276,7 @@ object RemoteCache { val smi = scalaModuleInfo.value val artifacts = (pushRemoteCacheConfiguration / remoteCacheArtifacts).value val nonPom = artifacts.filterNot(isPomArtifact).toVector + val converter = fileConverter.value m.withModule(log) { case (ivy, md, _) => val resolver = ivy.getSettings.getResolver(r.name) @@ -280,7 +309,7 @@ object RemoteCache { findJar(classifier, v, jars) match { case Some(jar) => - extractJar(art, jar) + extractJar(art, jar, converter) log.info(s"remote cache artifact extracted for $p $classifier") case None => @@ -368,11 +397,16 @@ object RemoteCache { jars.find(_.toString.endsWith(suffix)) } - private def extractJar(cacheArtifact: RemoteCacheArtifact, jar: File): Unit = + private def extractJar( + cacheArtifact: RemoteCacheArtifact, + jar: File, + converter: FileConverter + ): Unit = cacheArtifact match { case a: CompileRemoteCacheArtifact => extractCache(jar, a.extractDirectory, preserveLastModified = true) { output => extractAnalysis(output, a.analysisFile) + extractResourceList(output, converter) } case a: TestRemoteCacheArtifact => @@ -410,6 +444,23 @@ object RemoteCache { } } + private def extractResourceList(output: File, converter: FileConverter): Unit = { + import sbt.librarymanagement.LibraryManagementCodec._ + import sjsonnew.support.scalajson.unsafe.{ Converter, Parser } + import xsbti.VirtualFileRef + + val resourceFilesToDelete = output / "META-INF" / "resources.json" + if (resourceFilesToDelete.exists) { + val readFile = IO.read(resourceFilesToDelete) + val parseFile = Parser.parseUnsafe(readFile) + val resourceFiles = Converter.fromJsonUnsafe[Seq[String]](parseFile) + val paths = resourceFiles.map(f => converter.toPath(VirtualFileRef.of(f))) + val filesToDelete = paths.map(_.toFile) + for (file <- filesToDelete if file.getAbsolutePath.startsWith(output.getAbsolutePath)) + IO.delete(file) + } + } + private def extractTestResult(output: File, testResult: File): Unit = { //val expandedTestResult = output / "META-INF" / "succeeded_tests" //if (expandedTestResult.exists) { From d3398630410cb2db3219db064b5763f71e2f32bc Mon Sep 17 00:00:00 2001 From: Sebastian Alfers Date: Sun, 20 Jun 2021 15:41:59 +0200 Subject: [PATCH 3/4] Fix carriage return in supershell progress state --- .../sbt/internal/util/ProgressState.scala | 2 +- .../sbt/internal/util/ProgressStateSpec.scala | 41 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 internal/util-logging/src/test/scala/sbt/internal/util/ProgressStateSpec.scala diff --git a/internal/util-logging/src/main/scala/sbt/internal/util/ProgressState.scala b/internal/util-logging/src/main/scala/sbt/internal/util/ProgressState.scala index 37ba3ccab..5ea305920 100644 --- a/internal/util-logging/src/main/scala/sbt/internal/util/ProgressState.scala +++ b/internal/util-logging/src/main/scala/sbt/internal/util/ProgressState.scala @@ -107,7 +107,7 @@ private[sbt] final class ProgressState( val parts = new String(bytes, "UTF-8").split(System.lineSeparator) def appendLine(l: String, appendNewline: Boolean): Unit = { toWrite ++= l.getBytes("UTF-8") - toWrite ++= clearScreenBytes + if (!l.getBytes("UTF-8").endsWith("\r")) toWrite ++= clearScreenBytes if (appendNewline) toWrite ++= lineSeparatorBytes } parts.dropRight(1).foreach(appendLine(_, true)) diff --git a/internal/util-logging/src/test/scala/sbt/internal/util/ProgressStateSpec.scala b/internal/util-logging/src/test/scala/sbt/internal/util/ProgressStateSpec.scala new file mode 100644 index 000000000..4c3ae123a --- /dev/null +++ b/internal/util-logging/src/test/scala/sbt/internal/util/ProgressStateSpec.scala @@ -0,0 +1,41 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.internal.util + +import java.io.{ File, PrintStream } + +import org.scalatest.{ BeforeAndAfterAll, FlatSpec } +import sbt.internal.util.Terminal.SimpleTerminal + +import scala.io.Source + +class ProgressStateSpec extends FlatSpec with BeforeAndAfterAll { + + private lazy val fileIn = new File("/tmp/tmp.txt") + private lazy val fileOut = Source.fromFile("/tmp/tmp.txt") + + override def afterAll(): Unit = { + fileIn.delete() + fileOut.close() + super.afterAll() + } + + "test" should "not clear after carriage return (\\r) " in { + val ps = new ProgressState(1, 8) + val in = "Hello\r\nWorld".getBytes() + + ps.write(SimpleTerminal, in, new PrintStream(fileIn), hasProgress = true) + + val clearScreenBytes = ConsoleAppender.ClearScreenAfterCursor.getBytes("UTF-8") + val check = fileOut.getLines().toList.map { line => + line.getBytes("UTF-8").endsWith(clearScreenBytes) + } + + assert(check === List(false, true)) + } +} From 8628f95e66fff96a29570eee441f1414906f7dbc Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sun, 20 Jun 2021 17:32:42 -0400 Subject: [PATCH 4/4] lazily initialize ProxyTerminal to SimpleTerminal Problem ------- Console.systemOut is hooked up to Terminal.get, which internally calls ProxyTerminal, which lets us deffer the wiring of terminal to activeTerminal. This mechanism allows us to swap out the terminal capable of standard out forwarding for sbtn. However, as it stands this breaks the contract of being able to use Console.systemOut with wrapped inside of `Terminal.withStreams() {...}`. Solution -------- Check if `activeTerminal.get` returns `null`, and if so initialize it to the conventional `Terminal.SimpleTerminal`, which behaves as expected. --- .../src/main/scala/sbt/internal/util/Terminal.scala | 13 +++++++++++-- .../src/test/scala/ManagedLoggerSpec.scala | 4 ++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/internal/util-logging/src/main/scala/sbt/internal/util/Terminal.scala b/internal/util-logging/src/main/scala/sbt/internal/util/Terminal.scala index dac378cdf..2c2a6434c 100644 --- a/internal/util-logging/src/main/scala/sbt/internal/util/Terminal.scala +++ b/internal/util-logging/src/main/scala/sbt/internal/util/Terminal.scala @@ -346,7 +346,7 @@ object Terminal { consoleTerminalHolder.set(newConsoleTerminal()) if (hasVirtualIO) { hasProgress.set(isServer && isAnsiSupported) - activeTerminal.set(consoleTerminalHolder.get) + Terminal.set(consoleTerminalHolder.get) try withOut(withIn(f)) finally { jline.TerminalFactory.reset() @@ -382,7 +382,16 @@ object Terminal { } private[this] object ProxyTerminal extends Terminal { - private def t: Terminal = activeTerminal.get + private def t: Terminal = { + val current = activeTerminal.get + // if the activeTerminal is yet to be initialized on use, + // initialize to the conventional simple terminal for compatibility and testing + if (current ne null) current + else { + Terminal.set(Terminal.SimpleTerminal) + activeTerminal.get + } + } override private[sbt] def progressState: ProgressState = t.progressState override private[sbt] def enterRawMode(): Unit = t.enterRawMode() override private[sbt] def exitRawMode(): Unit = t.exitRawMode() diff --git a/internal/util-logging/src/test/scala/ManagedLoggerSpec.scala b/internal/util-logging/src/test/scala/ManagedLoggerSpec.scala index 9d617662f..d8b07e310 100644 --- a/internal/util-logging/src/test/scala/ManagedLoggerSpec.scala +++ b/internal/util-logging/src/test/scala/ManagedLoggerSpec.scala @@ -21,8 +21,8 @@ class ManagedLoggerSpec extends FlatSpec with Matchers { "ManagedLogger" should "log to console" in { val log = newLogger("foo") context.addAppender("foo", asyncStdout -> Level.Info) - log.info("test") - log.debug("test") + log.info("test_info") + log.debug("test_debug") } it should "support event logging" in {