From f82c0c4c5f5624c1e3375ac2821efad1266449aa Mon Sep 17 00:00:00 2001 From: Amina Adewusi Date: Fri, 18 Jun 2021 18:01:45 +0100 Subject: [PATCH] fixes remote caching not managing resource files What is the problem? When using remote caching, the resource files are not tracked so if they have changed, pullRemoteCache will deliver both the old resource file as well as the changed one. This is a problem, because it's not the behaviour that our users will expect and it's not in keeping with the contract of this feature. Why is this happening? Zinc, sbt's incremental compiler, keeps track of changes that have been made. It keeps this in what is called the Analysis file. However, resource files are not tracked in the Analysis file, so remote caching is not invalidating the unchanged resource file in place of the latest version. What is the solution? PullRemoteCache deletes all of the resources files. After this, copyResources is called by PackageBin, which puts the latest version of the resources back. --- .../main/scala/sbt/internal/RemoteCache.scala | 79 +++++++++++++++---- 1 file changed, 65 insertions(+), 14 deletions(-) diff --git a/main/src/main/scala/sbt/internal/RemoteCache.scala b/main/src/main/scala/sbt/internal/RemoteCache.scala index 6e46f607f..f116f5eb4 100644 --- a/main/src/main/scala/sbt/internal/RemoteCache.scala +++ b/main/src/main/scala/sbt/internal/RemoteCache.scala @@ -10,29 +10,34 @@ package internal import java.io.File import java.nio.file.Path -import Keys._ -import SlashSyntax0._ -import ScopeFilter.Make._ -import Project._ // for tag and inTask() -import org.apache.ivy.core.module.descriptor.{ Artifact => IArtifact, DefaultArtifact } -import org.apache.ivy.core.resolve.DownloadOptions +import org.apache.ivy.core.module.descriptor.{ DefaultArtifact, Artifact => IArtifact } import org.apache.ivy.core.report.DownloadStatus +import org.apache.ivy.core.resolve.DownloadOptions import org.apache.ivy.plugins.resolver.DependencyResolver -import std.TaskExtra._ // for join +import sbt.Defaults.prefix +import sbt.Keys._ +import sbt.Project._ +import sbt.ScopeFilter.Make._ +import sbt.SlashSyntax0._ import sbt.coursierint.LMCoursier +import sbt.internal.inc.{ HashUtil, JarUtils } +import sbt.internal.librarymanagement._ +import sbt.internal.remotecache._ +import sbt.io.IO +import sbt.io.Path.{ flat, rebase } +import sbt.io.syntax._ import sbt.librarymanagement._ import sbt.librarymanagement.ivy.{ Credentials, IvyPaths, UpdateOptions } import sbt.librarymanagement.syntax._ import sbt.nio.FileStamp import sbt.nio.Keys.{ inputFileStamps, outputFileStamps } -import sbt.internal.librarymanagement._ -import sbt.io.IO -import sbt.io.syntax._ -import sbt.internal.remotecache._ -import sbt.internal.inc.{ HashUtil, JarUtils } +import sbt.std.TaskExtra._ import sbt.util.InterfaceUtil.toOption import sbt.util.Logger +import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter } +import xsbti.FileConverter + import scala.annotation.nowarn object RemoteCache { @@ -152,6 +157,25 @@ object RemoteCache { ) ++ inConfig(Compile)(configCacheSettings(compileArtifact(Compile, cachedCompileClassifier))) ++ inConfig(Test)(configCacheSettings(testArtifact(Test, cachedTestClassifier)))) + def getResourceFilePaths() = Def.task { + import sbt.librarymanagement.LibraryManagementCodec._ + val t = classDirectory.value + val dirs = resourceDirectories.value.toSet + val flt: File => Option[File] = flat(t) + val cacheDirectory = crossTarget.value / (prefix(configuration.value.name) + "caches") + + val converter = fileConverter.value + val transform: File => Option[File] = (f: File) => rebase(dirs, t)(f).orElse(flt(f)) + val resourcesInClassesDir = resources.value + .flatMap(x => transform(x).toList) + .map(f => converter.toVirtualFile(f.toPath).toString) + val json = Converter.toJson[Seq[String]](resourcesInClassesDir).get + val tmp = CompactPrinter(json) + val file = cacheDirectory / "resources.json" + IO.write(file, tmp) + file + } + @nowarn def configCacheSettings[A <: RemoteCacheArtifact]( cacheArtifactTask: Def.Initialize[Task[A]] @@ -166,6 +190,10 @@ object RemoteCache { if (af.exists) { JarUtils.includeInJar(artp, Vector(af -> s"META-INF/inc_compile.zip")) } + val rf = getResourceFilePaths.value + if (rf.exists) { + JarUtils.includeInJar(artp, Vector(rf -> s"META-INF/resources.json")) + } // val testStream = (test / streams).?.value // testStream foreach { s => // val sf = Defaults.succeededFile(s.cacheDirectory) @@ -248,6 +276,7 @@ object RemoteCache { val smi = scalaModuleInfo.value val artifacts = (pushRemoteCacheConfiguration / remoteCacheArtifacts).value val nonPom = artifacts.filterNot(isPomArtifact).toVector + val converter = fileConverter.value m.withModule(log) { case (ivy, md, _) => val resolver = ivy.getSettings.getResolver(r.name) @@ -280,7 +309,7 @@ object RemoteCache { findJar(classifier, v, jars) match { case Some(jar) => - extractJar(art, jar) + extractJar(art, jar, converter) log.info(s"remote cache artifact extracted for $p $classifier") case None => @@ -368,11 +397,16 @@ object RemoteCache { jars.find(_.toString.endsWith(suffix)) } - private def extractJar(cacheArtifact: RemoteCacheArtifact, jar: File): Unit = + private def extractJar( + cacheArtifact: RemoteCacheArtifact, + jar: File, + converter: FileConverter + ): Unit = cacheArtifact match { case a: CompileRemoteCacheArtifact => extractCache(jar, a.extractDirectory, preserveLastModified = true) { output => extractAnalysis(output, a.analysisFile) + extractResourceList(output, converter) } case a: TestRemoteCacheArtifact => @@ -410,6 +444,23 @@ object RemoteCache { } } + private def extractResourceList(output: File, converter: FileConverter): Unit = { + import sbt.librarymanagement.LibraryManagementCodec._ + import sjsonnew.support.scalajson.unsafe.{ Converter, Parser } + import xsbti.VirtualFileRef + + val resourceFilesToDelete = output / "META-INF" / "resources.json" + if (resourceFilesToDelete.exists) { + val readFile = IO.read(resourceFilesToDelete) + val parseFile = Parser.parseUnsafe(readFile) + val resourceFiles = Converter.fromJsonUnsafe[Seq[String]](parseFile) + val paths = resourceFiles.map(f => converter.toPath(VirtualFileRef.of(f))) + val filesToDelete = paths.map(_.toFile) + for (file <- filesToDelete if file.getAbsolutePath.startsWith(output.getAbsolutePath)) + IO.delete(file) + } + } + private def extractTestResult(output: File, testResult: File): Unit = { //val expandedTestResult = output / "META-INF" / "succeeded_tests" //if (expandedTestResult.exists) {