diff --git a/build.sbt b/build.sbt index ac2a97c0d..cad744baa 100644 --- a/build.sbt +++ b/build.sbt @@ -764,6 +764,11 @@ lazy val mainProj = (project in file("main")) exclude[IncompatibleResultTypeProblem]("sbt.internal.GlobalPluginData._3"), exclude[IncompatibleResultTypeProblem]("sbt.internal.GlobalPluginData._4"), exclude[DirectMissingMethodProblem]("sbt.internal.GlobalPluginData._6"), + // Updating remote vcs projects (sbt#1284) + exclude[DirectMissingMethodProblem]("sbt.Resolvers.creates"), + exclude[DirectMissingMethodProblem]("sbt.Resolvers.uniqueSubdirectoryFor"), + exclude[DirectMissingMethodProblem]("sbt.Resolvers.run"), + exclude[MissingClassProblem]("sbt.Resolvers$DistributedVCS") ), ) .dependsOn(lmCore, lmCoursierShadedPublishing) diff --git a/main/src/main/scala/sbt/BuildPaths.scala b/main/src/main/scala/sbt/BuildPaths.scala index 0f6c014d3..792f8b9ed 100644 --- a/main/src/main/scala/sbt/BuildPaths.scala +++ b/main/src/main/scala/sbt/BuildPaths.scala @@ -42,6 +42,12 @@ object BuildPaths: val globalZincDirectory = AttributeKey[File]("global-zinc-directory", "The base directory for Zinc internals.", DSetting) + val repositoryUpdateCompleted = AttributeKey[Boolean]( + "repository-update-completed", + "Indicates that remote project repositories have been updated.", + DSetting + ) + import sbt.io.syntax.* def getGlobalBase(state: State): File = { diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 081cd20cd..9edc4a3bb 100644 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -123,7 +123,7 @@ import xsbti.compile.{ TransactionalManagerType } -object Defaults extends BuildCommon { +object Defaults extends BuildCommon with DefExtra { final val CacheDirectoryName = "cache" def configSrcSub(key: SettingKey[File]): Initialize[File] = @@ -324,6 +324,7 @@ object Defaults extends BuildCommon { else TaskCancellationStrategy.Null }, envVars :== Map.empty, + repositoryUpdateStrategy :== RepositoryUpdateStrategy.Manual, sbtVersion := appConfiguration.value.provider.id.version, sbtBinaryVersion := binarySbtVersion(sbtVersion.value), pluginCrossBuild / sbtVersion := sbtVersion.value, @@ -2511,7 +2512,23 @@ object Defaults extends BuildCommon { else ClassLoaderLayeringStrategy.ScalaLibrary }, publishLocal / skip := (publish / skip).value, - publishM2 / skip := (publish / skip).value + publishM2 / skip := (publish / skip).value, + fetchSource := Def.uncached { + val uri = thisProjectRef.value.build + val log = streams.value.log + RetrieveUnit(uri) match { + case None => () + case Some(vcs) => + val strategy = repositoryUpdateStrategy.value + val lb = Project.extract(state.value).get(Keys.loadedBuild) + val vcsRoot = lb.units.get(uri).map(_.localBase).getOrElse(baseDirectory.value) + if (Resolvers.shouldUpdate(vcsRoot, strategy)) { + log.info(s"Updating remote project: $vcsRoot ...") + if (Resolvers.updateRepository(vcsRoot, vcs, log)) + log.warn("Remote dependencies updated. Run `reload` to pick up changes.") + } + } + }, ) // build.sbt is treated a Scala source of metabuild, so to enable deprecation flag on build.sbt we set the option here. lazy val deprecationSettings: Seq[Setting[?]] = @@ -3317,12 +3334,19 @@ object Classpaths { dependencyPositions.value ) ), + maybeUpdateRemoteProjects := Def.uncached(Def.taskDyn { + val buildDeps = buildDependencies.value + val thisRef = thisProjectRef.value + val lb = Project.extract(state.value).get(Keys.loadedBuild) + val vcsRootRefs = Resolvers.transitiveVcsRootRefs(thisRef, buildDeps, lb) + if (vcsRootRefs.isEmpty) Def.task(()) + else Def.sequential(vcsRootRefs.map(_ / fetchSource).toSeq) + }.value), updateFull := Def.uncached(updateTask.value), - update := Def.uncached(updateWithoutDetails("update").value), update := Def.uncached { - val report = update.value - val log = streams.value.log - ConflictWarning(conflictWarning.value, report, log) + val _ = maybeUpdateRemoteProjects.value + val report = updateWithoutDetails("update").value + ConflictWarning(conflictWarning.value, report, streams.value.log) report }, update / evictionWarningOptions := evictionWarningOptions.value, diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index 86f16699f..7abda798e 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -537,6 +537,10 @@ object Keys { val ivySbt = taskKey[Any]("Provides the sbt interface to Ivy.").withRank(CTask) val ivyModule = taskKey[Any]("Provides the sbt interface to a configured Ivy module.").withRank(CTask) val updateCacheName = taskKey[String]("Defines the directory name used to store the update cache files (inside the streams cacheDirectory).").withRank(DTask) + val repositoryUpdateStrategy = settingKey[RepositoryUpdateStrategy]("Strategy for updating remote VCS project dependencies.").withRank(CSetting) + val fetchSource = taskKey[Unit]("Fetches or updates the VCS source for this project if it is backed by a remote VCS repository.").withRank(DTask) + val updateRemoteProjects = taskKey[Unit]("Force-updates all remote VCS project dependencies, ignoring strategy.").withRank(BTask) + val maybeUpdateRemoteProjects = taskKey[Unit]("Updates remote VCS project dependencies according to repositoryUpdateStrategy.").withRank(DTask) val update = taskKey[UpdateReport]("Resolves and optionally retrieves dependencies, producing a report.").withRank(ATask) val updateFull = taskKey[UpdateReport]("Resolves and optionally retrieves dependencies, producing a full report with callers.").withRank(CTask) val evicted = taskKey[EvictionWarning]("Display detailed eviction warnings.").withRank(CTask) diff --git a/main/src/main/scala/sbt/Main.scala b/main/src/main/scala/sbt/Main.scala index 4edb316fa..1bb3351e0 100644 --- a/main/src/main/scala/sbt/Main.scala +++ b/main/src/main/scala/sbt/Main.scala @@ -387,6 +387,7 @@ object BuiltinCommands { act, continuous, clearCaches, + updateRemoteProjectsCommand, Clean.cleanFull, NetworkChannel.disconnect, waitCmd, @@ -1045,6 +1046,35 @@ object BuiltinCommands { def registerCompilerCache(s: State): State = Clean.registerCompilerCache(s) def clearCaches: Command = Clean.clearCaches + def updateRemoteProjectsCommand: Command = + Command.command( + "updateRemoteProjects", + Help.more( + "updateRemoteProjects", + "Force-updates all remote VCS project dependencies." + ) + ) { s => + if (s.get(BuildPaths.repositoryUpdateCompleted).getOrElse(false)) { + s.remove(BuildPaths.repositoryUpdateCompleted) + } else { + val log = s.log + val extracted = Project.extract(s) + val isUpdated = Resolvers.updateLoadedBuild( + lb = extracted.get(Keys.loadedBuild), + log = log, + extracted = extracted, + force = true + ) + + if (isUpdated) { + log.warn("Remote dependencies updated. Reloading...") + "reload" :: s.put(BuildPaths.repositoryUpdateCompleted, true) + } else { + s + } + } + } + private[sbt] def waitCmd: Command = Command.arb(_ => ContinuousCommands.waitWatch.examples() ~> " ".examples() ~> matched(any.*).examples() diff --git a/main/src/main/scala/sbt/RepositoryUpdateStrategy.scala b/main/src/main/scala/sbt/RepositoryUpdateStrategy.scala new file mode 100644 index 000000000..2d8b6b7d1 --- /dev/null +++ b/main/src/main/scala/sbt/RepositoryUpdateStrategy.scala @@ -0,0 +1,33 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt + +import scala.concurrent.duration.FiniteDuration + +/** + * Strategy for updating remote VCS project dependencies. + * + * Used with the `repositoryUpdateStrategy` setting key to control when + * remote project dependencies are updated from their upstream repositories. + */ +enum RepositoryUpdateStrategy { + + /** + * Never update automatically. + * Users must manually delete staging directories or run `updateRemoteProjects`. + */ + case Manual + + /** Update on every `update` task invocation. */ + case Always + + /** Update at most once per the given interval. */ + case Every(interval: FiniteDuration) + +} diff --git a/main/src/main/scala/sbt/Resolvers.scala b/main/src/main/scala/sbt/Resolvers.scala index 14312c732..d4f5987d4 100644 --- a/main/src/main/scala/sbt/Resolvers.scala +++ b/main/src/main/scala/sbt/Resolvers.scala @@ -19,11 +19,24 @@ import BuildLoader.ResolveInfo import RichURI.fromURI import java.util.Locale -import scala.sys.process.Process +import scala.sys.process.{ BasicIO, Process } import scala.util.control.NonFatal import sbt.internal.util.Util +import sbt.internal.{ BuildDependencies, LoadedBuild, LoadedBuildUnit } +import sbt.util.Logger +import sbt.internal.RetrieveUnit object Resolvers { + + private[sbt] sealed trait RemoteVcs + private[sbt] object RemoteVcs { + case class Git(uri: URI) extends RemoteVcs + case class Hg(uri: URI) extends RemoteVcs + case class Svn(uri: URI) extends RemoteVcs + } + + private val LastUpdatedFileName = ".sbt-last-updated" + type Resolver = BuildLoader.Resolver val local: Resolver = (info: ResolveInfo) => { @@ -47,10 +60,8 @@ object Resolvers { } val subversion: Resolver = (info: ResolveInfo) => { - def normalized(uri: URI) = uri.copy(scheme = "svn") - val uri = info.uri.withoutMarkerScheme - val localCopy = uniqueSubdirectoryFor(normalized(uri), in = info.staging) + val localCopy = uniqueSubdirectoryFor(uri.copy(scheme = "svn"), in = info.staging) val from = uri.withoutFragment.toASCIIString val to = localCopy.getAbsolutePath @@ -58,28 +69,37 @@ object Resolvers { val revision = uri.getFragment Some { () => creates(localCopy) { - run("svn", "checkout", "-q", "-r", revision, from, to) + run(cwd = None, log = None, "svn", "checkout", "-q", "-r", revision, from, to) } } } else Some { () => creates(localCopy) { - run("svn", "checkout", "-q", from, to) + run(cwd = None, log = None, "svn", "checkout", "-q", from, to) } } } - val mercurial: Resolver = new DistributedVCS { - override val scheme = "hg" + val mercurial: Resolver = (info: ResolveInfo) => { + val uri = info.uri.withoutMarkerScheme + val localCopy = uniqueSubdirectoryFor(uri.copy(scheme = "hg"), in = info.staging) + val from = uri.withoutFragment.toASCIIString - override def clone(from: String, to: File): Unit = { - run("hg", "clone", "-q", from, to.getAbsolutePath) - } - - override def checkout(branch: String, in: File): Unit = { - run(Some(in), "hg", "checkout", "-q", branch) - } - }.toResolver + if (uri.hasFragment) { + val branch = uri.getFragment + Some { () => + creates(localCopy) { + run(cwd = None, log = None, "hg", "clone", "-q", from, localCopy.getAbsolutePath) + run(Some(localCopy), log = None, "hg", "checkout", "-q", branch) + } + } + } else + Some { () => + creates(localCopy) { + run(cwd = None, log = None, "hg", "clone", "-q", from, localCopy.getAbsolutePath) + } + } + } val git: Resolver = (info: ResolveInfo) => { val uri = info.uri.withoutMarkerScheme @@ -90,61 +110,46 @@ object Resolvers { val branch = uri.getFragment Some { () => creates(localCopy) { - run("git", "clone", from, localCopy.getAbsolutePath) - run(Some(localCopy), "git", "checkout", "-q", branch) + run(cwd = None, log = None, "git", "clone", from, localCopy.getAbsolutePath) + run(Some(localCopy), log = None, "git", "checkout", "-q", branch) } } } else Some { () => creates(localCopy) { - run("git", "clone", "--depth", "1", from, localCopy.getAbsolutePath) + run( + cwd = None, + log = None, + "git", + "clone", + "--depth", + "1", + from, + localCopy.getAbsolutePath + ) } } } - abstract class DistributedVCS { - val scheme: String - - def clone(from: String, to: File): Unit - - def checkout(branch: String, in: File): Unit - - def toResolver: Resolver = (info: ResolveInfo) => { - val uri = info.uri.withoutMarkerScheme - val localCopy = uniqueSubdirectoryFor(normalized(uri), in = info.staging) - val from = uri.withoutFragment.toASCIIString - - if (uri.hasFragment) { - val branch = uri.getFragment - Some { () => - creates(localCopy) { - clone(from, to = localCopy) - checkout(branch, in = localCopy) - } - } - } else - Some { () => - creates(localCopy) { clone(from, to = localCopy) } - } - } - - private def normalized(uri: URI) = uri.copy(scheme = scheme) - } - - def run(command: String*): Unit = - run(None, command*) - - def run(cwd: Option[File], command: String*): Unit = { - val result = Process( + private def run(cwd: Option[File], log: Option[Logger], command: String*): Unit = { + val process = Process( if (Util.isNonCygwinWindows) "cmd" +: "/c" +: command else command, cwd - ).! + ) + val result = (log match { + case Some(log) => + val io = BasicIO(false, log).withInput(_.close()) + process.run(io).exitValue() + case None => + process.run().exitValue() + }) + if (result != 0) sys.error("Nonzero exit code (" + result + "): " + command.mkString(" ")) } - def creates(file: File)(f: => Unit) = { + private def creates(file: File)(f: => Unit): File = { if (!file.exists) try { f @@ -156,7 +161,7 @@ object Resolvers { file } - def uniqueSubdirectoryFor(uri: URI, in: File) = { + private def uniqueSubdirectoryFor(uri: URI, in: File): File = { in.mkdirs() val base = new File(in, Hash.halfHashString(uri.normalize.toASCIIString)) val last = shortName(uri) match { @@ -175,4 +180,178 @@ object Resolvers { private def dropExtensions(name: String): String = name.takeWhile(_ != '.') + private def markUpdated(dir: File): Unit = { + val marker = new File(dir, LastUpdatedFileName) + IO.write(marker, System.currentTimeMillis().toString) + } + + private[sbt] def shouldUpdate(dir: File, strategy: RepositoryUpdateStrategy): Boolean = + strategy match { + case RepositoryUpdateStrategy.Manual => false + case RepositoryUpdateStrategy.Always => true + case RepositoryUpdateStrategy.Every(interval) => + val marker = new File(dir, LastUpdatedFileName) + if (!marker.exists()) true + else { + try { + val lastUpdated = IO.read(marker).trim.toLong + val elapsed = System.currentTimeMillis() - lastUpdated + elapsed >= interval.toMillis + } catch { + case NonFatal(_) => true + } + } + } + + private[sbt] def updateLoadedBuild( + lb: LoadedBuild, + log: Logger, + extracted: Extracted, + force: Boolean + ): Boolean = { + def needsUpdate(uri: URI, unit: LoadedBuildUnit): Boolean = { + val rootProjectId = unit.rootProjects.headOption.getOrElse("root") + val projectRef = ProjectRef(uri, rootProjectId) + val strategy = extracted + .getOpt(projectRef / Keys.repositoryUpdateStrategy) + .getOrElse(RepositoryUpdateStrategy.Manual) + shouldUpdate(unit.localBase, strategy) + } + + log.info("Updating remote repos") + val repos = (for { + (uri, unit) <- lb.units if unit.localBase.exists() + vcs <- RetrieveUnit(uri) if force || needsUpdate(uri, unit) + } yield (unit.localBase, vcs)) + + val isUpdated = repos.foldLeft(false) { case (acc, (repo, vcs)) => + log.info(s"Updating remote project: $repo ...") + acc || Resolvers.updateRepository(repo, vcs, log) + } + + isUpdated + } + + private[sbt] def transitiveVcsRootRefs( + startRef: ProjectRef, + buildDependencies: BuildDependencies, + lb: LoadedBuild + ): Set[ProjectRef] = + for { + uri <- collectTransitiveRemoteBuildURIs(startRef, buildDependencies) + if RetrieveUnit(uri).isDefined + unit <- lb.units.get(uri) + rootId = unit.rootProjects.headOption.getOrElse("root") + } yield ProjectRef(uri, rootId) + + private def collectTransitiveRemoteBuildURIs( + startRef: ProjectRef, + buildDeps: BuildDependencies + ): Set[URI] = { + @annotation.tailrec + def go( + queue: List[ProjectRef], + visited: Set[ProjectRef], + result: Set[URI] + ): Set[URI] = + queue match { + case Nil => result + case ref :: rest => + if (visited.contains(ref)) go(rest, visited, result) + else { + val classpathDeps = buildDeps.classpath.getOrElse(ref, Nil).map(_.project) + val aggregateDeps = buildDeps.aggregate.getOrElse(ref, Nil) + val allDeps = classpathDeps ++ aggregateDeps + val newResult = allDeps.foldLeft(result) { (acc, dep) => + if (dep.build != startRef.build) acc + dep.build else acc + } + go(allDeps.toList ::: rest, visited + ref, newResult) + } + } + go(List(startRef), Set.empty, Set.empty) + } + + private[sbt] def updateRepository(localCopy: File, uri: URI, log: Logger): Boolean = + RetrieveUnit(uri) match { + case Some(vcs) => updateRepository(localCopy, vcs, log) + case None => false + } + + private[sbt] def updateRepository(localCopy: File, vcs: RemoteVcs, log: Logger): Boolean = + vcs match { + case RemoteVcs.Git(uri) => updateGit(localCopy, uri, log) + case RemoteVcs.Hg(uri) => updateMercurial(localCopy, uri, log) + case RemoteVcs.Svn(uri) => updateSubversion(localCopy, uri, log) + } + + private def updateGit(localCopy: File, uri: URI, log: Logger): Boolean = + try { + val status = captureOutput(Some(localCopy), "git", "status", "--porcelain", "-uno") + if (status.nonEmpty) { + log.warn( + s"Skipping update of $localCopy: uncommitted changes detected. " + + "Commit or discard them before updating." + ) + false + } else { + val headBefore = captureOutput(Some(localCopy), "git", "rev-parse", "HEAD") + val ref = if (fromURI(uri).hasFragment) uri.getFragment else "HEAD" + run(Some(localCopy), Some(log), "git", "fetch", "origin", ref) + run(Some(localCopy), Some(log), "git", "reset", "--hard", "FETCH_HEAD") + markUpdated(localCopy) + captureOutput(Some(localCopy), "git", "rev-parse", "HEAD") != headBefore + } + } catch { + case NonFatal(e) => + log.error( + s"Failed to update git repository at $localCopy: ${e.getMessage}" + ) + throw e + } + + private def captureOutput(cwd: Option[File], command: String*): String = + Process( + if (Util.isNonCygwinWindows) "cmd" +: "/c" +: command else command, + cwd + ).!!.trim + + private def updateMercurial(localCopy: File, uri: URI, log: Logger): Boolean = + try { + val idBefore = captureOutput(Some(localCopy), "hg", "id", "-i") + if (fromURI(uri).hasFragment) { + val branch = uri.getFragment + run(Some(localCopy), Some(log), "hg", "pull") + run(Some(localCopy), Some(log), "hg", "update", branch) + } else { + run(Some(localCopy), Some(log), "hg", "pull", "-u") + } + markUpdated(localCopy) + captureOutput(Some(localCopy), "hg", "id", "-i") != idBefore + } catch { + case NonFatal(e) => + log.error( + s"Failed to update mercurial repository at $localCopy: ${e.getMessage}" + ) + throw e + } + + private def updateSubversion(localCopy: File, uri: URI, log: Logger): Boolean = + try { + val revBefore = captureOutput(Some(localCopy), "svn", "info", "--show-item", "revision") + if (fromURI(uri).hasFragment) { + val revision = uri.getFragment + run(Some(localCopy), Some(log), "svn", "update", "-q", "-r", revision) + } else { + run(Some(localCopy), Some(log), "svn", "update", "-q") + } + markUpdated(localCopy) + captureOutput(Some(localCopy), "svn", "info", "--show-item", "revision") != revBefore + } catch { + case NonFatal(e) => + log.error( + s"Failed to update subversion working copy at $localCopy: ${e.getMessage}" + ) + throw e + } + } diff --git a/main/src/main/scala/sbt/internal/RetrieveUnit.scala b/main/src/main/scala/sbt/internal/RetrieveUnit.scala index 93126743e..55b68dc7b 100644 --- a/main/src/main/scala/sbt/internal/RetrieveUnit.scala +++ b/main/src/main/scala/sbt/internal/RetrieveUnit.scala @@ -12,17 +12,30 @@ package internal import java.io.File import java.net.URI import sbt.internal.BuildLoader.ResolveInfo +import sbt.Resolvers.RemoteVcs object RetrieveUnit { def apply(info: ResolveInfo): Option[() => File] = { - info.uri match { - case Scheme("svn") | Scheme("svn+ssh") => Resolvers.subversion(info) - case Scheme("hg") => Resolvers.mercurial(info) - case Scheme("git") => Resolvers.git(info) - case Path(path) if path.endsWith(".git") => Resolvers.git(info) - case Scheme("http") | Scheme("https") | Scheme("ftp") => Resolvers.remote(info) - case Scheme("file") => Resolvers.local(info) - case _ => None + apply(info.uri) match { + case Some(RemoteVcs.Svn(_)) => Resolvers.subversion(info) + case Some(RemoteVcs.Hg(_)) => Resolvers.mercurial(info) + case Some(RemoteVcs.Git(_)) => Resolvers.git(info) + case _ => + info.uri match { + case Scheme("http") | Scheme("https") | Scheme("ftp") => Resolvers.remote(info) + case Scheme("file") => Resolvers.local(info) + case _ => None + } + } + } + + def apply(uri: URI): Option[RemoteVcs] = { + uri match { + case Scheme("svn") | Scheme("svn+ssh") => Some(RemoteVcs.Svn(uri)) + case Scheme("hg") => Some(RemoteVcs.Hg(uri)) + case Scheme("git") => Some(RemoteVcs.Git(uri)) + case Path(path) if path.endsWith(".git") => Some(RemoteVcs.Git(uri)) + case _ => None } } diff --git a/notes/2.0.0/remote-project-update.md b/notes/2.0.0/remote-project-update.md new file mode 100644 index 000000000..07856b4f0 --- /dev/null +++ b/notes/2.0.0/remote-project-update.md @@ -0,0 +1,25 @@ +### Updating remote VCS project dependencies + +sbt now supports automatic updating of remote VCS project dependencies +(Git, Mercurial, Subversion) via a configurable update strategy. + +By default the strategy is `Manual` — remote projects are never updated +automatically during `update`. To change this, set `repositoryUpdateStrategy` +per dependency: + +```scala +lazy val dep = RootProject(uri("git:file:///path/to/repo")) + +lazy val root = project.dependsOn(dep).settings( + // Pull latest on every `update`: + dep / repositoryUpdateStrategy := RepositoryUpdateStrategy.Always, + // Or pull at most once per hour: + dep / repositoryUpdateStrategy := RepositoryUpdateStrategy.Every(1.hour), +) +``` + +When changes are detected, sbt will warn that a `reload` is needed to pick +up the new sources. + +The `updateRemoteProjects` command force-updates all remote VCS dependencies +regardless of strategy and automatically reloads the build. diff --git a/sbt-app/src/sbt-test/project/remote-project-dedup/add-newservice.sh b/sbt-app/src/sbt-test/project/remote-project-dedup/add-newservice.sh new file mode 100755 index 000000000..4eea88c94 --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-dedup/add-newservice.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -e + +cd upstream-repo +git config commit.gpgsign false + +cat > src/main/scala/upstream/NewService.scala << 'EOF' +package upstream + +object NewService { + def provide: String = "new-service" +} +EOF + +git add . +git commit -m "add NewService" diff --git a/sbt-app/src/sbt-test/project/remote-project-dedup/build.sbt b/sbt-app/src/sbt-test/project/remote-project-dedup/build.sbt new file mode 100644 index 000000000..5343ca874 --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-dedup/build.sbt @@ -0,0 +1 @@ +lazy val root = project in file(".") diff --git a/sbt-app/src/sbt-test/project/remote-project-dedup/changes/MainWithNew.scala b/sbt-app/src/sbt-test/project/remote-project-dedup/changes/MainWithNew.scala new file mode 100644 index 000000000..f31a5c3d2 --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-dedup/changes/MainWithNew.scala @@ -0,0 +1,7 @@ +import upstream.{ Service, NewService } + +object Main { + def main(args: Array[String]): Unit = { + val _ = (Service.provide, NewService.provide) + } +} diff --git a/sbt-app/src/sbt-test/project/remote-project-dedup/changes/build-with-deps.sbt b/sbt-app/src/sbt-test/project/remote-project-dedup/changes/build-with-deps.sbt new file mode 100644 index 000000000..34f71f71f --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-dedup/changes/build-with-deps.sbt @@ -0,0 +1,38 @@ +lazy val a = RootProject(uri(s"git:file://${System.getProperty("user.dir")}/upstream-repo")) +lazy val b = (project in file("b")).dependsOn(a) +lazy val c = (project in file("c")).dependsOn(a) +lazy val d = (project in file("d")).dependsOn(b, c) + +lazy val snapshotReflog = taskKey[Unit]("Records current HEAD reflog count in staging repo") +lazy val checkFetchCount = inputKey[Unit]("Asserts exactly N git resets happened since snapshot") + +lazy val root = (project in file(".")).aggregate(b, c, d).settings( + a / repositoryUpdateStrategy := RepositoryUpdateStrategy.Always, + snapshotReflog := { + val staging = baseDirectory.value / "global" / "staging" + for { + hashDir <- staging.listFiles() if hashDir.isDirectory + repoDir <- hashDir.listFiles() if repoDir.isDirectory && (repoDir / ".git").exists() + } { + val headLog = repoDir / ".git" / "logs" / "HEAD" + val count = if (headLog.exists()) IO.readLines(headLog).size else 0 + IO.write(repoDir / ".git" / "reflog-baseline", count.toString) + } + }, + checkFetchCount := { + val expected = complete.Parsers.spaceDelimited("").parsed.head.toInt + val staging = baseDirectory.value / "global" / "staging" + val gitDirs = for { + hashDir <- staging.listFiles() if hashDir.isDirectory + repoDir <- hashDir.listFiles() if repoDir.isDirectory && (repoDir / ".git").exists() + } yield repoDir + assert(gitDirs.length == 1, s"Expected 1 staging repo, got ${gitDirs.length}") + val repo = gitDirs.head + val headLog = repo / ".git" / "logs" / "HEAD" + val currentCount = if (headLog.exists()) IO.readLines(headLog).size else 0 + val baseline = repo / ".git" / "reflog-baseline" + val beforeCount = if (baseline.exists()) IO.read(baseline).trim.toInt else 0 + val fetchCount = currentCount - beforeCount + assert(fetchCount == expected, s"Expected fetch count $expected, got $fetchCount") + } +) diff --git a/sbt-app/src/sbt-test/project/remote-project-dedup/d/src/main/scala/Main.scala b/sbt-app/src/sbt-test/project/remote-project-dedup/d/src/main/scala/Main.scala new file mode 100644 index 000000000..beb5b0f2a --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-dedup/d/src/main/scala/Main.scala @@ -0,0 +1,7 @@ +import upstream.Service + +object Main { + def main(args: Array[String]): Unit = { + val _ = Service.provide + } +} diff --git a/sbt-app/src/sbt-test/project/remote-project-dedup/setup.sh b/sbt-app/src/sbt-test/project/remote-project-dedup/setup.sh new file mode 100755 index 000000000..79c7d3a3e --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-dedup/setup.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -e + +rm -rf global/staging upstream-repo +mkdir -p upstream-repo/src/main/scala/upstream +cd upstream-repo + +git init +git config user.email "test@test.com" +git config user.name "Test" +git config commit.gpgsign false + +cat > build.sbt << 'EOF' +name := "upstream-dep" +organization := "upstream" +version := "0.1.0" +EOF + +cat > src/main/scala/upstream/Service.scala << 'EOF' +package upstream + +object Service { + def provide: String = "service" +} +EOF + +git add . +git commit -m "initial commit" diff --git a/sbt-app/src/sbt-test/project/remote-project-dedup/test b/sbt-app/src/sbt-test/project/remote-project-dedup/test new file mode 100644 index 000000000..3ae707aa8 --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-dedup/test @@ -0,0 +1,15 @@ +$ exec bash setup.sh +$ copy-file changes/build-with-deps.sbt build.sbt +> reload +> compile + +$ exec bash add-newservice.sh +$ copy-file changes/MainWithNew.scala d/src/main/scala/Main.scala + +# Snapshot HEAD reflog, update, verify exactly 1 fetch (dedup across diamond) +> snapshotReflog +> d/update +> checkFetchCount 1 + +> reload +> compile diff --git a/sbt-app/src/sbt-test/project/remote-project-git-branch/add-branch-class.sh b/sbt-app/src/sbt-test/project/remote-project-git-branch/add-branch-class.sh new file mode 100755 index 000000000..b58f985a9 --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-git-branch/add-branch-class.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -e + +cd upstream-repo +git config commit.gpgsign false + +cat > src/main/scala/upstream/BranchOnly.scala << 'EOF' +package upstream + +object BranchOnly { + def value: String = "branch-only" +} +EOF + +git add . +git commit -m "add BranchOnly on feature branch" diff --git a/sbt-app/src/sbt-test/project/remote-project-git-branch/build.sbt b/sbt-app/src/sbt-test/project/remote-project-git-branch/build.sbt new file mode 100644 index 000000000..5343ca874 --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-git-branch/build.sbt @@ -0,0 +1 @@ +lazy val root = project in file(".") diff --git a/sbt-app/src/sbt-test/project/remote-project-git-branch/changes/MainWithBranchOnly.scala b/sbt-app/src/sbt-test/project/remote-project-git-branch/changes/MainWithBranchOnly.scala new file mode 100644 index 000000000..2a906183d --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-git-branch/changes/MainWithBranchOnly.scala @@ -0,0 +1,7 @@ +import upstream.{ Greeter, BranchOnly } + +object Main { + def main(args: Array[String]): Unit = { + val _ = (Greeter.greet, BranchOnly.value) + } +} diff --git a/sbt-app/src/sbt-test/project/remote-project-git-branch/changes/build-with-branch.sbt b/sbt-app/src/sbt-test/project/remote-project-git-branch/changes/build-with-branch.sbt new file mode 100644 index 000000000..5f4179d99 --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-git-branch/changes/build-with-branch.sbt @@ -0,0 +1,5 @@ +lazy val dep = RootProject(uri(s"git:file://${System.getProperty("user.dir")}/upstream-repo#feature")) + +lazy val root = (project in file(".")).dependsOn(dep).settings( + dep / repositoryUpdateStrategy := RepositoryUpdateStrategy.Always +) diff --git a/sbt-app/src/sbt-test/project/remote-project-git-branch/setup.sh b/sbt-app/src/sbt-test/project/remote-project-git-branch/setup.sh new file mode 100755 index 000000000..24c63d0a6 --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-git-branch/setup.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -e + +rm -rf global/staging upstream-repo +mkdir -p upstream-repo/src/main/scala/upstream +cd upstream-repo + +git init +git config user.email "test@test.com" +git config user.name "Test" +git config commit.gpgsign false + +cat > build.sbt << 'EOF' +name := "upstream-dep" +organization := "upstream" +version := "0.1.0" +EOF + +cat > src/main/scala/upstream/Placeholder.scala << 'EOF' +package upstream +EOF +git add . +git commit -m "initial commit" + +# Create feature branch with Greeter class +git checkout -b feature + +cat > src/main/scala/upstream/Greeter.scala << 'EOF' +package upstream + +object Greeter { + def greet: String = "hello-from-feature" +} +EOF +git add . +git commit -m "add Greeter on feature branch" diff --git a/sbt-app/src/sbt-test/project/remote-project-git-branch/src/main/scala/Main.scala b/sbt-app/src/sbt-test/project/remote-project-git-branch/src/main/scala/Main.scala new file mode 100644 index 000000000..41417e249 --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-git-branch/src/main/scala/Main.scala @@ -0,0 +1,7 @@ +import upstream.Greeter + +object Main { + def main(args: Array[String]): Unit = { + val _ = Greeter.greet + } +} diff --git a/sbt-app/src/sbt-test/project/remote-project-git-branch/test b/sbt-app/src/sbt-test/project/remote-project-git-branch/test new file mode 100644 index 000000000..7291525cb --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-git-branch/test @@ -0,0 +1,11 @@ +$ exec bash setup.sh +$ copy-file changes/build-with-branch.sbt build.sbt +> reload +> compile + +# Ensures that update fetches from the correct branch +$ exec bash add-branch-class.sh +$ copy-file changes/MainWithBranchOnly.scala src/main/scala/Main.scala +> update +> reload +> compile diff --git a/sbt-app/src/sbt-test/project/remote-project-transitive-deps/build.sbt b/sbt-app/src/sbt-test/project/remote-project-transitive-deps/build.sbt new file mode 100644 index 000000000..5343ca874 --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-transitive-deps/build.sbt @@ -0,0 +1 @@ +lazy val root = project in file(".") diff --git a/sbt-app/src/sbt-test/project/remote-project-transitive-deps/changes/build-with-dep.sbt b/sbt-app/src/sbt-test/project/remote-project-transitive-deps/changes/build-with-dep.sbt new file mode 100644 index 000000000..509044ca0 --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-transitive-deps/changes/build-with-dep.sbt @@ -0,0 +1,6 @@ +lazy val aProject = ProjectRef( + uri(s"git:file://${System.getProperty("user.dir")}/upstream-repo"), + "a" +) + +lazy val root = (project in file(".")).dependsOn(aProject) diff --git a/sbt-app/src/sbt-test/project/remote-project-transitive-deps/setup.sh b/sbt-app/src/sbt-test/project/remote-project-transitive-deps/setup.sh new file mode 100644 index 000000000..a002de23e --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-transitive-deps/setup.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -e + +rm -rf global/staging upstream-repo +mkdir -p upstream-repo/a/src/main/scala/upstream +mkdir -p upstream-repo/b/src/main/scala/upstream +cd upstream-repo + +git init +git config user.email "test@test.com" +git config user.name "Test" +git config commit.gpgsign false + +cat > build.sbt << 'EOF' +lazy val b = project.in(file("b")) +lazy val a = project.in(file("a")).dependsOn(b) +EOF + +cat > b/src/main/scala/upstream/BService.scala << 'EOF' +package upstream + +object BService { + def provide: String = "from-B" +} +EOF + +cat > a/src/main/scala/upstream/AService.scala << 'EOF' +package upstream + +object AService { + def provide: String = s"from-A-via-${BService.provide}" +} +EOF + +git add . +git commit -m "initial commit" diff --git a/sbt-app/src/sbt-test/project/remote-project-transitive-deps/src/main/scala/Main.scala b/sbt-app/src/sbt-test/project/remote-project-transitive-deps/src/main/scala/Main.scala new file mode 100644 index 000000000..6bc3afe71 --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-transitive-deps/src/main/scala/Main.scala @@ -0,0 +1,7 @@ +import upstream.{ AService, BService } + +object Main { + def main(args: Array[String]): Unit = { + val _ = (AService.provide, BService.provide) + } +} diff --git a/sbt-app/src/sbt-test/project/remote-project-transitive-deps/test b/sbt-app/src/sbt-test/project/remote-project-transitive-deps/test new file mode 100644 index 000000000..f72ad1547 --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-transitive-deps/test @@ -0,0 +1,10 @@ +# Given: +# - VCS project `b` +# - VCS project `a`, which depends on `b` +# - Project `root`, which depends on `a` +# Ensures that `root` has access to `b` + +$ exec bash setup.sh +$ copy-file changes/build-with-dep.sbt build.sbt +> reload +> compile diff --git a/sbt-app/src/sbt-test/project/remote-project-update/add-every-class.sh b/sbt-app/src/sbt-test/project/remote-project-update/add-every-class.sh new file mode 100644 index 000000000..17318a67c --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-update/add-every-class.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -e + +cd upstream-repo +git config commit.gpgsign false + +cat > src/main/scala/upstream/EveryClass.scala << 'EOF' +package upstream + +object EveryClass { + def value: String = "every" +} +EOF + +git add . +git commit -m "add EveryClass" diff --git a/sbt-app/src/sbt-test/project/remote-project-update/add-helper.sh b/sbt-app/src/sbt-test/project/remote-project-update/add-helper.sh new file mode 100644 index 000000000..95a2aa78d --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-update/add-helper.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -e + +cd upstream-repo +git config commit.gpgsign false + +cat > src/main/scala/upstream/Helper.scala << 'EOF' +package upstream + +object Helper { + def help: String = "helped" +} +EOF + +git add . +git commit -m "add Helper class" diff --git a/sbt-app/src/sbt-test/project/remote-project-update/add-manual-class.sh b/sbt-app/src/sbt-test/project/remote-project-update/add-manual-class.sh new file mode 100755 index 000000000..bf0ee73a6 --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-update/add-manual-class.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -e + +cd upstream-repo +git config commit.gpgsign false + +cat > src/main/scala/upstream/ManualClass.scala << 'EOF' +package upstream + +object ManualClass { + def value: String = "manual" +} +EOF + +git add . +git commit -m "add ManualClass" diff --git a/sbt-app/src/sbt-test/project/remote-project-update/add-utils.sh b/sbt-app/src/sbt-test/project/remote-project-update/add-utils.sh new file mode 100644 index 000000000..b2f46255a --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-update/add-utils.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -e + +cd upstream-repo +git config commit.gpgsign false + +cat > src/main/scala/upstream/Utils.scala << 'EOF' +package upstream + +object Utils { + def util: String = "utility" +} +EOF + +git add . +git commit -m "add Utils class" diff --git a/sbt-app/src/sbt-test/project/remote-project-update/build.sbt b/sbt-app/src/sbt-test/project/remote-project-update/build.sbt new file mode 100644 index 000000000..5343ca874 --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-update/build.sbt @@ -0,0 +1 @@ +lazy val root = project in file(".") diff --git a/sbt-app/src/sbt-test/project/remote-project-update/changes/MainCheckEvery.scala b/sbt-app/src/sbt-test/project/remote-project-update/changes/MainCheckEvery.scala new file mode 100644 index 000000000..4f75e3992 --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-update/changes/MainCheckEvery.scala @@ -0,0 +1,7 @@ +import upstream.{ Greeter, Helper, Utils, ManualClass, EveryClass } + +object Main { + def main(args: Array[String]): Unit = { + val _ = (Greeter.greet, Helper.help, Utils.util, ManualClass.value, EveryClass.value) + } +} diff --git a/sbt-app/src/sbt-test/project/remote-project-update/changes/MainCheckManual.scala b/sbt-app/src/sbt-test/project/remote-project-update/changes/MainCheckManual.scala new file mode 100644 index 000000000..6fe06bd1e --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-update/changes/MainCheckManual.scala @@ -0,0 +1,8 @@ +import upstream.{ Greeter, Helper, Utils } + +object Main { + def main(args: Array[String]): Unit = { + val _ = (Greeter.greet, Helper.help, Utils.util) + val _ = ManualClass.value + } +} diff --git a/sbt-app/src/sbt-test/project/remote-project-update/changes/MainWithHelper.scala b/sbt-app/src/sbt-test/project/remote-project-update/changes/MainWithHelper.scala new file mode 100644 index 000000000..78e6a7974 --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-update/changes/MainWithHelper.scala @@ -0,0 +1,7 @@ +import upstream.{ Greeter, Helper } + +object Main { + def main(args: Array[String]): Unit = { + val _ = (Greeter.greet, Helper.help) + } +} diff --git a/sbt-app/src/sbt-test/project/remote-project-update/changes/MainWithUtils.scala b/sbt-app/src/sbt-test/project/remote-project-update/changes/MainWithUtils.scala new file mode 100644 index 000000000..4ee42a53a --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-update/changes/MainWithUtils.scala @@ -0,0 +1,7 @@ +import upstream.{ Greeter, Helper, Utils } + +object Main { + def main(args: Array[String]): Unit = { + val _ = (Greeter.greet, Helper.help, Utils.util) + } +} diff --git a/sbt-app/src/sbt-test/project/remote-project-update/changes/build-with-dep-every.sbt b/sbt-app/src/sbt-test/project/remote-project-update/changes/build-with-dep-every.sbt new file mode 100644 index 000000000..d65365837 --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-update/changes/build-with-dep-every.sbt @@ -0,0 +1,7 @@ +import scala.concurrent.duration.* + +lazy val dep = RootProject(uri(s"git:file://${System.getProperty("user.dir")}/upstream-repo")) + +lazy val root = (project in file(".")).dependsOn(dep).settings( + dep / repositoryUpdateStrategy := RepositoryUpdateStrategy.Every(3.second) +) diff --git a/sbt-app/src/sbt-test/project/remote-project-update/changes/build-with-dep-manual.sbt b/sbt-app/src/sbt-test/project/remote-project-update/changes/build-with-dep-manual.sbt new file mode 100644 index 000000000..9220f27bc --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-update/changes/build-with-dep-manual.sbt @@ -0,0 +1,4 @@ +lazy val dep = RootProject(uri(s"git:file://${System.getProperty("user.dir")}/upstream-repo")) + +// Uses the default Manual strategy +lazy val root = (project in file(".")).dependsOn(dep) diff --git a/sbt-app/src/sbt-test/project/remote-project-update/changes/build-with-dep.sbt b/sbt-app/src/sbt-test/project/remote-project-update/changes/build-with-dep.sbt new file mode 100644 index 000000000..ec21bc516 --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-update/changes/build-with-dep.sbt @@ -0,0 +1,5 @@ +lazy val dep = RootProject(uri(s"git:file://${System.getProperty("user.dir")}/upstream-repo")) + +lazy val root = (project in file(".")).dependsOn(dep).settings( + dep / repositoryUpdateStrategy := RepositoryUpdateStrategy.Always +) diff --git a/sbt-app/src/sbt-test/project/remote-project-update/setup.sh b/sbt-app/src/sbt-test/project/remote-project-update/setup.sh new file mode 100644 index 000000000..71b43633b --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-update/setup.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -e + +rm -rf global/staging upstream-repo +mkdir -p upstream-repo/src/main/scala/upstream +cd upstream-repo +git init +git config user.email "test@test.com" +git config user.name "Test" +git config commit.gpgsign false + +cat > build.sbt << 'EOF' +name := "upstream-dep" +organization := "upstream" +version := "0.1.0" +EOF + +cat > src/main/scala/upstream/Greeter.scala << 'EOF' +package upstream + +object Greeter { + def greet: String = "hello" +} +EOF + +git add . +git commit -m "initial commit" diff --git a/sbt-app/src/sbt-test/project/remote-project-update/src/main/scala/Main.scala b/sbt-app/src/sbt-test/project/remote-project-update/src/main/scala/Main.scala new file mode 100644 index 000000000..41417e249 --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-update/src/main/scala/Main.scala @@ -0,0 +1,7 @@ +import upstream.Greeter + +object Main { + def main(args: Array[String]): Unit = { + val _ = Greeter.greet + } +} diff --git a/sbt-app/src/sbt-test/project/remote-project-update/test b/sbt-app/src/sbt-test/project/remote-project-update/test new file mode 100644 index 000000000..7fd9677ba --- /dev/null +++ b/sbt-app/src/sbt-test/project/remote-project-update/test @@ -0,0 +1,39 @@ +# Ensures that initial fetch works correctly +$ exec bash setup.sh +$ copy-file changes/build-with-dep.sbt build.sbt +> reload +> compile + +# Ensures that `updateRemoteProjects` works correctly +$ exec bash add-helper.sh +$ copy-file changes/MainWithHelper.scala src/main/scala/Main.scala +> updateRemoteProjects +> compile + +# Ensures update with `Always` strategy +$ exec bash add-utils.sh +$ copy-file changes/MainWithUtils.scala src/main/scala/Main.scala +> update +> reload +> compile + +# Ensures that `Manual` strategy does nothing on `update` +# compile must fail +$ copy-file changes/build-with-dep-manual.sbt build.sbt +> reload +$ exec bash add-manual-class.sh +$ copy-file changes/MainCheckManual.scala src/main/scala/Main.scala +> update +-> compile + +# Ensures that `Every` strategy updates after interval elapses +# compile must fail and succeed after sleep +$ copy-file changes/build-with-dep-every.sbt build.sbt +> reload +$ exec bash add-every-class.sh +$ copy-file changes/MainCheckEvery.scala src/main/scala/Main.scala +-> compile +$ sleep 3000 +> update +> reload +> compile