Merge pull request #2780 from eed3si9n/wip/template_resolver

Safer template resolver

build.sbt

@@ -445,12 +445,11 @@ lazy val actionsProj = (project in mainPath / "actions").
 
 // General command support and core commands not specific to a build system
 lazy val commandProj = (project in mainPath / "command").
-  dependsOn(interfaceProj, ioProj, logProj, completeProj, classpathProj, crossProj).
+  dependsOn(interfaceProj, ioProj, logProj, completeProj, classpathProj, crossProj, ivyProj).
   settings(
     testedBaseSettings,
     name := "Command",
-    libraryDependencies ++= Seq(launcherInterface, templateResolverApi, giter8),
+    libraryDependencies ++= Seq(launcherInterface, templateResolverApi)
-    dependencyOverrides += plexusUtils
   )
 
 // Fixes scope=Scope for Setting (core defined in collectionProj) to define the settings system used in build definitions
@@ -489,7 +488,6 @@ lazy val mavenResolverPluginProj = (project in file("sbt-maven-resolver")).
     baseSettings,
     name := "sbt-maven-resolver",
     libraryDependencies ++= aetherLibs,
-    dependencyOverrides += plexusUtils,
     sbtPlugin := true
   )
 

compile/ivy/src/main/scala/sbt/compiler/ComponentCompiler.scala

@@ -91,9 +91,10 @@ private[compiler] class IvyComponentCompiler(compiler: RawCompiler, manager: Com
   private val sbtOrg = xsbti.ArtifactInfo.SbtOrganization
   private val sbtOrgTemp = JsonUtil.sbtOrgTemp
   private val modulePrefixTemp = "temp-module-"
-  private val ivySbt: IvySbt = new IvySbt(ivyConfiguration)
+
   private val sbtVersion = ComponentManager.version
   private val buffered = new BufferedLogger(FullLogger(log))
+  private val updateUtil = new UpdateUtil(ivyConfiguration, buffered)
 
   def apply(): File = {
     // binID is of the form "org.example-compilerbridge-1.0.0-bin_2.11.7__50.0"
@@ -115,7 +116,7 @@ private[compiler] class IvyComponentCompiler(compiler: RawCompiler, manager: Com
       buffered bufferQuietly {
 
         IO.withTemporaryDirectory { retrieveDirectory =>
-          (update(getModule(sourcesModule), retrieveDirectory)(_.getName endsWith "-sources.jar")) match {
+          (updateUtil.update(updateUtil.getModule(sourcesModule), retrieveDirectory)(_.getName endsWith "-sources.jar")) match {
             case Some(sources) =>
               AnalyzingCompiler.compileSources(sources, targetJar, xsbtiJars, sourcesModule.name, compiler, log)
               manager.define(binID, Seq(targetJar))
@@ -128,67 +129,4 @@ private[compiler] class IvyComponentCompiler(compiler: RawCompiler, manager: Com
       }
     }
 
-  /**
-   * Returns a dummy module that depends on `moduleID`.
-   * Note: Sbt's implementation of Ivy requires us to do this, because only the dependencies
-   *       of the specified module will be downloaded.
-   */
-  private def getModule(moduleID: ModuleID): ivySbt.Module = {
-    val sha1 = Hash.toHex(Hash(moduleID.name))
-    val dummyID = ModuleID(sbtOrgTemp, modulePrefixTemp + sha1, moduleID.revision, moduleID.configurations)
-    getModule(dummyID, Seq(moduleID))
-  }
-
-  private def getModule(moduleID: ModuleID, deps: Seq[ModuleID], uo: UpdateOptions = UpdateOptions()): ivySbt.Module = {
-    val moduleSetting = InlineConfiguration(
-      module = moduleID,
-      moduleInfo = ModuleInfo(moduleID.name),
-      dependencies = deps,
-      configurations = Seq(Configurations.Component),
-      ivyScala = None)
-
-    new ivySbt.Module(moduleSetting)
-  }
-
-  private def dependenciesNames(module: ivySbt.Module): String = module.moduleSettings match {
-    // `module` is a dummy module, we will only fetch its dependencies.
-    case ic: InlineConfiguration =>
-      ic.dependencies map {
-        case mID: ModuleID =>
-          import mID._
-          s"$organization % $name % $revision"
-      } mkString ", "
-    case _ =>
-      s"unknown"
-  }
-
-  private def update(module: ivySbt.Module, retrieveDirectory: File)(predicate: File => Boolean): Option[Seq[File]] = {
-
-    val retrieveConfiguration = new RetrieveConfiguration(retrieveDirectory, Resolver.defaultRetrievePattern, false)
-    val updateConfiguration = new UpdateConfiguration(Some(retrieveConfiguration), true, UpdateLogging.DownloadOnly)
-
-    buffered.info(s"Attempting to fetch ${dependenciesNames(module)}. This operation may fail.")
-    IvyActions.updateEither(module, updateConfiguration, UnresolvedWarningConfiguration(), LogicalClock.unknown, None, buffered) match {
-      case Left(unresolvedWarning) =>
-        buffered.debug("Couldn't retrieve module ${dependenciesNames(module)}.")
-        None
-
-      case Right(updateReport) =>
-        val allFiles =
-          for {
-            conf <- updateReport.configurations
-            m <- conf.modules
-            (_, f) <- m.artifacts
-          } yield f
-
-        buffered.debug(s"Files retrieved for ${dependenciesNames(module)}:")
-        buffered.debug(allFiles mkString ", ")
-
-        allFiles filter predicate match {
-          case Seq() => None
-          case files => Some(files)
-        }
-
-    }
-  }
 }

ivy/src/main/scala/sbt/UpdateUtil.scala

@@ -0,0 +1,78 @@
+package sbt
+
+import java.io.File
+import scala.util.Try
+
+private[sbt] class UpdateUtil(ivyConfiguration: IvyConfiguration, log: Logger) {
+  private[sbt] val ivySbt: IvySbt = new IvySbt(ivyConfiguration)
+  private val sbtOrgTemp = JsonUtil.sbtOrgTemp
+  private val modulePrefixTemp = "temp-module-"
+  // private val buffered = new BufferedLogger(FullLogger(log))
+
+  /**
+   * Returns a dummy module that depends on `moduleID`.
+   * Note: Sbt's implementation of Ivy requires us to do this, because only the dependencies
+   *       of the specified module will be downloaded.
+   */
+  def getModule(moduleId: ModuleID): ivySbt.Module = getModule(moduleId, None)
+
+  def getModule(moduleId: ModuleID, ivyScala: Option[IvyScala]): ivySbt.Module = {
+    val sha1 = Hash.toHex(Hash(moduleId.name))
+    val dummyID = ModuleID(sbtOrgTemp, modulePrefixTemp + sha1, moduleId.revision, moduleId.configurations)
+    getModule(dummyID, Seq(moduleId), UpdateOptions(), ivyScala)
+  }
+
+  def getModule(moduleId: ModuleID, deps: Seq[ModuleID],
+    uo: UpdateOptions = UpdateOptions(), ivyScala: Option[IvyScala]): ivySbt.Module = {
+    val moduleSetting = InlineConfiguration(
+      module = moduleId,
+      moduleInfo = ModuleInfo(moduleId.name),
+      dependencies = deps,
+      configurations = Seq(Configurations.Component),
+      ivyScala = ivyScala)
+    new ivySbt.Module(moduleSetting)
+  }
+
+  private def dependenciesNames(module: ivySbt.Module): String =
+    module.moduleSettings match {
+      // `module` is a dummy module, we will only fetch its dependencies.
+      case ic: InlineConfiguration =>
+        ic.dependencies map {
+          case mID: ModuleID =>
+            import mID._
+            s"$organization % $name % $revision"
+        } mkString ", "
+      case _ =>
+        s"unknown"
+    }
+
+  def update(module: ivySbt.Module, retrieveDirectory: File)(predicate: File => Boolean): Option[Seq[File]] = {
+
+    val retrieveConfiguration = new RetrieveConfiguration(retrieveDirectory, Resolver.defaultRetrievePattern, false)
+    val updateConfiguration = new UpdateConfiguration(Some(retrieveConfiguration), true, UpdateLogging.DownloadOnly)
+
+    log.debug(s"Attempting to fetch ${dependenciesNames(module)}. This operation may fail.")
+    IvyActions.updateEither(module, updateConfiguration, UnresolvedWarningConfiguration(), LogicalClock.unknown, None, log) match {
+      case Left(unresolvedWarning) =>
+        log.debug("Couldn't retrieve module ${dependenciesNames(module)}.")
+        None
+
+      case Right(updateReport) =>
+        val allFiles =
+          for {
+            conf <- updateReport.configurations
+            m <- conf.modules
+            (_, f) <- m.artifacts
+          } yield f
+
+        log.debug(s"Files retrieved for ${dependenciesNames(module)}:")
+        log.debug(allFiles mkString ", ")
+
+        allFiles filter predicate match {
+          case Seq() => None
+          case files => Some(files)
+        }
+
+    }
+  }
+}

main/command/src/main/scala/sbt/BasicCommands.scala

@@ -16,7 +16,7 @@ import java.io.File
 import scala.util.control.NonFatal
 
 object BasicCommands {
-  lazy val allBasicCommands = Seq(nop, ignore, help, completionsCommand, templateCommand, multi, ifLast, append, setOnFailure, clearOnFailure, stashOnFailure, popOnFailure, reboot, call, early, exit, continuous, history, shell, read, alias) ++ compatCommands
+  lazy val allBasicCommands = Seq(nop, ignore, help, completionsCommand, multi, ifLast, append, setOnFailure, clearOnFailure, stashOnFailure, popOnFailure, reboot, call, early, exit, continuous, history, shell, read, alias) ++ compatCommands
 
   def nop = Command.custom(s => success(() => s))
   def ignore = Command.command(FailureWall)(idFun)
@@ -80,31 +80,6 @@ object BasicCommands {
     state
   }
 
-  def templateCommand = Command.make(TemplateCommand, templateBrief, templateDetailed)(templateCommandParser)
-  def templateCommandParser(state: State) =
-    {
-      val p = (token(Space) ~> repsep(StringBasic, token(Space))) | (token(EOF) map { case _ => Nil })
-      val trs = (state get templateResolvers) match {
-        case Some(trs) => trs.toList
-        case None      => Nil
-      }
-      applyEffect(p)({ inputArg =>
-        val arguments = inputArg.toList ++
-          (state.remainingCommands.toList match {
-            case "shell" :: Nil => Nil
-            case xs             => xs
-          })
-        trs find { tr =>
-          tr.isDefined(arguments.toArray)
-        } match {
-          case Some(tr) => tr.run(arguments.toArray)
-          case None =>
-            System.err.println("Template not found for: " + arguments.mkString(" "))
-        }
-        "exit" :: state.copy(remainingCommands = Nil)
-      })
-    }
-
   def multiParser(s: State): Parser[Seq[String]] =
     {
       val nonSemi = token(charClass(_ != ';').+, hide = const(true))

main/command/src/main/scala/sbt/BasicKeys.scala

@@ -11,5 +11,5 @@ object BasicKeys {
   private[sbt] val classLoaderCache = AttributeKey[classpath.ClassLoaderCache]("class-loader-cache", "Caches class loaders based on the classpath entries and last modified times.", 10)
   private[sbt] val OnFailureStack = AttributeKey[List[Option[String]]]("on-failure-stack", "Stack that remembers on-failure handlers.", 10)
   private[sbt] val explicitGlobalLogLevels = AttributeKey[Boolean]("explicit-global-log-levels", "True if the global logging levels were explicitly set by the user.", 10)
-  private[sbt] val templateResolvers = AttributeKey[Seq[TemplateResolver]]("templateResolvers", "List of template resolvers.", 1000)
+  private[sbt] val templateResolverInfos = AttributeKey[Seq[TemplateResolverInfo]]("templateResolverInfos", "List of template resolver infos.", 1000)
 }

main/command/src/main/scala/sbt/State.scala

@@ -254,3 +254,5 @@ object State {
   private[sbt] def getBoolean(s: State, key: AttributeKey[Boolean], default: Boolean): Boolean =
     s.get(key) getOrElse default
 }
+
+case class TemplateResolverInfo(module: ModuleID, implementationClass: String)

main/src/main/scala/sbt/Defaults.scala

@@ -160,7 +160,7 @@ object Defaults extends BuildCommon {
     maxErrors :== 100,
     fork :== false,
     initialize :== {},
-    templateResolvers :== Nil,
+    templateResolverInfos :== Nil,
     forcegc :== sys.props.get("sbt.task.forcegc").map(java.lang.Boolean.parseBoolean).getOrElse(GCUtil.defaultForceGarbageCollection),
     minForcegcInterval :== GCUtil.defaultMinForcegcInterval
   ))

main/src/main/scala/sbt/Giter8TemplateResolver.scala

@@ -1,32 +0,0 @@
-package sbt
-
-import sbt.template.TemplateResolver
-
-object Giter8TemplateResolver extends TemplateResolver {
-  def isDefined(args0: Array[String]): Boolean =
-    {
-      val args = args0.toList filterNot { _.startsWith("-") }
-      // Mandate .g8
-      val Github = """^([^\s/]+)/([^\s/]+?)(?:\.g8)$""".r
-      val Local = """^file://(\S+)(?:\.g8)(?:/)?$""".r
-      object GitUrl {
-        val NativeUrl = "^(git[@|://].*)$".r
-        val HttpsUrl = "^(https://.*)$".r
-        val HttpUrl = "^(http://.*)$".r
-        val SshUrl = "^(ssh://.*)$".r
-        def unapplySeq(s: Any): Option[List[String]] =
-          NativeUrl.unapplySeq(s) orElse
-            HttpsUrl.unapplySeq(s) orElse
-            HttpUrl.unapplySeq(s) orElse
-            SshUrl.unapplySeq(s)
-      }
-      args.headOption match {
-        case Some(Github(_, _)) => true
-        case Some(Local(_))     => true
-        case GitUrl(uri)        => uri contains (".g8")
-        case _                  => false
-      }
-    }
-  def run(args: Array[String]): Unit =
-    giter8.Giter8.run(args)
-}

main/src/main/scala/sbt/Keys.scala

@@ -346,7 +346,7 @@ object Keys {
   val sbtVersion = SettingKey[String]("sbt-version", "Provides the version of sbt.  This setting should be not be modified.", AMinusSetting)
   val sbtBinaryVersion = SettingKey[String]("sbt-binary-version", "Defines the binary compatibility version substring.", BPlusSetting)
   val skip = TaskKey[Boolean]("skip", "For tasks that support it (currently only 'compile' and 'update'), setting skip to true will force the task to not to do its work.  This exact semantics may vary by task.", BSetting)
-  val templateResolvers = SettingKey[Seq[TemplateResolver]]("templateResolvers", "Template resolvers used for 'new'.", BSetting)
+  val templateResolverInfos = SettingKey[Seq[TemplateResolverInfo]]("templateResolverInfos", "Template resolvers used for 'new'.", BSetting)
 
   // special
   val sessionVars = AttributeKey[SessionVar.Map]("session-vars", "Bindings that exist for the duration of the session.", Invisible)

main/src/main/scala/sbt/Main.scala

@@ -80,6 +80,7 @@ import CommandStrings._
 import BasicCommandStrings._
 import BasicCommands._
 import CommandUtil._
+import TemplateCommandUtil.templateCommand
 
 object BuiltinCommands {
   def initialAttributes = AttributeMap.empty

main/src/main/scala/sbt/Project.scala

@@ -7,7 +7,7 @@ import java.io.File
 import java.net.URI
 import java.util.Locale
 import Project.{ Initialize => _, Setting => _, _ }
-import Keys.{ appConfiguration, stateBuildStructure, commands, configuration, historyPath, projectCommand, sessionSettings, shellPrompt, templateResolvers, thisProject, thisProjectRef, watch }
+import Keys.{ appConfiguration, stateBuildStructure, commands, configuration, historyPath, projectCommand, sessionSettings, shellPrompt, templateResolverInfos, thisProject, thisProjectRef, watch }
 import Scope.{ GlobalScope, ThisScope }
 import Def.{ Flattened, Initialize, ScopedKey, Setting }
 import Types.{ const, idFun }
@@ -359,13 +359,13 @@ object Project extends ProjectExtra {
       val allCommands = commandsIn(ref) ++ commandsIn(BuildRef(ref.build)) ++ (commands in Global get structure.data toList)
       val history = get(historyPath) flatMap idFun
       val prompt = get(shellPrompt)
-      val trs = (templateResolvers in Global get structure.data).toList.flatten
+      val trs = (templateResolverInfos in Global get structure.data).toList.flatten
       val watched = get(watch)
       val commandDefs = allCommands.distinct.flatten[Command].map(_ tag (projectCommand, true))
       val newDefinedCommands = commandDefs ++ BasicCommands.removeTagged(s.definedCommands, projectCommand)
       val newAttrs = setCond(Watched.Configuration, watched, s.attributes).
         put(historyPath.key, history).
-        put(templateResolvers.key, trs)
+        put(templateResolverInfos.key, trs)
       s.copy(attributes = setCond(shellPrompt.key, prompt, newAttrs), definedCommands = newDefinedCommands)
     }
   def setCond[T](key: AttributeKey[T], vopt: Option[T], attributes: AttributeMap): AttributeMap =

main/src/main/scala/sbt/TemplateCommand.scala

@@ -0,0 +1,91 @@
+package sbt
+
+import java.lang.reflect.InvocationTargetException
+import java.io.File
+import xsbti.AppConfiguration
+import sbt.classpath.ClasspathUtilities
+import BasicCommandStrings._
+import BasicKeys._
+import complete.{ Parser, DefaultParsers }
+import DefaultParsers._
+import Command.applyEffect
+import Path._
+
+private[sbt] object TemplateCommandUtil {
+  def templateCommand = Command.make(TemplateCommand, templateBrief, templateDetailed)(templateCommandParser)
+  def templateCommandParser(state: State) =
+    {
+      val p = (token(Space) ~> repsep(StringBasic, token(Space))) | (token(EOF) map { case _ => Nil })
+      val infos = (state get templateResolverInfos) match {
+        case Some(infos) => infos.toList
+        case None        => Nil
+      }
+      val log = state.globalLogging.full
+      val extracted = (Project extract state)
+      val (s2, ivyConf) = extracted.runTask(Keys.ivyConfiguration, state)
+      val globalBase = BuildPaths.getGlobalBase(state)
+      val ivyScala = extracted.get(Keys.ivyScala in Keys.updateSbtClassifiers)
+      applyEffect(p)({ inputArg =>
+        val arguments = inputArg.toList ++
+          (state.remainingCommands.toList match {
+            case "shell" :: Nil => Nil
+            case xs             => xs
+          })
+        run(infos, arguments, state.configuration, ivyConf, globalBase, ivyScala, log)
+        "exit" :: s2.copy(remainingCommands = Nil)
+      })
+    }
+
+  private def run(infos: List[TemplateResolverInfo], arguments: List[String], config: AppConfiguration,
+    ivyConf: IvyConfiguration, globalBase: File, ivyScala: Option[IvyScala], log: Logger): Unit =
+    infos find { info =>
+      val loader = infoLoader(info, config, ivyConf, globalBase, ivyScala, log)
+      val hit = tryTemplate(info, arguments, loader)
+      if (hit) {
+        runTemplate(info, arguments, loader)
+      }
+      hit
+    } match {
+      case Some(_) => // do nothing
+      case None    => System.err.println("Template not found for: " + arguments.mkString(" "))
+    }
+  private def tryTemplate(info: TemplateResolverInfo, arguments: List[String], loader: ClassLoader): Boolean =
+    {
+      val resultObj = call(info.implementationClass, "isDefined", loader)(
+        classOf[Array[String]])(arguments.toArray)
+      resultObj.asInstanceOf[Boolean]
+    }
+  private def runTemplate(info: TemplateResolverInfo, arguments: List[String], loader: ClassLoader): Unit =
+    call(info.implementationClass, "run", loader)(classOf[Array[String]])(arguments.toArray)
+  private def infoLoader(info: TemplateResolverInfo, config: AppConfiguration,
+    ivyConf: IvyConfiguration, globalBase: File, ivyScala: Option[IvyScala], log: Logger): ClassLoader =
+    ClasspathUtilities.toLoader(classpathForInfo(info, ivyConf, globalBase, ivyScala, log), config.provider.loader)
+  private def call(interfaceClassName: String, methodName: String, loader: ClassLoader)(argTypes: Class[_]*)(args: AnyRef*): AnyRef =
+    {
+      val interfaceClass = getInterfaceClass(interfaceClassName, loader)
+      val interface = interfaceClass.newInstance.asInstanceOf[AnyRef]
+      val method = interfaceClass.getMethod(methodName, argTypes: _*)
+      try { method.invoke(interface, args: _*) }
+      catch {
+        case e: InvocationTargetException => throw e.getCause
+      }
+    }
+  private def getInterfaceClass(name: String, loader: ClassLoader) = Class.forName(name, true, loader)
+
+  // Cache files under ~/.sbt/0.13/templates/org_name_version
+  private def classpathForInfo(info: TemplateResolverInfo, ivyConf: IvyConfiguration, globalBase: File, ivyScala: Option[IvyScala], log: Logger): List[File] =
+    {
+      val updateUtil = new UpdateUtil(ivyConf, log)
+      val templatesBaseDirectory = new File(globalBase, "templates")
+      val templateId = s"${info.module.organization}_${info.module.name}_${info.module.revision}"
+      val templateDirectory = new File(templatesBaseDirectory, templateId)
+      def jars = (templateDirectory ** -DirectoryFilter).get
+      if (!(info.module.revision endsWith "-SNAPSHOT") && jars.nonEmpty) jars.toList
+      else {
+        IO.createDirectory(templateDirectory)
+        val m = updateUtil.getModule(info.module.copy(configurations = Some("component")), ivyScala)
+        val xs = updateUtil.update(m, templateDirectory)(_ => true).toList.flatten
+        xs
+      }
+    }
+}

main/src/main/scala/sbt/plugins/Giter8ResolverPlugin.scala

@@ -12,6 +12,8 @@ object Giter8TemplatePlugin extends AutoPlugin {
 
   override lazy val globalSettings: Seq[Setting[_]] =
     Seq(
-      templateResolvers += Giter8TemplateResolver
+      templateResolverInfos +=
+        TemplateResolverInfo(ModuleID("org.scala-sbt.sbt-giter8-resolver", "sbt-giter8-resolver", "0.1.0") cross CrossVersion.binary,
+          "sbtgiter8resolver.Giter8TemplateResolver")
     )
 }

notes/0.13.13.markdown

@@ -12,7 +12,7 @@
 
 ### Improvements
 
-- Adds `new` command and `templateResolvers`. See below for more details.
+- Adds `new` command and `templateResolverInfos`. See below for more details.
 - Auto plugins can add synthetic subprojects. See below for more details.
 - Supports wildcard exclusions in POMs [#1431][1431]/[sbt/ivy#22][sbt-ivy-22]/[#2731][2731] by [@jtgrabowski][@jtgrabowski]
 - Adds the ability to call `aggregateProjects(..)` for the current project inside a build sbt file. [#2682][2682] by [@xuwei-k][@xuwei-k]
@@ -32,7 +32,7 @@
 - Fixes forked tests being reported as successful when the test harness fails. [#2442][2442]/[#2722][2722]/[#2730][2730] by [@eed3si9n][@eed3si9n]/[@dwijnand][@dwijnand]
 - Fixes incorrect installation path on Windows. [sbt/sbt-launcher-package#110][110] by [@dwijnand][@dwijnand]
 
-### new command and templateResolvers
+### new command and templateResolverInfos
 
 sbt 0.13.13 adds a `new` command, which helps create new build definitions.
 The `new` command is extensible via a mechanism called the template resolver,

project/Dependencies.scala

@@ -22,7 +22,6 @@ object Dependencies {
   lazy val launcherInterface = "org.scala-sbt" % "launcher-interface" % "1.0.0-M1"
   lazy val rawLauncher = "org.scala-sbt" % "launcher" % "1.0.0-M1"
   lazy val templateResolverApi = "org.scala-sbt" % "template-resolver" % "0.1"
-  lazy val giter8 = "org.foundweekends.giter8" %% "giter8" % "0.7.1"
 
   private def scala211Module(name: String, moduleVersion: String) =
     Def.setting {