Merge pull request #2780 from eed3si9n/wip/template_resolver

Safer template resolver
This commit is contained in:
eugene yokota 2016-10-13 22:29:40 -04:00 committed by GitHub
commit a6cfb528c9
16 changed files with 190 additions and 138 deletions

View File

@ -445,12 +445,11 @@ lazy val actionsProj = (project in mainPath / "actions").
// General command support and core commands not specific to a build system
lazy val commandProj = (project in mainPath / "command").
dependsOn(interfaceProj, ioProj, logProj, completeProj, classpathProj, crossProj).
dependsOn(interfaceProj, ioProj, logProj, completeProj, classpathProj, crossProj, ivyProj).
settings(
testedBaseSettings,
name := "Command",
libraryDependencies ++= Seq(launcherInterface, templateResolverApi, giter8),
dependencyOverrides += plexusUtils
libraryDependencies ++= Seq(launcherInterface, templateResolverApi)
)
// Fixes scope=Scope for Setting (core defined in collectionProj) to define the settings system used in build definitions
@ -489,7 +488,6 @@ lazy val mavenResolverPluginProj = (project in file("sbt-maven-resolver")).
baseSettings,
name := "sbt-maven-resolver",
libraryDependencies ++= aetherLibs,
dependencyOverrides += plexusUtils,
sbtPlugin := true
)

View File

@ -91,9 +91,10 @@ private[compiler] class IvyComponentCompiler(compiler: RawCompiler, manager: Com
private val sbtOrg = xsbti.ArtifactInfo.SbtOrganization
private val sbtOrgTemp = JsonUtil.sbtOrgTemp
private val modulePrefixTemp = "temp-module-"
private val ivySbt: IvySbt = new IvySbt(ivyConfiguration)
private val sbtVersion = ComponentManager.version
private val buffered = new BufferedLogger(FullLogger(log))
private val updateUtil = new UpdateUtil(ivyConfiguration, buffered)
def apply(): File = {
// binID is of the form "org.example-compilerbridge-1.0.0-bin_2.11.7__50.0"
@ -115,7 +116,7 @@ private[compiler] class IvyComponentCompiler(compiler: RawCompiler, manager: Com
buffered bufferQuietly {
IO.withTemporaryDirectory { retrieveDirectory =>
(update(getModule(sourcesModule), retrieveDirectory)(_.getName endsWith "-sources.jar")) match {
(updateUtil.update(updateUtil.getModule(sourcesModule), retrieveDirectory)(_.getName endsWith "-sources.jar")) match {
case Some(sources) =>
AnalyzingCompiler.compileSources(sources, targetJar, xsbtiJars, sourcesModule.name, compiler, log)
manager.define(binID, Seq(targetJar))
@ -128,67 +129,4 @@ private[compiler] class IvyComponentCompiler(compiler: RawCompiler, manager: Com
}
}
/**
* Returns a dummy module that depends on `moduleID`.
* Note: Sbt's implementation of Ivy requires us to do this, because only the dependencies
* of the specified module will be downloaded.
*/
private def getModule(moduleID: ModuleID): ivySbt.Module = {
val sha1 = Hash.toHex(Hash(moduleID.name))
val dummyID = ModuleID(sbtOrgTemp, modulePrefixTemp + sha1, moduleID.revision, moduleID.configurations)
getModule(dummyID, Seq(moduleID))
}
private def getModule(moduleID: ModuleID, deps: Seq[ModuleID], uo: UpdateOptions = UpdateOptions()): ivySbt.Module = {
val moduleSetting = InlineConfiguration(
module = moduleID,
moduleInfo = ModuleInfo(moduleID.name),
dependencies = deps,
configurations = Seq(Configurations.Component),
ivyScala = None)
new ivySbt.Module(moduleSetting)
}
private def dependenciesNames(module: ivySbt.Module): String = module.moduleSettings match {
// `module` is a dummy module, we will only fetch its dependencies.
case ic: InlineConfiguration =>
ic.dependencies map {
case mID: ModuleID =>
import mID._
s"$organization % $name % $revision"
} mkString ", "
case _ =>
s"unknown"
}
private def update(module: ivySbt.Module, retrieveDirectory: File)(predicate: File => Boolean): Option[Seq[File]] = {
val retrieveConfiguration = new RetrieveConfiguration(retrieveDirectory, Resolver.defaultRetrievePattern, false)
val updateConfiguration = new UpdateConfiguration(Some(retrieveConfiguration), true, UpdateLogging.DownloadOnly)
buffered.info(s"Attempting to fetch ${dependenciesNames(module)}. This operation may fail.")
IvyActions.updateEither(module, updateConfiguration, UnresolvedWarningConfiguration(), LogicalClock.unknown, None, buffered) match {
case Left(unresolvedWarning) =>
buffered.debug("Couldn't retrieve module ${dependenciesNames(module)}.")
None
case Right(updateReport) =>
val allFiles =
for {
conf <- updateReport.configurations
m <- conf.modules
(_, f) <- m.artifacts
} yield f
buffered.debug(s"Files retrieved for ${dependenciesNames(module)}:")
buffered.debug(allFiles mkString ", ")
allFiles filter predicate match {
case Seq() => None
case files => Some(files)
}
}
}
}

View File

@ -0,0 +1,78 @@
package sbt
import java.io.File
import scala.util.Try
private[sbt] class UpdateUtil(ivyConfiguration: IvyConfiguration, log: Logger) {
private[sbt] val ivySbt: IvySbt = new IvySbt(ivyConfiguration)
private val sbtOrgTemp = JsonUtil.sbtOrgTemp
private val modulePrefixTemp = "temp-module-"
// private val buffered = new BufferedLogger(FullLogger(log))
/**
* Returns a dummy module that depends on `moduleID`.
* Note: Sbt's implementation of Ivy requires us to do this, because only the dependencies
* of the specified module will be downloaded.
*/
def getModule(moduleId: ModuleID): ivySbt.Module = getModule(moduleId, None)
def getModule(moduleId: ModuleID, ivyScala: Option[IvyScala]): ivySbt.Module = {
val sha1 = Hash.toHex(Hash(moduleId.name))
val dummyID = ModuleID(sbtOrgTemp, modulePrefixTemp + sha1, moduleId.revision, moduleId.configurations)
getModule(dummyID, Seq(moduleId), UpdateOptions(), ivyScala)
}
def getModule(moduleId: ModuleID, deps: Seq[ModuleID],
uo: UpdateOptions = UpdateOptions(), ivyScala: Option[IvyScala]): ivySbt.Module = {
val moduleSetting = InlineConfiguration(
module = moduleId,
moduleInfo = ModuleInfo(moduleId.name),
dependencies = deps,
configurations = Seq(Configurations.Component),
ivyScala = ivyScala)
new ivySbt.Module(moduleSetting)
}
private def dependenciesNames(module: ivySbt.Module): String =
module.moduleSettings match {
// `module` is a dummy module, we will only fetch its dependencies.
case ic: InlineConfiguration =>
ic.dependencies map {
case mID: ModuleID =>
import mID._
s"$organization % $name % $revision"
} mkString ", "
case _ =>
s"unknown"
}
def update(module: ivySbt.Module, retrieveDirectory: File)(predicate: File => Boolean): Option[Seq[File]] = {
val retrieveConfiguration = new RetrieveConfiguration(retrieveDirectory, Resolver.defaultRetrievePattern, false)
val updateConfiguration = new UpdateConfiguration(Some(retrieveConfiguration), true, UpdateLogging.DownloadOnly)
log.debug(s"Attempting to fetch ${dependenciesNames(module)}. This operation may fail.")
IvyActions.updateEither(module, updateConfiguration, UnresolvedWarningConfiguration(), LogicalClock.unknown, None, log) match {
case Left(unresolvedWarning) =>
log.debug("Couldn't retrieve module ${dependenciesNames(module)}.")
None
case Right(updateReport) =>
val allFiles =
for {
conf <- updateReport.configurations
m <- conf.modules
(_, f) <- m.artifacts
} yield f
log.debug(s"Files retrieved for ${dependenciesNames(module)}:")
log.debug(allFiles mkString ", ")
allFiles filter predicate match {
case Seq() => None
case files => Some(files)
}
}
}
}

View File

@ -16,7 +16,7 @@ import java.io.File
import scala.util.control.NonFatal
object BasicCommands {
lazy val allBasicCommands = Seq(nop, ignore, help, completionsCommand, templateCommand, multi, ifLast, append, setOnFailure, clearOnFailure, stashOnFailure, popOnFailure, reboot, call, early, exit, continuous, history, shell, read, alias) ++ compatCommands
lazy val allBasicCommands = Seq(nop, ignore, help, completionsCommand, multi, ifLast, append, setOnFailure, clearOnFailure, stashOnFailure, popOnFailure, reboot, call, early, exit, continuous, history, shell, read, alias) ++ compatCommands
def nop = Command.custom(s => success(() => s))
def ignore = Command.command(FailureWall)(idFun)
@ -80,31 +80,6 @@ object BasicCommands {
state
}
def templateCommand = Command.make(TemplateCommand, templateBrief, templateDetailed)(templateCommandParser)
def templateCommandParser(state: State) =
{
val p = (token(Space) ~> repsep(StringBasic, token(Space))) | (token(EOF) map { case _ => Nil })
val trs = (state get templateResolvers) match {
case Some(trs) => trs.toList
case None => Nil
}
applyEffect(p)({ inputArg =>
val arguments = inputArg.toList ++
(state.remainingCommands.toList match {
case "shell" :: Nil => Nil
case xs => xs
})
trs find { tr =>
tr.isDefined(arguments.toArray)
} match {
case Some(tr) => tr.run(arguments.toArray)
case None =>
System.err.println("Template not found for: " + arguments.mkString(" "))
}
"exit" :: state.copy(remainingCommands = Nil)
})
}
def multiParser(s: State): Parser[Seq[String]] =
{
val nonSemi = token(charClass(_ != ';').+, hide = const(true))

View File

@ -11,5 +11,5 @@ object BasicKeys {
private[sbt] val classLoaderCache = AttributeKey[classpath.ClassLoaderCache]("class-loader-cache", "Caches class loaders based on the classpath entries and last modified times.", 10)
private[sbt] val OnFailureStack = AttributeKey[List[Option[String]]]("on-failure-stack", "Stack that remembers on-failure handlers.", 10)
private[sbt] val explicitGlobalLogLevels = AttributeKey[Boolean]("explicit-global-log-levels", "True if the global logging levels were explicitly set by the user.", 10)
private[sbt] val templateResolvers = AttributeKey[Seq[TemplateResolver]]("templateResolvers", "List of template resolvers.", 1000)
private[sbt] val templateResolverInfos = AttributeKey[Seq[TemplateResolverInfo]]("templateResolverInfos", "List of template resolver infos.", 1000)
}

View File

@ -254,3 +254,5 @@ object State {
private[sbt] def getBoolean(s: State, key: AttributeKey[Boolean], default: Boolean): Boolean =
s.get(key) getOrElse default
}
case class TemplateResolverInfo(module: ModuleID, implementationClass: String)

View File

@ -160,7 +160,7 @@ object Defaults extends BuildCommon {
maxErrors :== 100,
fork :== false,
initialize :== {},
templateResolvers :== Nil,
templateResolverInfos :== Nil,
forcegc :== sys.props.get("sbt.task.forcegc").map(java.lang.Boolean.parseBoolean).getOrElse(GCUtil.defaultForceGarbageCollection),
minForcegcInterval :== GCUtil.defaultMinForcegcInterval
))

View File

@ -1,32 +0,0 @@
package sbt
import sbt.template.TemplateResolver
object Giter8TemplateResolver extends TemplateResolver {
def isDefined(args0: Array[String]): Boolean =
{
val args = args0.toList filterNot { _.startsWith("-") }
// Mandate .g8
val Github = """^([^\s/]+)/([^\s/]+?)(?:\.g8)$""".r
val Local = """^file://(\S+)(?:\.g8)(?:/)?$""".r
object GitUrl {
val NativeUrl = "^(git[@|://].*)$".r
val HttpsUrl = "^(https://.*)$".r
val HttpUrl = "^(http://.*)$".r
val SshUrl = "^(ssh://.*)$".r
def unapplySeq(s: Any): Option[List[String]] =
NativeUrl.unapplySeq(s) orElse
HttpsUrl.unapplySeq(s) orElse
HttpUrl.unapplySeq(s) orElse
SshUrl.unapplySeq(s)
}
args.headOption match {
case Some(Github(_, _)) => true
case Some(Local(_)) => true
case GitUrl(uri) => uri contains (".g8")
case _ => false
}
}
def run(args: Array[String]): Unit =
giter8.Giter8.run(args)
}

View File

@ -346,7 +346,7 @@ object Keys {
val sbtVersion = SettingKey[String]("sbt-version", "Provides the version of sbt. This setting should be not be modified.", AMinusSetting)
val sbtBinaryVersion = SettingKey[String]("sbt-binary-version", "Defines the binary compatibility version substring.", BPlusSetting)
val skip = TaskKey[Boolean]("skip", "For tasks that support it (currently only 'compile' and 'update'), setting skip to true will force the task to not to do its work. This exact semantics may vary by task.", BSetting)
val templateResolvers = SettingKey[Seq[TemplateResolver]]("templateResolvers", "Template resolvers used for 'new'.", BSetting)
val templateResolverInfos = SettingKey[Seq[TemplateResolverInfo]]("templateResolverInfos", "Template resolvers used for 'new'.", BSetting)
// special
val sessionVars = AttributeKey[SessionVar.Map]("session-vars", "Bindings that exist for the duration of the session.", Invisible)

View File

@ -80,6 +80,7 @@ import CommandStrings._
import BasicCommandStrings._
import BasicCommands._
import CommandUtil._
import TemplateCommandUtil.templateCommand
object BuiltinCommands {
def initialAttributes = AttributeMap.empty

View File

@ -7,7 +7,7 @@ import java.io.File
import java.net.URI
import java.util.Locale
import Project.{ Initialize => _, Setting => _, _ }
import Keys.{ appConfiguration, stateBuildStructure, commands, configuration, historyPath, projectCommand, sessionSettings, shellPrompt, templateResolvers, thisProject, thisProjectRef, watch }
import Keys.{ appConfiguration, stateBuildStructure, commands, configuration, historyPath, projectCommand, sessionSettings, shellPrompt, templateResolverInfos, thisProject, thisProjectRef, watch }
import Scope.{ GlobalScope, ThisScope }
import Def.{ Flattened, Initialize, ScopedKey, Setting }
import Types.{ const, idFun }
@ -359,13 +359,13 @@ object Project extends ProjectExtra {
val allCommands = commandsIn(ref) ++ commandsIn(BuildRef(ref.build)) ++ (commands in Global get structure.data toList)
val history = get(historyPath) flatMap idFun
val prompt = get(shellPrompt)
val trs = (templateResolvers in Global get structure.data).toList.flatten
val trs = (templateResolverInfos in Global get structure.data).toList.flatten
val watched = get(watch)
val commandDefs = allCommands.distinct.flatten[Command].map(_ tag (projectCommand, true))
val newDefinedCommands = commandDefs ++ BasicCommands.removeTagged(s.definedCommands, projectCommand)
val newAttrs = setCond(Watched.Configuration, watched, s.attributes).
put(historyPath.key, history).
put(templateResolvers.key, trs)
put(templateResolverInfos.key, trs)
s.copy(attributes = setCond(shellPrompt.key, prompt, newAttrs), definedCommands = newDefinedCommands)
}
def setCond[T](key: AttributeKey[T], vopt: Option[T], attributes: AttributeMap): AttributeMap =

View File

@ -0,0 +1,91 @@
package sbt
import java.lang.reflect.InvocationTargetException
import java.io.File
import xsbti.AppConfiguration
import sbt.classpath.ClasspathUtilities
import BasicCommandStrings._
import BasicKeys._
import complete.{ Parser, DefaultParsers }
import DefaultParsers._
import Command.applyEffect
import Path._
private[sbt] object TemplateCommandUtil {
def templateCommand = Command.make(TemplateCommand, templateBrief, templateDetailed)(templateCommandParser)
def templateCommandParser(state: State) =
{
val p = (token(Space) ~> repsep(StringBasic, token(Space))) | (token(EOF) map { case _ => Nil })
val infos = (state get templateResolverInfos) match {
case Some(infos) => infos.toList
case None => Nil
}
val log = state.globalLogging.full
val extracted = (Project extract state)
val (s2, ivyConf) = extracted.runTask(Keys.ivyConfiguration, state)
val globalBase = BuildPaths.getGlobalBase(state)
val ivyScala = extracted.get(Keys.ivyScala in Keys.updateSbtClassifiers)
applyEffect(p)({ inputArg =>
val arguments = inputArg.toList ++
(state.remainingCommands.toList match {
case "shell" :: Nil => Nil
case xs => xs
})
run(infos, arguments, state.configuration, ivyConf, globalBase, ivyScala, log)
"exit" :: s2.copy(remainingCommands = Nil)
})
}
private def run(infos: List[TemplateResolverInfo], arguments: List[String], config: AppConfiguration,
ivyConf: IvyConfiguration, globalBase: File, ivyScala: Option[IvyScala], log: Logger): Unit =
infos find { info =>
val loader = infoLoader(info, config, ivyConf, globalBase, ivyScala, log)
val hit = tryTemplate(info, arguments, loader)
if (hit) {
runTemplate(info, arguments, loader)
}
hit
} match {
case Some(_) => // do nothing
case None => System.err.println("Template not found for: " + arguments.mkString(" "))
}
private def tryTemplate(info: TemplateResolverInfo, arguments: List[String], loader: ClassLoader): Boolean =
{
val resultObj = call(info.implementationClass, "isDefined", loader)(
classOf[Array[String]])(arguments.toArray)
resultObj.asInstanceOf[Boolean]
}
private def runTemplate(info: TemplateResolverInfo, arguments: List[String], loader: ClassLoader): Unit =
call(info.implementationClass, "run", loader)(classOf[Array[String]])(arguments.toArray)
private def infoLoader(info: TemplateResolverInfo, config: AppConfiguration,
ivyConf: IvyConfiguration, globalBase: File, ivyScala: Option[IvyScala], log: Logger): ClassLoader =
ClasspathUtilities.toLoader(classpathForInfo(info, ivyConf, globalBase, ivyScala, log), config.provider.loader)
private def call(interfaceClassName: String, methodName: String, loader: ClassLoader)(argTypes: Class[_]*)(args: AnyRef*): AnyRef =
{
val interfaceClass = getInterfaceClass(interfaceClassName, loader)
val interface = interfaceClass.newInstance.asInstanceOf[AnyRef]
val method = interfaceClass.getMethod(methodName, argTypes: _*)
try { method.invoke(interface, args: _*) }
catch {
case e: InvocationTargetException => throw e.getCause
}
}
private def getInterfaceClass(name: String, loader: ClassLoader) = Class.forName(name, true, loader)
// Cache files under ~/.sbt/0.13/templates/org_name_version
private def classpathForInfo(info: TemplateResolverInfo, ivyConf: IvyConfiguration, globalBase: File, ivyScala: Option[IvyScala], log: Logger): List[File] =
{
val updateUtil = new UpdateUtil(ivyConf, log)
val templatesBaseDirectory = new File(globalBase, "templates")
val templateId = s"${info.module.organization}_${info.module.name}_${info.module.revision}"
val templateDirectory = new File(templatesBaseDirectory, templateId)
def jars = (templateDirectory ** -DirectoryFilter).get
if (!(info.module.revision endsWith "-SNAPSHOT") && jars.nonEmpty) jars.toList
else {
IO.createDirectory(templateDirectory)
val m = updateUtil.getModule(info.module.copy(configurations = Some("component")), ivyScala)
val xs = updateUtil.update(m, templateDirectory)(_ => true).toList.flatten
xs
}
}
}

View File

@ -12,6 +12,8 @@ object Giter8TemplatePlugin extends AutoPlugin {
override lazy val globalSettings: Seq[Setting[_]] =
Seq(
templateResolvers += Giter8TemplateResolver
templateResolverInfos +=
TemplateResolverInfo(ModuleID("org.scala-sbt.sbt-giter8-resolver", "sbt-giter8-resolver", "0.1.0") cross CrossVersion.binary,
"sbtgiter8resolver.Giter8TemplateResolver")
)
}

View File

@ -12,7 +12,7 @@
### Improvements
- Adds `new` command and `templateResolvers`. See below for more details.
- Adds `new` command and `templateResolverInfos`. See below for more details.
- Auto plugins can add synthetic subprojects. See below for more details.
- Supports wildcard exclusions in POMs [#1431][1431]/[sbt/ivy#22][sbt-ivy-22]/[#2731][2731] by [@jtgrabowski][@jtgrabowski]
- Adds the ability to call `aggregateProjects(..)` for the current project inside a build sbt file. [#2682][2682] by [@xuwei-k][@xuwei-k]
@ -32,7 +32,7 @@
- Fixes forked tests being reported as successful when the test harness fails. [#2442][2442]/[#2722][2722]/[#2730][2730] by [@eed3si9n][@eed3si9n]/[@dwijnand][@dwijnand]
- Fixes incorrect installation path on Windows. [sbt/sbt-launcher-package#110][110] by [@dwijnand][@dwijnand]
### new command and templateResolvers
### new command and templateResolverInfos
sbt 0.13.13 adds a `new` command, which helps create new build definitions.
The `new` command is extensible via a mechanism called the template resolver,

View File

@ -22,7 +22,6 @@ object Dependencies {
lazy val launcherInterface = "org.scala-sbt" % "launcher-interface" % "1.0.0-M1"
lazy val rawLauncher = "org.scala-sbt" % "launcher" % "1.0.0-M1"
lazy val templateResolverApi = "org.scala-sbt" % "template-resolver" % "0.1"
lazy val giter8 = "org.foundweekends.giter8" %% "giter8" % "0.7.1"
private def scala211Module(name: String, moduleVersion: String) =
Def.setting {

View File

@ -4,7 +4,7 @@ addSbtPlugin("com.eed3si9n" % "sbt-doge" % "0.1.5")
addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.5.4")
addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "0.8.5")
addSbtPlugin("com.typesafe.sbt" % "sbt-javaversioncheck" % "0.1.0")
addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0")
addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0")
addSbtPlugin("com.typesafe.sbt" % "sbt-scalariform" % "1.3.0") // 1.6.0 is out but is a hard upgrade
addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "0.8.2")
addSbtPlugin("me.lessis" % "bintray-sbt" % "0.3.0")