retrieve to build, update-classifiers action

set retrieve := true to have dependencies retrieved to the build
the location is by default shared by all projects in a build
  (<built-root>/lib_managed/), but can be per-project
update-classifiers and update-sbt-classifiers retrieves artifacts with classifiers
  for project dependencies and for sbt, respectively
The default setting is classifiers := Seq("javadoc", "sources")

ivy/Ivy.scala

@@ -283,13 +283,16 @@ private object IvySbt
 		configurations.foreach(artifact.addConfiguration)
 		artifact
 	}
-	private def extra(artifact: Artifact) =
+	private[sbt] def extra(artifact: Artifact, unqualify: Boolean = false): java.util.Map[String, String] =
 	{
 		val ea = artifact.classifier match { case Some(c) => artifact.extra("e:classifier" -> c); case None => artifact }
-		javaMap(ea.extraAttributes)
+		javaMap(ea.extraAttributes, unqualify)
 	}
-	private def javaMap(map: Map[String,String]) =
+	private[sbt] def javaMap(m: Map[String,String], unqualify: Boolean = false) =
+	{
+		val map = m map { case (k, v) => (k.stripPrefix("e:"), v) }
 		if(map.isEmpty) null else scala.collection.JavaConversions.asJavaMap(map)
+	}
 
 	private object javaMap
 	{

ivy/IvyActions.scala

@@ -23,8 +23,8 @@ import plugins.parser.m2.{PomModuleDescriptorParser,PomModuleDescriptorWriter}
 final class PublishPatterns(val deliverIvyPattern: Option[String], val srcArtifactPatterns: Seq[String])
 final class PublishConfiguration(val patterns: PublishPatterns, val status: String, val resolverName: String, val configurations: Option[Seq[Configuration]], val logging: UpdateLogging.Value)
 
-final class UpdateConfiguration(val retrieve: Option[RetrieveConfiguration], val logging: UpdateLogging.Value)
+final class UpdateConfiguration(val retrieve: Option[RetrieveConfiguration], val missingOk: Boolean, val logging: UpdateLogging.Value)
-final class RetrieveConfiguration(val retrieveDirectory: File, val outputPattern: String, val synchronize: Boolean)
+final class RetrieveConfiguration(val retrieveDirectory: File, val outputPattern: String)
 final class MakePomConfiguration(val file: File, val configurations: Option[Iterable[Configuration]] = None, val extra: NodeSeq = NodeSeq.Empty, val process: Node => Node = n => n, val filterRepositories: MavenRepository => Boolean = _ => true)
 
 /** Configures logging during an 'update'.  `level` determines the amount of other information logged.
@@ -81,11 +81,7 @@ object IvyActions
 			ivy.deliver(revID, revID.getRevision, getDeliverIvyPattern(patterns), options)
 		}
 	}
-	// because Ivy.deliver does not provide the delivered File location, we duplicate the logic here
+
-	def deliverFile(module: IvySbt#Module, configuration: PublishConfiguration): File =
-		module.withModule { (ivy,md,_) =>
-			ivy.getSettings.resolveFile(IvyPatternHelper.substitute(getDeliverIvyPattern(configuration.patterns), md.getResolvedModuleRevisionId))
-		}
 	def getDeliverIvyPattern(patterns: PublishPatterns) = patterns.deliverIvyPattern.getOrElse(error("No Ivy pattern specified"))
 
 	// todo: map configurations, extra dependencies
@@ -106,21 +102,72 @@ object IvyActions
 	/** Resolves and retrieves dependencies.  'ivyConfig' is used to produce an Ivy file and configuration.
 	* 'updateConfig' configures the actual resolution and retrieval process. */
 	def update(module: IvySbt#Module, configuration: UpdateConfiguration): UpdateReport =
-	{
 		module.withModule { case (ivy, md, default) =>
-			import configuration.{retrieve => rConf, logging}
+			val (report, err) = resolve(configuration.logging)(ivy, md, default)
-			val report = resolve(logging)(ivy, md, default)
+			err match
-			IvyRetrieve.updateReport(report)
+			{
+				case Some(x) if !configuration.missingOk => throw x
+				case _ =>
+					val uReport = IvyRetrieve updateReport report
+					configuration.retrieve match
+					{
+						case Some(rConf) => retrieve(ivy, uReport, rConf)
+						case None => uReport
+					}
+			}
 		}
+
+	def transitiveScratch(ivySbt: IvySbt, id: ModuleID, label: String, deps: Seq[ModuleID], classifiers: Seq[String], c: UpdateConfiguration, ivyScala: Option[IvyScala]): UpdateReport =
+	{
+		val base = id.copy(name = id.name + "$" + label)
+		val module = new ivySbt.Module(InlineConfiguration(base, deps).copy(ivyScala = ivyScala))
+		val report = update(module, c)
+		transitive(ivySbt, id, report, classifiers, c, ivyScala)
 	}
-	private def resolve(logging: UpdateLogging.Value)(ivy: Ivy, module: DefaultModuleDescriptor, defaultConf: String): ResolveReport =
+	def transitive(ivySbt: IvySbt, module: ModuleID, report: UpdateReport, classifiers: Seq[String], c: UpdateConfiguration, ivyScala: Option[IvyScala]): UpdateReport =
+		updateClassifiers(ivySbt, module, report.allModules, classifiers, new UpdateConfiguration(c.retrieve, true, c.logging), ivyScala)
+	def updateClassifiers(ivySbt: IvySbt, id: ModuleID, modules: Seq[ModuleID], classifiers: Seq[String], configuration: UpdateConfiguration, ivyScala: Option[IvyScala]): UpdateReport =
+	{
+		assert(!classifiers.isEmpty, "classifiers cannot be empty")
+		val baseModules = modules map { m => ModuleID(m.organization, m.name, m.revision, crossVersion = m.crossVersion) }
+		val deps = baseModules.distinct map { m => m.copy(explicitArtifacts = classifiers map { c => Artifact(m.name, c) }) }
+		val base = id.copy(name = id.name + classifiers.mkString("$","_",""))
+		val module = new ivySbt.Module(InlineConfiguration(base, deps).copy(ivyScala = ivyScala))
+		update(module, configuration)
+	}
+	private def resolve(logging: UpdateLogging.Value)(ivy: Ivy, module: DefaultModuleDescriptor, defaultConf: String): (ResolveReport, Option[ResolveException]) =
 	{
 		val resolveOptions = new ResolveOptions
 		resolveOptions.setLog(ivyLogLevel(logging))
 		val resolveReport = ivy.resolve(module, resolveOptions)
-		if(resolveReport.hasError)
+		val err =
-			throw new ResolveException(resolveReport.getAllProblemMessages.toArray.map(_.toString).distinct)
+			if(resolveReport.hasError)
-		resolveReport
+				Some(new ResolveException(resolveReport.getAllProblemMessages.toArray.map(_.toString).distinct))
+			else None
+		(resolveReport, err)
+	}
+	private def retrieve(ivy: Ivy, report: UpdateReport, config: RetrieveConfiguration): UpdateReport =
+		retrieve(ivy, report, config.retrieveDirectory, config.outputPattern)
+	
+	private def retrieve(ivy: Ivy, report: UpdateReport, base: File, pattern: String): UpdateReport =
+	{
+		val toCopy = new collection.mutable.HashSet[(File,File)]
+		val retReport = report retrieve { (conf, mid, art, cached) =>
+			val to = retrieveTarget(conf, mid, art, base, pattern)
+			toCopy += ((cached, to))
+			to
+		}
+		IO.copy( toCopy )
+		retReport
+	}
+	private def retrieveTarget(conf: String, mid: ModuleID, art: Artifact, base: File, pattern: String): File =
+		new File(base, substitute(conf, mid, art, pattern))
+
+	private def substitute(conf: String, mid: ModuleID, art: Artifact, pattern: String): String =
+	{
+		val mextra = IvySbt.javaMap(mid.extraAttributes, true)
+		val aextra = IvySbt.extra(art, true)
+		IvyPatternHelper.substitute(pattern, mid.organization, mid.name, mid.revision, art.name, art.`type`, art.extension, conf, mextra, aextra)
 	}
 
 	import UpdateLogging.{Quiet, Full, DownloadOnly}

ivy/IvyInterface.scala

@@ -11,7 +11,7 @@ import org.apache.ivy.util.url.CredentialsStore
 
 final case class ModuleID(organization: String, name: String, revision: String, configurations: Option[String] = None, isChanging: Boolean = false, isTransitive: Boolean = true, explicitArtifacts: Seq[Artifact] = Nil, extraAttributes: Map[String,String] = Map.empty, crossVersion: Boolean = false)
 {
-	override def toString = organization + ":" + name + ":" + revision
+	override def toString = organization + ":" + name + ":" + revision + (configurations match { case Some(s) => ":" + s; case None => "" })
 	def cross(v: Boolean) = copy(crossVersion = v)
 	// () required for chaining
 	def notTransitive() = intransitive()

ivy/IvyRetrieve.scala

@@ -4,6 +4,7 @@
 package sbt
 
 import java.io.File
+import collection.mutable
 
 import org.apache.ivy.core.{module, report}
 import module.descriptor.{Artifact => IvyArtifact}
@@ -16,7 +17,7 @@ object IvyRetrieve
 		( for( conf <- report.getConfigurations) yield (conf, report.getConfigurationReport(conf)) ).toMap
 
 	def moduleReports(confReport: ConfigurationResolveReport): Map[ModuleID, ModuleReport] =
-		moduleReportMap(confReport) map { case (mid, arts) => (mid, new ModuleReport(mid, artifactReports(arts)) ) }
+		moduleReportMap(confReport) map { case (mid, arts) => (mid, artifactReports(mid, arts) ) }
 
 	def moduleReportMap(confReport: ConfigurationResolveReport): Map[ModuleID, Seq[ArtifactDownloadReport]] =
 	{
@@ -25,13 +26,20 @@ object IvyRetrieve
 				(toModuleID(revId), (confReport getDownloadReports revId).toSeq)
 		modules.toMap
 	}
-	def artifactReports(artReport: Seq[ArtifactDownloadReport]): Map[Artifact, File] =
+	def artifactReports(mid: ModuleID, artReport: Seq[ArtifactDownloadReport]): ModuleReport =
-		artReport map { r =>
+	{
-			val art = r.getArtifact
+		val missing = new mutable.ListBuffer[Artifact]
-			val file0 = r.getLocalFile
+		val resolved = new mutable.ListBuffer[(Artifact,File)]
-			val file = if(file0 eq null) error("No file for " + art) else file0
+		for(r <- artReport) {
-			(toArtifact(art), file)
+			val file = r.getLocalFile
-		} toMap;
+			val art = toArtifact(r.getArtifact)
+			if(file eq null)
+				missing += art
+			else
+				resolved += ((art,file))
+		}
+		new ModuleReport(mid, resolved.toMap, missing.toSet)
+	}
  
 	def toModuleID(revID: ModuleRevisionId): ModuleID =
 		ModuleID(revID.getOrganisation, revID.getName, revID.getRevision)
@@ -52,12 +60,25 @@ object IvyRetrieve
 final class UpdateReport(val configurations: Map[String, ConfigurationReport])
 {
 	override def toString = "Update report:\n" + configurations.values.mkString
+	def allModules: Seq[ModuleID] = configurations.values.toSeq.flatMap(_.allModules).distinct
+	def retrieve(f: (String, ModuleID, Artifact, File) => File): UpdateReport =
+		new UpdateReport(configurations map { case (k,v) => (k, v retrieve f)} )
 }
 final class ConfigurationReport(val configuration: String, val modules: Map[ModuleID, ModuleReport])
 {
 	override def toString = "\t" + configuration + ":\n" + modules.values.mkString
+	def allModules: Seq[ModuleID] = modules.keys.toSeq
+	def retrieve(f: (String, ModuleID, Artifact, File) => File): ConfigurationReport =
+		new ConfigurationReport(configuration, modules map { case (k,v) =>  (k, v.retrieve( (mid,art,file) => f(configuration, mid, art, file)) ) })
 }
-final class ModuleReport(val module: ModuleID, val artifacts: Map[Artifact, File])
+final class ModuleReport(val module: ModuleID, val artifacts: Map[Artifact, File], val missingArtifacts: Set[Artifact])
 {
-	override def toString = "\t\t" + module + ": " + (if(artifacts.size <= 1) "" else "\n") + artifacts.mkString("\n\t\t\t") + "\n"
+	override def toString =
+	{
+		val arts = artifacts.map(_.toString) ++ missingArtifacts.map(art => "(MISSING) " + art)
+		"\t\t" + module + ": " +
+			(if(arts.size <= 1) "" else "\n\t\t\t") + arts.mkString("\n\t\t\t") + "\n"
+	}
+	def retrieve(f: (ModuleID, Artifact, File) => File): ModuleReport =
+		new ModuleReport(module, artifacts.map { case (art,file) => (art, f(module, art, file)) }, missingArtifacts)
 }

main/Build.scala

@@ -11,7 +11,7 @@ package sbt
 	import collection.mutable
 	import Compiler.{Compilers,Inputs}
 	import Project.{inScope, ScopedKey, ScopeLocal, Setting}
-	import Keys.{appConfiguration, configuration, thisProject, thisProjectRef}
+	import Keys.{appConfiguration, baseDirectory, configuration, thisProject, thisProjectRef}
 	import TypeFunctions.{Endo,Id}
 	import tools.nsc.reporters.ConsoleReporter
 	import Build.{analyzed, data}
@@ -328,7 +328,8 @@ object Load
 				transformSettings(projectScope(ref), uri, rootProject, settings)
 			}
 			val buildScope = Scope(Select(BuildRef(uri)), Global, Global, Global)
-			pluginGlobal ++ inScope(buildScope)(build.buildSettings) ++ projectSettings
+			val buildBase = baseDirectory :== build.localBase
+			pluginGlobal ++ inScope(buildScope)(buildBase +: build.buildSettings) ++ projectSettings
 		}
 	def transformSettings(thisScope: Scope, uri: URI, rootProject: URI => String, settings: Seq[Setting[_]]): Seq[Setting[_]] =
 		Project.transform(Scope.resolveScope(thisScope, uri, rootProject), settings)
@@ -372,7 +373,11 @@ object Load
 		(new PartBuildUnit(unit, defined.map(d => (d.id, d)).toMap, rootProjects, buildSettings(unit)), externals)
 	}
 	def buildSettings(unit: BuildUnit): Seq[Setting[_]] =
-		unit.definitions.builds.flatMap(_.settings)
+	{
+		val buildScope = GlobalScope.copy(project = Select(BuildRef(unit.uri)))
+		val resolve = Scope.resolveBuildScope(buildScope, unit.uri)
+		Project.transform(resolve, unit.definitions.builds.flatMap(_.settings))
+	}
 
 	@tailrec def loadAll(bases: List[URI], references: Map[URI, List[ProjectReference]], externalLoader: URI => BuildUnit, builds: Map[URI, PartBuildUnit]): (Map[URI, List[ProjectReference]], Map[URI, PartBuildUnit]) =
 		bases match

main/CacheIvy.scala

@@ -11,7 +11,7 @@ package sbt
 	import Types.:+:
 	import scala.xml.NodeSeq
 	import sbinary.{DefaultProtocol,Format}
-	import DefaultProtocol.{immutableMapFormat, optionsAreFormat}
+	import DefaultProtocol.{immutableMapFormat, immutableSetFormat, optionsAreFormat}
 	import RepositoryHelpers._
 	import Ordering._
 
@@ -65,7 +65,7 @@ object CacheIvy
 	implicit def confReportFormat(implicit mf: Format[ModuleID], mr: Format[ModuleReport]): Format[ConfigurationReport] =
 		wrap[ConfigurationReport, (String,Map[ModuleID,ModuleReport])]( r => (r.configuration, r.modules), { case (c,m) => new ConfigurationReport(c,m) })
 	implicit def moduleReportFormat(implicit f: Format[Artifact], ff: Format[File], mid: Format[ModuleID]): Format[ModuleReport] =
-		wrap[ModuleReport, (ModuleID, Map[Artifact, File])]( m => (m.module, m.artifacts), { case (m, as) => new ModuleReport(m, as) })
+		wrap[ModuleReport, (ModuleID, Map[Artifact, File], Set[Artifact])]( m => (m.module, m.artifacts, m.missingArtifacts), { case (m, as, ms) => new ModuleReport(m, as,ms) })
 	implicit def artifactFormat(implicit sf: Format[String], of: Format[Seq[Configuration]], cf: Format[Configuration], uf: Format[Option[URL]]): Format[Artifact] =
 		wrap[Artifact, (String,String,String,Option[String],Seq[Configuration],Option[URL],Map[String,String])](
 			a => (a.name, a.`type`, a.extension, a.classifier, a.configurations.toSeq, a.url, a.extraAttributes),
@@ -126,7 +126,7 @@ object CacheIvy
 	implicitly[InputCache[Seq[Configuration]]]
 
 	object L2 {
-		implicit def updateConfToHL = (u: UpdateConfiguration) => u.retrieve :+: HNil
+		implicit def updateConfToHL = (u: UpdateConfiguration) => u.retrieve :+: u.missingOk :+: HNil
 		implicit def pomConfigurationHL = (c: PomConfiguration) => hash(c.file) :+: c.ivyScala :+: c.validate :+: HNil
 		implicit def ivyFileConfigurationHL = (c: IvyFileConfiguration) => hash(c.file) :+: c.ivyScala :+: c.validate :+: HNil
 		implicit def sshConnectionToHL = (s: SshConnection) => s.authentication :+: s.hostname :+: s.port :+: HNil
@@ -145,7 +145,7 @@ object CacheIvy
 	implicit def publishConfIC: InputCache[PublishConfiguration] = wrapIn
 
 	object L1 {
-		implicit def retrieveToHL = (r: RetrieveConfiguration) => exists(r.retrieveDirectory) :+: r.outputPattern :+: r.synchronize :+: HNil
+		implicit def retrieveToHL = (r: RetrieveConfiguration) => exists(r.retrieveDirectory) :+: r.outputPattern :+: HNil
 		implicit def ivyPathsToHL = (p: IvyPaths) => exists(p.baseDirectory) :+: p.cacheDirectory.map(exists.apply) :+: HNil
 		implicit def ivyScalaHL = (i: IvyScala) => i.scalaVersion :+: names(i.configurations) :+: i.checkExplicit :+: i.filterImplicit :+: HNil
 		implicit def configurationToHL = (c: Configuration) => c.name :+: c.description :+: c.isPublic :+: names(c.extendsConfigs) :+: c.transitive :+: HNil

main/Defaults.scala

@@ -57,7 +57,11 @@ object Defaults
 	def analysisMap[T](cp: Seq[Attributed[T]]): Map[T, inc.Analysis] =
 		(cp map extractAnalysis).toMap
 
-	def buildCore: Seq[Setting[_]] = inScope(GlobalScope)(Seq(
+	def buildCore: Seq[Setting[_]] = thisBuildCore ++ globalCore
+	def thisBuildCore: Seq[Setting[_]] = inScope(GlobalScope.copy(project = Select(ThisBuild)))(Seq(
+		managedDirectory <<= baseDirectory(_ / "lib_managed")
+	))
+	def globalCore: Seq[Setting[_]] = inScope(GlobalScope)(Seq(
 		pollInterval :== 500,
 		scalaHome :== None,
 		javaHome :== None,
@@ -74,6 +78,7 @@ object Defaults
 		timingFormat :== Aggregation.defaultFormat,
 		showSuccess :== true,
 		commands :== Nil,
+		retrieveManaged :== false,
 		settings <<= EvaluateTask.state map { state => Project.structure(state).data }
 	))
 	def projectCore: Seq[Setting[_]] = Seq(
@@ -473,8 +478,8 @@ object Classpaths
 		resolvers in GlobalScope :== Nil,
 		projectDescriptors <<= depMap,
 		retrievePattern :== "[type]/[organisation]/[module]/[artifact](-[revision])(-[classifier]).[ext]",
-		updateConfiguration <<= (retrieveConfiguration, ivyLoggingLevel)((conf,level) => new UpdateConfiguration(conf, level) ),
+		updateConfiguration <<= (retrieveConfiguration, ivyLoggingLevel)((conf,level) => new UpdateConfiguration(conf, false, level) ),
-		retrieveConfiguration :== None, //Some(new RetrieveConfiguration(managedDependencyPath asFile, retrievePattern, true))
+		retrieveConfiguration <<= (managedDirectory, retrievePattern, retrieveManaged) { (libm, pattern, enabled) => if(enabled) Some(new RetrieveConfiguration(libm, pattern)) else None },
 		ivyConfiguration <<= (fullResolvers, ivyPaths, otherResolvers, moduleConfigurations, offline, appConfiguration) map { (rs, paths, other, moduleConfs, off, app) =>
 			// todo: pass logger from streams directly to IvyActions
 			val lock = app.provider.scalaProvider.launcher.globalLock
@@ -490,6 +495,13 @@ object Classpaths
 		ivyModule <<= (ivySbt, moduleSettings) map { (ivySbt, settings) => new ivySbt.Module(settings) },
 		update <<= (ivyModule, updateConfiguration, cacheDirectory, streams) map { (module, config, cacheDirectory, s) =>
 			cachedUpdate(cacheDirectory / "update", module, config, s.log)
+		},
+		transitiveClassifiers :== Seq("sources", "javadoc"),
+		updateClassifiers <<= (ivySbt, projectID, update, transitiveClassifiers, updateConfiguration, ivyScala) map IvyActions.transitive,
+		updateSbtClassifiers <<= (ivySbt, projectID, transitiveClassifiers, updateConfiguration, appConfiguration, ivyScala) map { (is, pid, classifiers, c, app, ivyScala) =>
+			val id = app.provider.id
+			val module = ModuleID(id.groupID, id.name, id.version, crossVersion = id.crossVersioned)
+			IvyActions.transitiveScratch(is, pid, "sbt", module :: Nil, classifiers, c, ivyScala)
 		}
 	)
 

main/Keys.scala

@@ -150,6 +150,9 @@ object Keys
 	val ivyModule = TaskKey[IvySbt#Module]("ivy-module")
 	val classpathFilter = SettingKey[FileFilter]("classpath-filter")
 	val update = TaskKey[UpdateReport]("update")
+	val updateClassifiers = TaskKey[UpdateReport]("update-classifiers")
+	val transitiveClassifiers = SettingKey[Seq[String]]("transitive-classifiers")
+	val updateSbtClassifiers = TaskKey[UpdateReport]("update-sbt-classifiers")
 	
 	val publishConfiguration = TaskKey[PublishConfiguration]("publish-configuration")
 	val publishLocalConfiguration = TaskKey[PublishConfiguration]("publish-local-configuration")
@@ -190,6 +193,8 @@ object Keys
 	val artifacts = SettingKey[Seq[Artifact]]("artifacts")
 	val projectDescriptors = TaskKey[Map[ModuleRevisionId,ModuleDescriptor]]("project-descriptors")
 	val autoUpdate = SettingKey[Boolean]("auto-update")
+	val retrieveManaged = SettingKey[Boolean]("retrieve-managed")
+	val managedDirectory = SettingKey[File]("managed-directory")
 	
 	// special
 	val settings = TaskKey[Settings[Scope]]("settings")

main/Scope.scala

@@ -15,6 +15,9 @@ object Scope
 	def resolveScope(thisScope: Scope, current: URI, rootProject: URI => String): Scope => Scope =
 		resolveProject(current, rootProject) compose replaceThis(thisScope)
 
+	def resolveBuildScope(thisScope: Scope, current: URI): Scope => Scope =
+		buildResolve(current) compose replaceThis(thisScope)
+
 	def replaceThis(thisScope: Scope): Scope => Scope = (scope: Scope) =>
 		Scope(subThis(thisScope.project, scope.project), subThis(thisScope.config, scope.config), subThis(thisScope.task, scope.task), subThis(thisScope.extra, scope.extra))
 		
@@ -28,11 +31,15 @@ object Scope
 			case _ => scope.copy(task = Select(key))
 		}
 		
-	def resolveProject(uri: URI, rootProject: URI => String): Scope => Scope =
+	def mapReference(f: Reference => Reference): Scope => Scope =
 		{
-			case Scope(Select(ref), a,b,c) => Scope(Select(resolveReference(uri, rootProject, ref)), a,b,c)
+			case Scope(Select(ref), a,b,c) => Scope(Select(f(ref)), a,b,c)
 			case x => x
 		}
+	def resolveProject(uri: URI, rootProject: URI => String): Scope => Scope =
+		mapReference(ref => resolveReference(uri, rootProject, ref))
+	def buildResolve(uri: URI): Scope => Scope =
+		mapReference(ref => resolveBuildOnly(uri, ref))
 
 	def resolveBuildOnly(current: URI, ref: Reference): Reference =
 		ref match