luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #373 from alexarchambault/topic/fixes 

Various fixes
This commit is contained in:
Alexandre Archambault 2016-10-31 14:18:46 +01:00 committed by GitHub
parent e858287f19 92c22a03bd
commit 57a5c1b83f
22 changed files with 368 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
cli/src/main/scala-2.11/coursier/cli/Bootstrap.scala
View File

@ -11,6 +11,8 @@ import caseapp._
import coursier.cli.util.Zip
case class Bootstrap(
@Recurse
artifactOptions: ArtifactOptions,
@Recurse
options: BootstrapOptions
) extends App {
@ -78,6 +80,7 @@ case class Bootstrap(
def subFiles0 = helper.fetch(
sources = false,
javadoc = false,
artifactTypes = artifactOptions.artifactTypes,
subset = isolatedDeps.getOrElse(target, Seq.empty).toSet
)
@ -96,11 +99,19 @@ case class Bootstrap(
if (options.standalone)
(
Seq.empty[String],
helper.fetch(sources = false, javadoc = false)
helper.fetch(
sources = false,
javadoc = false,
artifactTypes = artifactOptions.artifactTypes
)
)
else
(
helper.artifacts(sources = false, javadoc = false).map(_.url),
helper.artifacts(
sources = false,
javadoc = false,
artifactTypes = artifactOptions.artifactTypes
).map(_.url),
Seq.empty[File]
)

8
cli/src/main/scala-2.11/coursier/cli/Fetch.scala
View File

@ -12,9 +12,13 @@ case class Fetch(
options: FetchOptions
) extends App {
val helper = new Helper(options.common, remainingArgs, ignoreErrors = options.force)
val helper = new Helper(options.common, remainingArgs, ignoreErrors = options.artifactOptions.force)
val files0 = helper.fetch(sources = options.sources, javadoc = options.javadoc)
val files0 = helper.fetch(
sources = options.sources,
javadoc = options.javadoc,
artifactTypes = options.artifactOptions.artifactTypes
)
val out =
if (options.classpath)

41
cli/src/main/scala-2.11/coursier/cli/Helper.scala
View File

@ -492,6 +492,7 @@ class Helper(
def artifacts(
sources: Boolean,
javadoc: Boolean,
artifactTypes: Set[String],
subset: Set[Dependency] = null
): Seq[Artifact] = {
@ -514,25 +515,34 @@ class Helper(
val res0 = Option(subset).fold(res)(res.subset)
if (classifier0.nonEmpty || sources || javadoc) {
var classifiers = classifier0
if (sources)
classifiers = classifiers :+ "sources"
if (javadoc)
classifiers = classifiers :+ "javadoc"
val artifacts0 =
if (classifier0.nonEmpty || sources || javadoc) {
var classifiers = classifier0
if (sources)
classifiers = classifiers :+ "sources"
if (javadoc)
classifiers = classifiers :+ "javadoc"
res0.classifiersArtifacts(classifiers.distinct)
} else
res0.artifacts
res0.dependencyClassifiersArtifacts(classifiers.distinct).map(_._2)
} else
res0.dependencyArtifacts.map(_._2)
if (artifactTypes("*"))
artifacts0
else
artifacts0.filter { artifact =>
artifactTypes(artifact.`type`)
}
}
def fetch(
sources: Boolean,
javadoc: Boolean,
artifactTypes: Set[String],
subset: Set[Dependency] = null
): Seq[File] = {
val artifacts0 = artifacts(sources, javadoc, subset)
val artifacts0 = artifacts(sources, javadoc, artifactTypes, subset)
val logger =
if (verbosityLevel >= 0)
@ -588,7 +598,15 @@ class Helper(
lazy val (parentLoader, filteredFiles) = {
val files0 = fetch(sources = false, javadoc = false)
// FIXME That shouldn't be hard-coded this way...
// This whole class ought to be rewritten more cleanly.
val artifactTypes = Set("jar")
val files0 = fetch(
sources = false,
javadoc = false,
artifactTypes = artifactTypes
)
if (isolated.isolated.isEmpty)
(baseLoader, files0)
@ -603,6 +621,7 @@ class Helper(
val isolatedFiles = fetch(
sources = false,
javadoc = false,
artifactTypes = artifactTypes,
subset = isolatedDeps.getOrElse(target, Seq.empty).toSet
)

33
cli/src/main/scala-2.11/coursier/cli/Options.scala
View File

@ -168,6 +168,33 @@ case class IsolatedLoaderOptions(
}
object ArtifactOptions {
def defaultArtifactTypes = Set("jar", "bundle")
}
case class ArtifactOptions(
@Help("Artifact types that should be retained (e.g. jar, src, doc, etc.) - defaults to jar,bundle")
@Value("type1,type2,...")
@Short("A")
artifactType: List[String],
@Help("Fetch artifacts even if the resolution is errored")
force: Boolean
) {
lazy val artifactTypes = {
val types0 = artifactType
.flatMap(_.split(','))
.filter(_.nonEmpty)
.toSet
if (types0.isEmpty)
ArtifactOptions.defaultArtifactTypes
else if (types0("*"))
Set("*")
else
types0
}
}
case class FetchOptions(
@Help("Fetch source artifacts")
@Short("S")
@ -178,8 +205,8 @@ case class FetchOptions(
@Help("Print java -cp compatible output")
@Short("p")
classpath: Boolean,
@Help("Fetch artifacts even if the resolution is errored")
force: Boolean,
@Recurse
artifactOptions: ArtifactOptions,
@Recurse
common: CommonOptions
)
@ -245,6 +272,8 @@ case class SparkSubmitOptions(
@Help("Maximum idle time of spark-submit (time with no output). Exit early if no output from spark-submit for more than this duration. Set to 0 for unlimited. (Default: 0)")
@Value("seconds")
maxIdleTime: Int,
@Recurse
artifactOptions: ArtifactOptions,
@Recurse
common: CommonOptions
)

8
cli/src/main/scala-2.11/coursier/cli/SparkSubmit.scala
View File

@ -70,8 +70,11 @@ case class SparkSubmit(
extraJars = rawExtraJars
)
val jars =
helper.fetch(sources = false, javadoc = false) ++
options.extraJars.map(new File(_))
helper.fetch(
sources = false,
javadoc = false,
artifactTypes = options.artifactOptions.artifactTypes
) ++ options.extraJars.map(new File(_))
val (scalaVersion, sparkVersion) =
if (options.sparkVersion.isEmpty)
@ -170,6 +173,7 @@ case class SparkSubmit(
sparkVersion,
options.noDefaultSubmitDependencies,
options.submitDependencies.flatMap(_.split(",")).filter(_.nonEmpty),
options.artifactOptions.artifactTypes,
options.common
)

7
cli/src/main/scala-2.11/coursier/cli/spark/Assembly.scala
View File

@ -140,14 +140,15 @@ object Assembly {
sparkVersion: String,
noDefault: Boolean,
extraDependencies: Seq[String],
options: CommonOptions
options: CommonOptions,
artifactTypes: Set[String] = Set("jar")
): Either[String, (File, Seq[File])] = {
val base = if (noDefault) Seq() else sparkAssemblyDependencies(scalaVersion, sparkVersion)
val helper = new Helper(options, extraDependencies ++ base)
val artifacts = helper.artifacts(sources = false, javadoc = false)
val jars = helper.fetch(sources = false, javadoc = false)
val artifacts = helper.artifacts(sources = false, javadoc = false, artifactTypes = artifactTypes)
val jars = helper.fetch(sources = false, javadoc = false, artifactTypes = artifactTypes)
val checksums = artifacts.map { a =>
val f = a.checksumUrls.get("SHA-1") match {

7
cli/src/main/scala-2.11/coursier/cli/spark/Submit.scala
View File

@ -11,6 +11,7 @@ object Submit {
sparkVersion: String,
noDefault: Boolean,
extraDependencies: Seq[String],
artifactTypes: Set[String],
common: CommonOptions
): Seq[File] = {
@ -43,7 +44,11 @@ object Submit {
(if (noDefault) Nil else defaultDependencies) ++ extraDependencies
)
helper.fetch(sources = false, javadoc = false) ++ extraCp
helper.fetch(
sources = false,
javadoc = false,
artifactTypes = artifactTypes
) ++ extraCp
}
def mainClassName = "org.apache.spark.deploy.SparkSubmit"

5
core/shared/src/main/scala/coursier/core/Definitions.scala
View File

@ -197,7 +197,10 @@ final case class Artifact(
attributes: Attributes,
changing: Boolean,
authentication: Option[Authentication]
)
) {
def `type`: String = attributes.`type`
def classifier: String = attributes.classifier
}
object Artifact {
trait Source {

7
core/shared/src/main/scala/coursier/ivy/IvyRepository.scala
View File

@ -72,10 +72,9 @@ case class IvyRepository(
case None =>
project.publications.collect {
case (conf, p)
if (conf == "*" ||
conf == dependency.configuration ||
project.allConfigurations.getOrElse(dependency.configuration, Set.empty).contains(conf)
) && p.classifier.isEmpty =>
if conf == "*" ||
conf == dependency.configuration ||
project.allConfigurations.getOrElse(dependency.configuration, Set.empty).contains(conf) =>
p
}
case Some(classifiers) =>

30
core/shared/src/main/scala/coursier/util/Print.scala
View File

@ -54,8 +54,23 @@ object Print {
resolution: Resolution,
printExclusions: Boolean,
reverse: Boolean
): String =
dependencyTree(roots, resolution, printExclusions, reverse, colors = true)
def dependencyTree(
roots: Seq[Dependency],
resolution: Resolution,
printExclusions: Boolean,
reverse: Boolean,
colors: Boolean
): String = {
val (red, yellow, reset) =
if (colors)
(Console.RED, Console.YELLOW, Console.RESET)
else
("", "", "")
case class Elem(dep: Dependency, excluded: Boolean) {
lazy val reconciledVersion = resolution.reconciledVersions
@ -65,7 +80,7 @@ object Print {
if (excluded)
resolution.reconciledVersions.get(dep.module) match {
case None =>
s"${Console.YELLOW}(excluded)${Console.RESET} ${dep.module}:${dep.version}"
s"$yellow(excluded)$reset ${dep.module}:${dep.version}"
case Some(version) =>
val versionMsg =
if (version == dep.version)
@ -74,7 +89,7 @@ object Print {
s"version $version"
s"${dep.module}:${dep.version} " +
s"${Console.RED}(excluded, $versionMsg present anyway)${Console.RESET}"
s"$red(excluded, $versionMsg present anyway)$reset"
}
else {
val versionStr =
@ -83,9 +98,10 @@ object Print {
else {
val assumeCompatibleVersions = compatibleVersions(dep.version, reconciledVersion)
(if (assumeCompatibleVersions) Console.YELLOW else Console.RED) +
(if (assumeCompatibleVersions) yellow else red) +
s"${dep.version} -> $reconciledVersion" +
Console.RESET
(if (assumeCompatibleVersions || colors) "" else " (possible incompatibility)") +
reset
}
s"${dep.module}:$versionStr"
@ -138,16 +154,16 @@ object Print {
) {
lazy val repr: String =
if (excluding)
s"${Console.YELLOW}(excluded by)${Console.RESET} $module:$version"
s"$yellow(excluded by)$reset $module:$version"
else if (wantVersion == gotVersion)
s"$module:$version"
else {
val assumeCompatibleVersions = compatibleVersions(wantVersion, gotVersion)
s"$module:$version " +
(if (assumeCompatibleVersions) Console.YELLOW else Console.RED) +
(if (assumeCompatibleVersions) yellow else red) +
s"(wants $dependsOn:$wantVersion, got $gotVersion)" +
Console.RESET
reset
}
}

71
plugin/src/main/scala-2.10/coursier/FallbackDependenciesRepository.scala
View File

@ -1,9 +1,68 @@
package coursier
import java.net.URL
import java.io.{ File, FileNotFoundException, IOException }
import java.net.{ HttpURLConnection, URL, URLConnection }
import scalaz.{ EitherT, Monad }
object FallbackDependenciesRepository {
def exists(url: URL): Boolean = {
// Sometimes HEAD attempts fail even though standard GETs are fine.
// E.g. https://github.com/NetLogo/NetLogo/releases/download/5.3.1/NetLogo.jar
// returning 403s. Hence the second attempt below.
val firstAttemptOpt = url.getProtocol match {
case "file" =>
Some(new File(url.getPath).exists()) // FIXME Escaping / de-escaping needed here?
case "http" | "https" =>
// HEAD request attempt, adapted from http://stackoverflow.com/questions/22541629/android-how-can-i-make-an-http-head-request/22545275#22545275
var conn: HttpURLConnection = null
try {
conn = url
.openConnection()
.asInstanceOf[HttpURLConnection]
conn.setRequestMethod("HEAD")
conn.getInputStream.close()
Some(true)
} catch {
case _: FileNotFoundException =>
Some(false)
case _: IOException => // error other than not found
None
} finally {
if (conn != null)
conn.disconnect()
}
case _ =>
None
}
firstAttemptOpt.getOrElse {
var conn: URLConnection = null
try {
conn = url.openConnection()
// NOT setting request type to HEAD here.
conn.getInputStream.close()
true
} catch {
case _: IOException =>
false
} finally {
conn match {
case conn0: HttpURLConnection => conn0.disconnect()
case _ =>
}
}
}
}
}
case class FallbackDependenciesRepository(
fallbacks: Map[(Module, String), (URL, Boolean)]
) extends Repository {
@ -44,12 +103,10 @@ case class FallbackDependenciesRepository(
else {
val (dirUrlStr, fileName) = urlStr.splitAt(idx + 1)
fetch(Artifact(dirUrlStr, Map.empty, Map.empty, Attributes("", ""), changing = true, None)).flatMap { listing =>
val files = coursier.core.compatibility.listWebPageFiles(dirUrlStr, listing)
if (files.contains(fileName)) {
// Not sure F.point will make that run like Task.apply would have
// if F = Task
EitherT.right(F.point(FallbackDependenciesRepository.exists(url))).flatMap { exists =>
if (exists) {
val proj = Project(
module,
version,

8
plugin/src/main/scala-2.10/coursier/Tasks.scala
View File

@ -971,7 +971,13 @@ object Tasks {
// use sbt logging?
println(
projectName + "\n" +
Print.dependencyTree(dependencies0, subRes, printExclusions = true, inverse)
Print.dependencyTree(
dependencies0,
subRes,
printExclusions = true,
inverse,
colors = !sys.props.get("sbt.log.noformat").toSeq.contains("true")
)
)
}

10
plugin/src/sbt-test/sbt-coursier/from-no-head/build.sbt Normal file
View File

@ -0,0 +1,10 @@
scalaVersion := "2.11.8"
coursierCachePolicies := {
if (sys.props("os.name").startsWith("Windows"))
coursierCachePolicies.value
else
Seq(coursier.CachePolicy.ForceDownload)
}
libraryDependencies += "ccl.northwestern.edu" % "netlogo" % "5.3.1" % "provided" from s"https://github.com/NetLogo/NetLogo/releases/download/5.3.1/NetLogo.jar"

11
plugin/src/sbt-test/sbt-coursier/from-no-head/project/plugins.sbt Normal file
View File

@ -0,0 +1,11 @@
{
val pluginVersion = sys.props.getOrElse(
"plugin.version",
throw new RuntimeException(
"""|The system property 'plugin.version' is not defined.
|Specify this property using the scriptedLaunchOpts -D.""".stripMargin
)
)
addSbtPlugin("io.get-coursier" % "sbt-coursier" % pluginVersion)
}

11
plugin/src/sbt-test/sbt-coursier/from-no-head/src/main/scala/Main.scala Normal file
View File

@ -0,0 +1,11 @@
import java.io.File
import java.nio.file.Files
object Main extends App {
// Not using directly the NetLogo 5.x lib, which seems to depend on Scala 2.9
// Can't find a way to check that NetLogo.jar is in the classpath
// These don't work, even with fork := true:
// assert(Thread.currentThread.getContextClassLoader.getResource("org/nlogo/nvm/Task.class") != null)
// Thread.currentThread.getContextClassLoader.getResource("org/nlogo/nvm/Task.class")
Files.write(new File("output").toPath, "OK".getBytes("UTF-8"))
}

3
plugin/src/sbt-test/sbt-coursier/from-no-head/test Normal file
View File

@ -0,0 +1,3 @@
$ delete output
> run
$ exists output

2
tests/jvm/src/it/scala/coursier/test/IvyLocalTests.scala
View File

@ -29,7 +29,7 @@ object IvyLocalTests extends TestSuite {
extraRepo = extraRepo
))
val artifacts = res.artifacts.map(_.url)
val artifacts = res.dependencyArtifacts.filter(_._2.`type` == "jar").map(_._2.url)
val anyJavadoc = artifacts.exists(_.contains("-javadoc"))
val anySources = artifacts.exists(_.contains("-sources"))

42
tests/jvm/src/test/resources/test-repo/http/ivy.abc.com/com.example/a_2.11/0.1.0-SNAPSHOT/ivys/ivy.xml Normal file
View File

@ -0,0 +1,42 @@
<?xml version="1.0" encoding="UTF-8"?>
<ivy-module version="2.0" xmlns:e="http://ant.apache.org/ivy/extra">
<info organisation="com.example" module="a_2.11" revision="0.1.0-SNAPSHOT" status="integration" publication="20161030134939">
<description/>
</info>
<configurations>
<conf name="pom" visibility="public" description=""/>
<conf extends="runtime" name="test" visibility="public" description=""/>
<conf name="provided" visibility="public" description=""/>
<conf name="docs" visibility="public" description=""/>
<conf name="optional" visibility="public" description=""/>
<conf name="compile" visibility="public" description=""/>
<conf name="sources" visibility="public" description=""/>
<conf extends="compile" name="runtime" visibility="public" description=""/>
</configurations>
<publications>
<artifact name="a_2.11" type="jar" ext="jar" conf="compile"/>
<artifact name="a_2.11" type="pom" ext="pom" conf="pom"/>
<artifact e:classifier="javadoc" name="a_2.11" type="doc" ext="jar" conf="compile">
</artifact>
<artifact e:classifier="sources" name="a_2.11" type="src" ext="jar" conf="compile">
</artifact>
<artifact e:classifier="tests-javadoc" name="a_2.11" type="doc" ext="jar" conf="test">
</artifact>
<artifact e:classifier="tests-sources" name="a_2.11" type="src" ext="jar" conf="test">
</artifact>
<artifact e:classifier="tests" name="a_2.11" type="jar" ext="jar" conf="test"/>
</publications>
<dependencies>
<dependency org="org.scala-lang" name="scala-library" rev="2.11.8" conf="compile->default(compile)">
</dependency>
</dependencies>
</ivy-module>

1
tests/jvm/src/test/resources/test-repo/http/ivy.abc.com/com.example/a_2.11/0.1.0-SNAPSHOT/ivys/ivy.xml.md5 Normal file
View File

@ -0,0 +1 @@
6cc6e8ae02e9c9ba222c05aa5b0ea838

1
tests/jvm/src/test/resources/test-repo/http/ivy.abc.com/com.example/a_2.11/0.1.0-SNAPSHOT/ivys/ivy.xml.sha1 Normal file
View File

@ -0,0 +1 @@
8b338542251660a1e91c560b77c53dd5870d3ff5

50
tests/jvm/src/test/scala/coursier/test/IvyTests.scala
View File

@ -1,6 +1,6 @@
package coursier.test
import coursier.Module
import coursier.{ Attributes, Dependency, Module }
import coursier.ivy.IvyRepository
import utest._
@ -46,10 +46,56 @@ object IvyTests extends TestSuite {
extraRepo = Some(sbtRepo)
)
* - CentralTests.withArtifact(mod, ver, extraRepo = Some(sbtRepo)) { artifact =>
* - CentralTests.withArtifact(mod, ver, "jar", extraRepo = Some(sbtRepo)) { artifact =>
assert(artifact.url == expectedArtifactUrl)
}
}
'testArtifacts - {
val dep = Dependency(
Module("com.example", "a_2.11"),
"0.1.0-SNAPSHOT",
transitive = false,
attributes = Attributes()
)
val repoBase = getClass.getResource("/test-repo/http/ivy.abc.com").toString.stripSuffix("/") + "/"
val repo = IvyRepository.fromPattern(
repoBase +: coursier.ivy.Pattern.default,
dropInfoAttributes = true
)
val mainJarUrl = repoBase + "com.example/a_2.11/0.1.0-SNAPSHOT/jars/a_2.11.jar"
val testJarUrl = repoBase + "com.example/a_2.11/0.1.0-SNAPSHOT/jars/a_2.11-tests.jar"
* - CentralTests.withArtifacts(
dep = dep,
artifactType = "jar",
extraRepo = Some(repo)
) {
case Seq(artifact) =>
assert(artifact.url == mainJarUrl)
case other =>
throw new Exception(s"Unexpected number of artifacts\n${other.mkString("\n")}")
}
* - CentralTests.withArtifacts(
dep = dep.copy(configuration = "test"),
artifactType = "jar",
extraRepo = Some(repo)
) {
case Seq(artifact1, artifact2) =>
val urls = Set(
artifact1.url,
artifact2.url
)
assert(urls == Set(mainJarUrl, testJarUrl))
case other =>
throw new Exception(s"Unexpected number of artifacts\n${other.mkString("\n")}")
}
}
}
}

54
tests/shared/src/test/scala/coursier/test/CentralTests.scala
View File

@ -100,14 +100,12 @@ object CentralTests extends TestSuite {
def withArtifact[T](
module: Module,
version: String,
artifactType: String,
extraRepo: Option[Repository] = None
)(
f: Artifact => T
): Future[T] = async {
val dep = Dependency(module, version, transitive = false, attributes = Attributes())
val res = await(resolve(Set(dep), extraRepo = extraRepo))
res.artifacts match {
): Future[T] =
withArtifacts(module, version, artifactType, extraRepo) {
case Seq(artifact) =>
f(artifact)
case other =>
@ -116,10 +114,41 @@ object CentralTests extends TestSuite {
"Artifacts:\n" + other.map(" " + _).mkString("\n")
)
}
def withArtifacts[T](
module: Module,
version: String,
artifactType: String,
extraRepo: Option[Repository] = None
)(
f: Seq[Artifact] => T
): Future[T] = {
val dep = Dependency(module, version, transitive = false, attributes = Attributes())
withArtifacts(dep, artifactType, extraRepo)(f)
}
def ensureArtifactHasExtension(module: Module, version: String, extension: String): Future[Unit] =
withArtifact(module, version) { artifact =>
def withArtifacts[T](
dep: Dependency,
artifactType: String,
extraRepo: Option[Repository]
)(
f: Seq[Artifact] => T
): Future[T] = async {
val res = await(resolve(Set(dep), extraRepo = extraRepo))
assert(res.errors.isEmpty)
assert(res.conflicts.isEmpty)
assert(res.isDone)
val artifacts = res.dependencyArtifacts.map(_._2).filter { a =>
a.`type` == artifactType
}
f(artifacts)
}
def ensureHasArtifactWithExtension(module: Module, version: String, artifactType: String, extension: String): Future[Unit] =
withArtifact(module, version, artifactType) { artifact =>
assert(artifact.url.endsWith("." + extension))
}
@ -275,7 +304,7 @@ object CentralTests extends TestSuite {
* - resolutionCheck(mod, version)
* - withArtifact(mod, version) { artifact =>
* - withArtifact(mod, version, "jar") { artifact =>
assert(artifact.url == expectedArtifactUrl)
}
}
@ -301,27 +330,30 @@ object CentralTests extends TestSuite {
'packaging - {
'aar - {
// random aar-based module found on Central
ensureArtifactHasExtension(
ensureHasArtifactWithExtension(
Module("com.yandex.android", "speechkit"),
"2.5.0",
"aar",
"aar"
)
}
'bundle - {
// has packaging bundle - ensuring coursier gives its artifact the .jar extension
ensureArtifactHasExtension(
ensureHasArtifactWithExtension(
Module("com.google.guava", "guava"),
"17.0",
"bundle",
"jar"
)
}
'mavenPlugin - {
// has packaging maven-plugin - ensuring coursier gives its artifact the .jar extension
ensureArtifactHasExtension(
ensureHasArtifactWithExtension(
Module("org.bytedeco", "javacpp"),
"1.1",
"maven-plugin",
"jar"
)
}