luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #606 from coursier/develop 

Various things
This commit is contained in:
Alexandre Archambault 2017-07-17 18:04:20 +02:00 committed by GitHub
parent 744e6186d6 19493a512a
commit a41c071943
17 changed files with 190 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
README.md
View File

@ -153,13 +153,13 @@ val fetch = Fetch.from(repositories, Cache.fetch())
Then run the resolution per-se,
```scala
val resolution = start.process.run(fetch).run
val resolution = start.process.run(fetch).unsafePerformSync
```
That will fetch and use metadata.
Check for errors in
```scala
val errors: Seq[(Dependency, Seq[String])] = resolution.errors
val errors: Seq[((Module, String), Seq[String])] = resolution.metadataErrors
```
These would mean that the resolution wasn't able to get metadata about some dependencies.
@ -171,7 +171,7 @@ import scalaz.concurrent.Task
val localArtifacts: Seq[FileError \/ File] = Task.gatherUnordered(
resolution.artifacts.map(Cache.file(_).run)
).run
).unsafePerformSync
```
@ -503,7 +503,7 @@ resolution is particularly complex, in which case `maxIterations` could be incre
Let's run the whole resolution,
```scala
val resolution = start.process.run(fetch).run
val resolution = start.process.run(fetch).unsafePerformSync
```
To get additional feedback during the resolution, we can give the `Cache.default` method above
@ -514,7 +514,7 @@ you can supply your own thread pool to `Cache.default`.
Now that the resolution is done, we can check for errors in
```scala
val errors: Seq[(Dependency, Seq[String])] = resolution.errors
val errors: Seq[((Module, String), Seq[String])] = resolution.metadataErrors
```
These would mean that the resolution wasn't able to get metadata about some dependencies.
@ -532,7 +532,7 @@ import scalaz.concurrent.Task
val localArtifacts: Seq[FileError \/ File] = Task.gatherUnordered(
resolution.artifacts.map(Cache.file(_).run)
).run
).unsafePerformSync
```
We're using the `Cache.file` method, that can also be given a `Logger` (for more feedback) and a custom thread pool.

1
appveyor.yml
View File

@ -33,6 +33,7 @@ cache:
- '%USERPROFILE%\.ivy2\cache'
- '%USERPROFILE%\.m2'
- '%USERPROFILE%\.sbt'
- C:\sbt
- '%USERPROFILE%\.coursier'
branches:
only:

1
core/shared/src/main/scala/coursier/core/Resolution.scala
View File

@ -1059,6 +1059,7 @@ final case class Resolution(
.toSeq
artifact <- source
.artifacts(dep, proj, overrideClassifiers)
if optional || !artifact.isOptional
} yield dep -> artifact
def dependencyArtifacts: Seq[(Dependency, Artifact)] =

12
doc/README.md
View File

@ -180,13 +180,13 @@ val fetch = Fetch.from(repositories, Cache.fetch())
Then run the resolution per-se,
```tut:silent
val resolution = start.process.run(fetch).run
val resolution = start.process.run(fetch).unsafePerformSync
```
That will fetch and use metadata.
Check for errors in
```tut:silent
val errors: Seq[(Dependency, Seq[String])] = resolution.errors
val errors: Seq[((Module, String), Seq[String])] = resolution.metadataErrors
```
These would mean that the resolution wasn't able to get metadata about some dependencies.
@ -198,7 +198,7 @@ import scalaz.concurrent.Task
val localArtifacts: Seq[FileError \/ File] = Task.gatherUnordered(
resolution.artifacts.map(Cache.file(_).run)
).run
).unsafePerformSync
```
@ -530,7 +530,7 @@ resolution is particularly complex, in which case `maxIterations` could be incre
Let's run the whole resolution,
```tut:silent
val resolution = start.process.run(fetch).run
val resolution = start.process.run(fetch).unsafePerformSync
```
To get additional feedback during the resolution, we can give the `Cache.default` method above
@ -541,7 +541,7 @@ you can supply your own thread pool to `Cache.default`.
Now that the resolution is done, we can check for errors in
```tut:silent
val errors: Seq[(Dependency, Seq[String])] = resolution.errors
val errors: Seq[((Module, String), Seq[String])] = resolution.metadataErrors
```
These would mean that the resolution wasn't able to get metadata about some dependencies.
@ -559,7 +559,7 @@ import scalaz.concurrent.Task
val localArtifacts: Seq[FileError \/ File] = Task.gatherUnordered(
resolution.artifacts.map(Cache.file(_).run)
).run
).unsafePerformSync
```
We're using the `Cache.file` method, that can also be given a `Logger` (for more feedback) and a custom thread pool.

115
sbt-coursier/src/main/scala/coursier/FallbackDependenciesRepository.scala
View File

@ -6,6 +6,8 @@ import java.net.{ HttpURLConnection, URL, URLConnection }
import scala.language.higherKinds
import scalaz.{ EitherT, Monad }
import scalaz.syntax.monad._
import scalaz.Scalaz.ToEitherOps
object FallbackDependenciesRepository {
@ -15,12 +17,12 @@ object FallbackDependenciesRepository {
// E.g. https://github.com/NetLogo/NetLogo/releases/download/5.3.1/NetLogo.jar
// returning 403s. Hence the second attempt below.
val firstAttemptOpt = url.getProtocol match {
case "file" =>
Some(new File(url.getPath).exists()) // FIXME Escaping / de-escaping needed here?
val protocolSpecificAttemptOpt = {
case "http" | "https" =>
def ifFile: Boolean =
new File(url.getPath).exists() // FIXME Escaping / de-escaping needed here?
def ifHttp: Option[Boolean] = {
// HEAD request attempt, adapted from http://stackoverflow.com/questions/22541629/android-how-can-i-make-an-http-head-request/22545275#22545275
var conn: HttpURLConnection = null
@ -31,34 +33,43 @@ object FallbackDependenciesRepository {
conn.setRequestMethod("HEAD")
conn.getInputStream.close()
Some(true)
} catch {
case _: FileNotFoundException =>
Some(false)
case _: IOException => // error other than not found
None
} finally {
}
catch {
case _: FileNotFoundException => Some(false)
case _: IOException => None // error other than not found
}
finally {
if (conn != null)
coursier.Cache.closeConn(conn)
}
case _ =>
None
}
url.getProtocol match {
case "file" => Some(ifFile)
case "http" | "https" => ifHttp
case _ => None
}
}
firstAttemptOpt.getOrElse {
def genericAttempt: Boolean = {
var conn: URLConnection = null
try {
conn = url.openConnection()
// NOT setting request type to HEAD here.
conn.getInputStream.close()
true
} catch {
case _: IOException =>
false
} finally {
}
catch {
case _: IOException => false
}
finally {
if (conn != null)
coursier.Cache.closeConn(conn)
}
}
protocolSpecificAttemptOpt
.getOrElse(genericAttempt)
}
}
@ -67,22 +78,23 @@ final case class FallbackDependenciesRepository(
fallbacks: Map[(Module, String), (URL, Boolean)]
) extends Repository {
private val source: Artifact.Source = new Artifact.Source {
def artifacts(
dependency: Dependency,
project: Project,
overrideClassifiers: Option[Seq[String]]
) =
fallbacks.get(dependency.moduleVersion) match {
case None => Nil
case Some((url, changing)) =>
val url0 = url.toString
val ext = url0.substring(url0.lastIndexOf('.') + 1)
Seq(
Artifact(url0, Map.empty, Map.empty, Attributes(ext, ""), changing, None)
)
}
}
private val source: Artifact.Source =
new Artifact.Source {
def artifacts(
dependency: Dependency,
project: Project,
overrideClassifiers: Option[Seq[String]]
) =
fallbacks
.get(dependency.moduleVersion)
.toSeq
.map {
case (url, changing) =>
val url0 = url.toString
val ext = url0.substring(url0.lastIndexOf('.') + 1)
Artifact(url0, Map.empty, Map.empty, Attributes(ext, ""), changing, None)
}
}
def find[F[_]](
module: Module,
@ -90,25 +102,22 @@ final case class FallbackDependenciesRepository(
fetch: Fetch.Content[F]
)(implicit
F: Monad[F]
): EitherT[F, String, (Artifact.Source, Project)] =
fallbacks.get((module, version)) match {
case None =>
EitherT.left(F.point("No fallback URL found"))
): EitherT[F, String, (Artifact.Source, Project)] = {
case Some((url, _)) =>
val res = fallbacks
.get((module, version))
.fold("No fallback URL found".left[(Artifact.Source, Project)]) {
case (url, _) =>
val urlStr = url.toExternalForm
val idx = urlStr.lastIndexOf('/')
val urlStr = url.toExternalForm
val idx = urlStr.lastIndexOf('/')
if (idx < 0 || urlStr.endsWith("/"))
EitherT.left(F.point(s"$url doesn't point to a file"))
else {
val (dirUrlStr, fileName) = urlStr.splitAt(idx + 1)
if (idx < 0 || urlStr.endsWith("/"))
s"$url doesn't point to a file".left
else {
val (dirUrlStr, fileName) = urlStr.splitAt(idx + 1)
// Not sure F.point will make that run like Task.apply would have
// if F = Task
EitherT.right(F.point(FallbackDependenciesRepository.exists(url))).flatMap { exists =>
if (exists) {
if (FallbackDependenciesRepository.exists(url)) {
val proj = Project(
module,
version,
@ -126,10 +135,12 @@ final case class FallbackDependenciesRepository(
Info.empty
)
EitherT.right(F.point((source, proj)))
(source, proj).right
} else
EitherT.left(F.point(s"$fileName not found under $dirUrlStr"))
s"$fileName not found under $dirUrlStr".left
}
}
}
}
EitherT(res.point)
}
}

23
sbt-coursier/src/main/scala/coursier/InterProjectRepository.scala
View File

@ -2,28 +2,27 @@ package coursier
import scala.language.higherKinds
import scalaz.{ -\/, \/-, Monad, EitherT }
import scalaz.{EitherT, Monad}
import scalaz.syntax.monad._
import scalaz.syntax.std.option._
final case class InterProjectRepository(projects: Seq[Project]) extends Repository {
private val map = projects
.map { proj => proj.moduleVersion -> proj }
.map(proj => proj.moduleVersion -> proj)
.toMap
def find[F[_]](
def find[F[_]: Monad](
module: Module,
version: String,
fetch: Fetch.Content[F]
)(implicit
F: Monad[F]
): EitherT[F, String, (Artifact.Source, Project)] = {
val res = map.get((module, version)) match {
case Some(proj) =>
\/-((Artifact.Source.empty, proj))
case None =>
-\/("Not found")
}
EitherT(F.point(res))
val res = map
.get((module, version))
.toRightDisjunction("Not found")
.map((Artifact.Source.empty, _))
EitherT(res.point)
}
}

54
sbt-coursier/src/main/scala/coursier/ResolutionError.scala
View File

@ -3,7 +3,8 @@ package coursier
import scala.collection.mutable.ArrayBuffer
sealed abstract class ResolutionError extends Product with Serializable {
def cause: Option[Throwable] = this match {
final def cause: Option[Throwable] = this match {
case ResolutionError.MaximumIterationsReached => None
case ResolutionError.UnknownException(ex) => Some(ex)
case ResolutionError.UnknownDownloadException(ex) => Some(ex)
@ -12,10 +13,10 @@ sealed abstract class ResolutionError extends Product with Serializable {
case _: ResolutionError.DownloadErrors => None
}
def message: String = this match {
final def message: String = this match {
case ResolutionError.MaximumIterationsReached =>
"Maximum number of iteration of dependency resolution reached"
case ResolutionError.UnknownException(ex) =>
case ResolutionError.UnknownException(_) =>
"Exception during resolution"
case ResolutionError.UnknownDownloadException(ex) =>
"Error while downloading / verifying artifacts"
@ -29,10 +30,10 @@ sealed abstract class ResolutionError extends Product with Serializable {
err.description(verbose = true)
}
def exception(): ResolutionException =
final def exception(): ResolutionException =
new ResolutionException(this)
def throwException(): Nothing =
final def throwException(): Nothing =
throw exception()
}
@ -49,40 +50,44 @@ object ResolutionError {
def grouped(errs: Seq[String]) =
errs
.map { s =>
val idx = s.indexOf(": ")
if (idx >= 0)
(s.take(idx), s.drop(idx + ": ".length))
else
("", s)
s.split(": ", 2) match {
case Array(k, v) => (k, v)
case _ => ("", s)
}
}
.groupBy(_._1)
.mapValues(_.map(_._2))
.toVector
.sortBy(_._1)
val lines = new ArrayBuffer[String]
lines += s"Encountered ${errors.length} error(s) in dependency resolution:"
val lines = new ArrayBuffer[String]
def print(s: String, indentLevel: Int = 0) =
lines += " " * indentLevel * 2 + s
def result = lines.mkString("\n")
print(s"Encountered ${errors.length} error(s) in dependency resolution:")
for (((mod, ver), errs) <- errors) {
lines += s" $mod:$ver:"
print(s"$mod:$ver:", 1)
for ((type0, errs0) <- grouped(errs))
if (type0.isEmpty)
for (err <- errs0)
lines += s" $err"
print(err, 2)
else
errs0 match {
case Seq(err) =>
lines += s" $type0: $err"
print(s"$type0: $err", 2)
case _ =>
lines += s" $type0:"
print(s"$type0:", 2)
for (err <- errs0)
lines += s" $err"
print(err, 3)
}
}
lines.mkString("\n")
result
}
}
@ -96,16 +101,21 @@ object ResolutionError {
.toVector
.sortBy(_._1)
val b = new ArrayBuffer[String]
val lines = new ArrayBuffer[String]
def print(s: String, indentLevel: Int = 0) =
lines += " " * indentLevel * 2 + s
def result = lines.mkString("\n")
for ((type0, errors) <- groupedArtifactErrors) {
b += s"${errors.size} $type0"
print(s"${errors.size} $type0")
if (verbose)
for (err <- errors)
b += " " + err
print(err, 1)
}
b.mkString("\n")
result
}
}
}

14
sbt-coursier/src/main/scala/coursier/SbtBootJars.scala
View File

@ -8,11 +8,13 @@ object SbtBootJars {
scalaVersion: String,
jars: Seq[File]
): Map[(Module, String), File] =
jars.collect {
case jar if jar.getName.endsWith(".jar") =>
val name = jar.getName.stripSuffix(".jar")
val mod = Module(scalaOrg, name)
jars
.collect {
case jar if jar.getName.endsWith(".jar") =>
val name = jar.getName.stripSuffix(".jar")
val mod = Module(scalaOrg, name)
(mod, scalaVersion) -> jar
}.toMap
(mod, scalaVersion) -> jar
}
.toMap
}

11
sbt-coursier/src/main/scala/coursier/Settings.scala
View File

@ -6,7 +6,7 @@ import scala.util.{Failure, Success, Try}
object Settings {
private lazy val baseDefaultVerbosityLevel =
private val baseDefaultVerbosityLevel =
if (System.console() == null) // non interactive mode
0
else
@ -15,17 +15,18 @@ object Settings {
def defaultVerbosityLevel(logger: Logger): Int = {
def fromOption(value: Option[String], description: String): Option[Int] =
value.filter(_.nonEmpty).flatMap {
str =>
value
.filter(_.nonEmpty)
.flatMap { str =>
Try(str.toInt) match {
case Success(level) => Some(level)
case Failure(ex) =>
case Failure(_) =>
logger.warn(
s"unrecognized $description value (should be an integer), ignoring it."
)
None
}
}
}
val fromEnv = fromOption(
sys.env.get("COURSIER_VERBOSITY"),

2
sbt-coursier/src/main/scala/coursier/ToSbt.scala
View File

@ -122,7 +122,7 @@ object ToSbt {
) = {
val depArtifacts1 =
classifiersOpt match {
case None => res.dependencyArtifacts
case None => res.dependencyArtifacts(withOptional = true)
case Some(cl) => res.dependencyClassifiersArtifacts(cl)
}

3
sbt-shading/src/main/scala/coursier/Shading.scala
View File

@ -91,7 +91,8 @@ object Shading {
)
}
val dependencyArtifacts = res.dependencyArtifacts
val dependencyArtifacts = res
.dependencyArtifacts(withOptional = true)
.filter { case (_, a) => classpathTypes(a.`type`) }
.groupBy(_._1)
.mapValues(_.map(_._2))

2
tests/jvm/src/it/scala/coursier/test/IvyLocalTests.scala
View File

@ -46,7 +46,7 @@ object IvyLocalTests extends TestSuite {
extraRepos = extraRepos
))
val artifacts = res.dependencyArtifacts.filter(_._2.`type` == "jar").map(_._2.url)
val artifacts = res.dependencyArtifacts(withOptional = true).filter(_._2.`type` == "jar").map(_._2.url)
val anyJavadoc = artifacts.exists(_.contains("-javadoc"))
val anySources = artifacts.exists(_.contains("-sources"))

6
tests/jvm/src/test/scala/coursier/test/IvyTests.scala
View File

@ -74,7 +74,8 @@ object IvyTests extends TestSuite {
dep = dep,
artifactType = "jar",
extraRepos = Seq(repo),
classifierOpt = None
classifierOpt = None,
optional = true
) {
case Seq(artifact) =>
assert(artifact.url == mainJarUrl)
@ -86,7 +87,8 @@ object IvyTests extends TestSuite {
dep = dep.copy(configuration = "test"),
artifactType = "jar",
extraRepos = Seq(repo),
classifierOpt = None
classifierOpt = None,
optional = true
) {
case Seq(artifact1, artifact2) =>
val urls = Set(

6
tests/jvm/src/test/scala/coursier/test/MavenTests.scala
View File

@ -29,7 +29,8 @@ object MavenTests extends TestSuite {
dep = dep,
artifactType = "jar",
extraRepos = Seq(repo),
classifierOpt = None
classifierOpt = None,
optional = true
) {
case Seq(artifact) =>
assert(artifact.url == mainJarUrl)
@ -41,7 +42,8 @@ object MavenTests extends TestSuite {
dep = dep,
artifactType = "src",
extraRepos = Seq(repo),
classifierOpt = Some("sources")
classifierOpt = Some("sources"),
optional = true
) {
case Seq(artifact) =>
assert(artifact.url == sourcesJarUrl)

2
tests/metadata

@ -1 +1 @@
Subproject commit 9a00b31af466454fbf2604e871c273ba9c1c9938
Subproject commit ac3871c925e035157203d6d799a9a2c7c2578d67

8
tests/shared/src/test/resources/resolutions/io.monix/monix_2.12/2.3.0 Normal file
View File

@ -0,0 +1,8 @@
io.monix:monix-eval_2.12:2.3.0:compile
io.monix:monix-execution_2.12:2.3.0:compile
io.monix:monix-reactive_2.12:2.3.0:compile
io.monix:monix-types_2.12:2.3.0:compile
io.monix:monix_2.12:2.3.0:compile
org.jctools:jctools-core:2.0.1:compile
org.reactivestreams:reactive-streams:1.0.0:compile
org.scala-lang:scala-library:2.12.2:compile

43
tests/shared/src/test/scala/coursier/test/CentralTests.scala
View File

@ -148,29 +148,32 @@ abstract class CentralTests extends TestSuite {
attributes: Attributes = Attributes(),
extraRepos: Seq[Repository] = Nil,
classifierOpt: Option[String] = None,
transitive: Boolean = false
transitive: Boolean = false,
optional: Boolean = true
)(
f: Seq[Artifact] => T
): Future[T] = {
val dep = Dependency(module, version, transitive = transitive, attributes = attributes)
withArtifacts(dep, artifactType, extraRepos, classifierOpt)(f)
withArtifacts(dep, artifactType, extraRepos, classifierOpt, optional)(f)
}
def withArtifacts[T](
dep: Dependency,
artifactType: String,
extraRepos: Seq[Repository],
classifierOpt: Option[String]
classifierOpt: Option[String],
optional: Boolean
)(
f: Seq[Artifact] => T
): Future[T] =
withArtifacts(Set(dep), artifactType, extraRepos, classifierOpt)(f)
withArtifacts(Set(dep), artifactType, extraRepos, classifierOpt, optional)(f)
def withArtifacts[T](
deps: Set[Dependency],
artifactType: String,
extraRepos: Seq[Repository],
classifierOpt: Option[String]
classifierOpt: Option[String],
optional: Boolean
)(
f: Seq[Artifact] => T
): Future[T] = async {
@ -181,7 +184,7 @@ abstract class CentralTests extends TestSuite {
assert(res.isDone)
val artifacts = classifierOpt
.fold(res.dependencyArtifacts)(c => res.dependencyClassifiersArtifacts(Seq(c)))
.fold(res.dependencyArtifacts(withOptional = optional))(c => res.dependencyClassifiersArtifacts(Seq(c)))
.map(_._2)
.filter {
if (artifactType == "*") _ => true
@ -431,7 +434,8 @@ abstract class CentralTests extends TestSuite {
),
"jar",
extraRepos = Nil,
classifierOpt = None
classifierOpt = None,
optional = true
) {
case Seq() =>
throw new Exception("Expected one JAR")
@ -552,7 +556,7 @@ abstract class CentralTests extends TestSuite {
assert(res.conflicts.isEmpty)
assert(res.isDone)
val dependencyArtifacts = res.dependencyArtifacts
val dependencyArtifacts = res.dependencyArtifacts(withOptional = true)
val zookeeperTestArtifacts = dependencyArtifacts.collect {
case (dep, artifact)
@ -778,6 +782,29 @@ abstract class CentralTests extends TestSuite {
* - resolutionCheck(mod, ver)
}
'optionalArtifacts - {
val mod = Module("io.monix", "monix_2.12")
val ver = "2.3.0"
val mainUrl = "https://repo1.maven.org/maven2/io/monix/monix_2.12/2.3.0/monix_2.12-2.3.0.jar"
* - resolutionCheck(mod, ver)
* - {
if (isActualCentral)
withArtifacts(mod, ver, "jar") { artifacts =>
val mainArtifactOpt = artifacts.find(_.url == mainUrl)
assert(mainArtifactOpt.nonEmpty)
assert(mainArtifactOpt.forall(_.isOptional))
}
}
* - withArtifacts(mod, ver, "jar", optional = false) { artifacts =>
val mainArtifactOpt = artifacts.find(_.url == mainUrl)
assert(mainArtifactOpt.isEmpty)
}
}
}
}