luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support for HTTP authentication

This commit is contained in:
Alexandre Archambault 2016-05-06 13:53:55 +02:00
parent 00484df435
commit f68ed5d42b
No known key found for this signature in database
GPG Key ID: 14640A6839C263A9
15 changed files with 411 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
appveyor.yml
View File

@ -15,10 +15,11 @@ install:
- cmd: SET SBT_OPTS=-XX:MaxPermSize=2g -Xmx4g
- cmd: SET COURSIER_NO_TERM=1
build_script:
- sbt ++2.11.8 clean compile coreJVM/publishLocal
- sbt ++2.11.8 clean compile coreJVM/publishLocal simple-web-server/publishLocal
- sbt ++2.10.6 clean compile
- sbt ++2.10.6 coreJVM/publishLocal cache/publishLocal # to make the scripted tests happy
test_script:
- ps: Start-Job { & java -jar -noverify C:\projects\coursier\coursier launch -r http://dl.bintray.com/scalaz/releases io.get-coursier:simple-web-server_2.11:1.0.0-SNAPSHOT -- -d /C:/projects/coursier/tests/jvm/src/test/resources/test-repo/http/abc.com -u user -P pass -r realm -v }
- sbt ++2.11.8 testsJVM/test # Would node be around for testsJS/test?
- sbt ++2.10.6 testsJVM/test plugin/scripted
cache:

71
cache/src/main/scala/coursier/Cache.scala vendored
View File

@ -8,7 +8,9 @@ import java.security.MessageDigest
import java.util.concurrent.{ ConcurrentHashMap, Executors, ExecutorService }
import java.util.regex.Pattern
import coursier.core.Authentication
import coursier.ivy.IvyRepository
import coursier.util.Base64.Encoder
import scala.annotation.tailrec
@ -18,6 +20,10 @@ import scalaz.concurrent.{ Task, Strategy }
import java.io.{ Serializable => _, _ }
trait AuthenticatedURLConnection extends URLConnection {
def authenticate(authentication: Authentication): Unit
}
object Cache {
// Check SHA-1 if available, else be fine with no checksum
@ -43,7 +49,7 @@ object Cache {
}
}
private def localFile(url: String, cache: File): File = {
private def localFile(url: String, cache: File, user: Option[String]): File = {
val path =
if (url.startsWith("file:///"))
url.stripPrefix("file://")
@ -62,7 +68,10 @@ object Cache {
else
throw new Exception(s"URL $url doesn't contain an absolute path")
new File(cache, escape(protocol + "/" + remaining0)) .toString
new File(
cache,
escape(protocol + "/" + user.fold("")(_ + "@") + remaining0.dropWhile(_ == '/'))
).toString
case _ =>
throw new Exception(s"No protocol found in URL $url")
@ -259,6 +268,17 @@ object Cache {
}
}
private val BasicRealm = (
"^" +
Pattern.quote("Basic realm=\"") +
"([^" + Pattern.quote("\"") + "]*)" +
Pattern.quote("\"") +
"$"
).r
private def basicAuthenticationEncode(user: String, password: String): String =
(user + ":" + password).getBytes("UTF-8").toBase64
/**
* Returns a `java.net.URL` for `s`, possibly using the custom protocol handlers found under the
* `coursier.cache.protocol` namespace.
@ -289,7 +309,7 @@ object Cache {
val referenceFileOpt = artifact
.extra
.get("metadata")
.map(a => localFile(a.url, cache))
.map(a => localFile(a.url, cache, a.authentication.map(_.user)))
def referenceFileExists: Boolean = referenceFileOpt.exists(_.exists())
@ -300,6 +320,20 @@ object Cache {
// (Maven 2 compatibility? - happens for snapshot versioning metadata,
// this is SO FSCKING CRAZY)
conn.setRequestProperty("User-Agent", "")
for (auth <- artifact.authentication)
conn match {
case authenticated: AuthenticatedURLConnection =>
authenticated.authenticate(auth)
case conn0: HttpURLConnection =>
conn0.setRequestProperty(
"Authorization",
"Basic " + basicAuthenticationEncode(auth.user, auth.password)
)
case _ =>
// FIXME Authentication is ignored
}
conn
}
@ -384,15 +418,28 @@ object Cache {
}
}
def is404(conn: URLConnection) =
def responseCode(conn: URLConnection): Option[Int] =
conn match {
case conn0: HttpURLConnection =>
conn0.getResponseCode == 404
Some(conn0.getResponseCode)
case _ =>
false
None
}
def remote(file: File, url: String): EitherT[Task, FileError, Unit] =
def realm(conn: URLConnection): Option[String] =
conn match {
case conn0: HttpURLConnection =>
Option(conn0.getHeaderField("WWW-Authenticate")).collect {
case BasicRealm(realm) => realm
}
case _ =>
None
}
def remote(
file: File,
url: String
): EitherT[Task, FileError, Unit] =
EitherT {
Task {
withLockFor(cache, file) {
@ -421,8 +468,10 @@ object Cache {
case _ => (false, conn0)
}
if (is404(conn))
if (responseCode(conn) == Some(404))
FileError.NotFound(url, permanent = Some(true)).left
else if (responseCode(conn) == Some(401))
FileError.Unauthorized(url, realm = realm(conn)).left
else {
for (len0 <- Option(conn.getContentLengthLong) if len0 >= 0L) {
val len = len0 + (if (partialDownload) alreadyDownloaded else 0L)
@ -534,7 +583,7 @@ object Cache {
val tasks =
for (url <- urls) yield {
val file = localFile(url, cache)
val file = localFile(url, cache, artifact.authentication.map(_.user))
val res =
if (url.startsWith("file:/")) {
@ -618,12 +667,12 @@ object Cache {
implicit val pool0 = pool
val localFile0 = localFile(artifact.url, cache)
val localFile0 = localFile(artifact.url, cache, artifact.authentication.map(_.user))
EitherT {
artifact.checksumUrls.get(sumType) match {
case Some(sumUrl) =>
val sumFile = localFile(sumUrl, cache)
val sumFile = localFile(sumUrl, cache, artifact.authentication.map(_.user))
Task {
val sumOpt = parseChecksum(

43
cache/src/main/scala/coursier/CacheParse.scala vendored
View File

@ -2,6 +2,7 @@ package coursier
import java.net.MalformedURLException
import coursier.core.Authentication
import coursier.ivy.IvyRepository
import coursier.util.Parse
@ -26,13 +27,49 @@ object CacheParse {
sys.error(s"Unrecognized repository: $r")
}
try {
Cache.url(url)
repo.success
val validatedUrl = try {
Cache.url(url).success
} catch {
case e: MalformedURLException =>
("Error parsing URL " + url + Option(e.getMessage).fold("")(" (" + _ + ")")).failure
}
validatedUrl.flatMap { url =>
Option(url.getUserInfo) match {
case None =>
repo.success
case Some(userInfo) =>
userInfo.split(":", 2) match {
case Array(user, password) =>
val baseUrl = new java.net.URL(
url.getProtocol,
url.getHost,
url.getPort,
url.getFile
).toString
val repo0 = repo match {
case m: MavenRepository =>
m.copy(
root = baseUrl,
authentication = Some(Authentication(user, password))
)
case i: IvyRepository =>
i.copy(
pattern = baseUrl,
authentication = Some(Authentication(user, password))
)
case r =>
sys.error(s"Unrecognized repository: $r")
}
repo0.success
case _ =>
s"No password found in user info of URL $url".failure
}
}
}
}
def repositories(l: Seq[String]): ValidationNel[String, Seq[Repository]] =

8
cache/src/main/scala/coursier/FileError.scala vendored
View File

@ -22,6 +22,14 @@ object FileError {
file
)
final case class Unauthorized(
file: String,
realm: Option[String]
) extends FileError(
"unauthorized",
file + realm.fold("")(" (" + _ + ")")
)
final case class ChecksumNotFound(
sumType: String,
file: String

106
cache/src/main/scala/coursier/util/Base64.scala vendored Normal file
View File

@ -0,0 +1,106 @@
package coursier.util
import scala.collection.mutable.ArrayBuilder
/**
* Base64 encoder
* @author Mark Lister
* This software is distributed under the 2-Clause BSD license. See the
* LICENSE file in the root of the repository.
*
* Copyright (c) 2014 - 2015 Mark Lister
*
* The repo for this Base64 encoder lives at https://github.com/marklister/base64
* Please send your issues, suggestions and pull requests there.
*/
object Base64 {
case class B64Scheme(encodeTable: Array[Char], strictPadding: Boolean = true,
postEncode: String => String = identity,
preDecode: String => String = identity) {
lazy val decodeTable = {
val b: Array[Int] = new Array[Int](256)
for (x <- encodeTable.zipWithIndex) {
b(x._1) = x._2.toInt
}
b
}
}
val base64 = new B64Scheme((('A' to 'Z') ++ ('a' to 'z') ++ ('0' to '9') ++ Seq('+', '/')).toArray)
val base64Url = new B64Scheme(base64.encodeTable.dropRight(2) ++ Seq('-', '_'), false,
_.replaceAllLiterally("=", "%3D"),
_.replaceAllLiterally("%3D", "="))
implicit class SeqEncoder(s: Seq[Byte]) {
def toBase64(implicit scheme: B64Scheme = base64): String = Encoder(s.toArray).toBase64
}
implicit class Encoder(b: Array[Byte]) {
val r = new StringBuilder((b.length + 3) * 4 / 3)
lazy val pad = (3 - b.length % 3) % 3
def toBase64(implicit scheme: B64Scheme = base64): String = {
def sixBits(x: Byte, y: Byte, z: Byte): Unit = {
val zz = (x & 0xff) << 16 | (y & 0xff) << 8 | (z & 0xff)
r += scheme.encodeTable(zz >> 18)
r += scheme.encodeTable(zz >> 12 & 0x3f)
r += scheme.encodeTable(zz >> 6 & 0x3f)
r += scheme.encodeTable(zz & 0x3f)
}
for (p <- 0 until b.length - 2 by 3) {
sixBits(b(p), b(p + 1), b(p + 2))
}
pad match {
case 0 =>
case 1 => sixBits(b(b.length - 2), b(b.length - 1), 0)
case 2 => sixBits(b(b.length - 1), 0, 0)
}
r.length = (r.length - pad)
r ++= "=" * pad
scheme.postEncode(r.toString())
}
}
implicit class Decoder(s: String) {
def toByteArray(implicit scheme: B64Scheme = base64): Array[Byte] = {
val pre = scheme.preDecode(s)
val cleanS = pre.replaceAll("=+$", "")
val pad = pre.length - cleanS.length
val computedPad = (4 - (cleanS.length % 4)) % 4
val r = new ArrayBuilder.ofByte
def threeBytes(a: Int, b: Int, c: Int, d: Int): Unit = {
val i = a << 18 | b << 12 | c << 6 | d
r += ((i >> 16).toByte)
r += ((i >> 8).toByte)
r += (i.toByte)
}
if (scheme.strictPadding) {
if (pad > 2) throw new java.lang.IllegalArgumentException("Invalid Base64 String: (excessive padding) " + s)
if (s.length % 4 != 0) throw new java.lang.IllegalArgumentException("Invalid Base64 String: (padding problem) " + s)
}
if (computedPad == 3) throw new java.lang.IllegalArgumentException("Invalid Base64 String: (string length) " + s)
try {
val s = (cleanS + "A" * computedPad)
for (x <- 0 until s.length - 1 by 4) {
val i = scheme.decodeTable(s.charAt(x)) << 18 |
scheme.decodeTable(s.charAt(x + 1)) << 12 |
scheme.decodeTable(s.charAt(x + 2)) << 6 |
scheme.decodeTable(s.charAt(x + 3))
r += ((i >> 16).toByte)
r += ((i >> 8).toByte)
r += (i.toByte)
}
} catch {
case e: NoSuchElementException => throw new java.lang.IllegalArgumentException("Invalid Base64 String: (invalid character)" + e.getMessage + s)
}
val res = r.result
res.slice(0, res.length - computedPad)
}
}
}

11
core/shared/src/main/scala/coursier/core/Definitions.scala
View File

@ -183,7 +183,8 @@ final case class Artifact(
checksumUrls: Map[String, String],
extra: Map[String, Artifact],
attributes: Attributes,
changing: Boolean
changing: Boolean,
authentication: Option[Authentication]
)
object Artifact {
@ -205,3 +206,11 @@ object Artifact {
}
}
}
case class Authentication(
user: String,
password: String
) {
override def toString: String =
s"Authentication($user, *******)"
}

3
core/shared/src/main/scala/coursier/core/Repository.scala
View File

@ -34,7 +34,8 @@ object Repository {
Map.empty,
Map.empty,
Attributes("asc", ""),
changing = underlying.changing
changing = underlying.changing,
authentication = underlying.authentication
)
.withDefaultChecksums
))

9
core/shared/src/main/scala/coursier/ivy/IvyRepository.scala
View File

@ -14,7 +14,8 @@ case class IvyRepository(
withSignatures: Boolean = true,
withArtifacts: Boolean = true,
// hack for SBT putting infos in properties
dropInfoAttributes: Boolean = false
dropInfoAttributes: Boolean = false,
authentication: Option[Authentication] = None
) extends Repository {
def metadataPattern: String = metadataPatternOpt.getOrElse(pattern)
@ -92,7 +93,8 @@ case class IvyRepository(
Map.empty,
Map.empty,
p.attributes,
changing = changing.getOrElse(project.version.contains("-SNAPSHOT")) // could be more reliable
changing = changing.getOrElse(project.version.contains("-SNAPSHOT")), // could be more reliable
authentication = authentication
)
if (withChecksums)
@ -127,7 +129,8 @@ case class IvyRepository(
Map.empty,
Map.empty,
Attributes("ivy", ""),
changing = changing.getOrElse(version.contains("-SNAPSHOT"))
changing = changing.getOrElse(version.contains("-SNAPSHOT")),
authentication = authentication
)
if (withChecksums)

14
core/shared/src/main/scala/coursier/maven/MavenRepository.scala
View File

@ -70,14 +70,15 @@ case class MavenRepository(
root: String,
changing: Option[Boolean] = None,
/** Hackish hack for sbt plugins mainly - what this does really sucks */
sbtAttrStub: Boolean = false
sbtAttrStub: Boolean = false,
authentication: Option[Authentication] = None
) extends Repository {
import Repository._
import MavenRepository._
val root0 = if (root.endsWith("/")) root else root + "/"
val source = MavenSource(root0, changing, sbtAttrStub)
val source = MavenSource(root0, changing, sbtAttrStub, authentication)
def projectArtifact(
module: Module,
@ -96,7 +97,8 @@ case class MavenRepository(
Map.empty,
Map.empty,
Attributes("pom", ""),
changing = changing.getOrElse(version.contains("-SNAPSHOT"))
changing = changing.getOrElse(version.contains("-SNAPSHOT")),
authentication = authentication
)
.withDefaultChecksums
.withDefaultSignature
@ -115,7 +117,8 @@ case class MavenRepository(
Map.empty,
Map.empty,
Attributes("pom", ""),
changing = true
changing = true,
authentication = authentication
)
.withDefaultChecksums
.withDefaultSignature
@ -140,7 +143,8 @@ case class MavenRepository(
Map.empty,
Map.empty,
Attributes("pom", ""),
changing = true
changing = true,
authentication = authentication
)
.withDefaultChecksums
.withDefaultSignature

12
core/shared/src/main/scala/coursier/maven/MavenSource.scala
View File

@ -6,7 +6,8 @@ case class MavenSource(
root: String,
changing: Option[Boolean] = None,
/** See doc on MavenRepository */
sbtAttrStub: Boolean
sbtAttrStub: Boolean,
authentication: Option[Authentication]
) extends Artifact.Source {
import Repository._
@ -21,7 +22,8 @@ case class MavenSource(
Map.empty,
Map.empty,
Attributes("jar", "src"), // Are these the right attributes?
changing = underlying.changing
changing = underlying.changing,
authentication = authentication
)
.withDefaultChecksums
.withDefaultSignature,
@ -30,7 +32,8 @@ case class MavenSource(
Map.empty,
Map.empty,
Attributes("jar", "javadoc"), // Same comment as above
changing = underlying.changing
changing = underlying.changing,
authentication = authentication
)
.withDefaultChecksums
.withDefaultSignature
@ -65,7 +68,8 @@ case class MavenSource(
Map.empty,
Map.empty,
publication.attributes,
changing = changing0
changing = changing0,
authentication = authentication
)
.withDefaultChecksums

2
plugin/src/main/scala-2.10/coursier/FallbackDependenciesRepository.scala
View File

@ -18,7 +18,7 @@ case class FallbackDependenciesRepository(
case None => Nil
case Some((url, changing)) =>
Seq(
Artifact(url.toString, Map.empty, Map.empty, Attributes("jar", ""), changing)
Artifact(url.toString, Map.empty, Map.empty, Attributes("jar", ""), changing, None)
)
}
}

11
project/travis.sh
View File

@ -28,7 +28,16 @@ function isMasterOrDevelop() {
}
# Required for ~/.ivy2/local repo tests
~/sbt coreJVM/publish-local
~/sbt coreJVM/publishLocal simple-web-server/publishLocal
# Required for HTTP authentication tests
./coursier launch \
io.get-coursier:simple-web-server_2.11:1.0.0-SNAPSHOT \
-r http://dl.bintray.com/scalaz/releases \
-- \
-d tests/jvm/src/test/resources/test-repo/http/abc.com \
-u user -P pass -r realm \
-v &
# TODO Add coverage once https://github.com/scoverage/sbt-scoverage/issues/111 is fixed

147
tests/jvm/src/test/scala/coursier/test/CacheFetchTests.scala Normal file
View File

@ -0,0 +1,147 @@
package coursier
package test
import java.io.File
import java.nio.file.Files
import coursier.cache.protocol.TestprotocolHandler
import coursier.core.Authentication
import utest._
import scala.util.Try
object CacheFetchTests extends TestSuite {
val tests = TestSuite {
def check(extraRepo: Repository): Unit = {
val tmpDir = Files.createTempDirectory("coursier-cache-fetch-tests").toFile
def cleanTmpDir() = {
def delete(f: File): Boolean =
if (f.isDirectory) {
val removedContent = f.listFiles().map(delete).forall(x => x)
val removedDir = f.delete()
removedContent && removedDir
} else
f.delete()
if (!delete(tmpDir))
Console.err.println(s"Warning: unable to remove temporary directory $tmpDir")
}
val res = try {
val fetch = Fetch.from(
Seq(
extraRepo,
MavenRepository("https://repo1.maven.org/maven2")
),
Cache.fetch(
tmpDir
)
)
val startRes = Resolution(
Set(
Dependency(
Module("com.github.alexarchambault", "coursier_2.11"), "1.0.0-M9-test"
)
)
)
startRes.process.run(fetch).run
} finally {
cleanTmpDir()
}
val errors = res.errors
assert(errors.isEmpty)
}
// using scala-test would allow to put the below comments in the test names...
* - {
// test that everything's fine with basic file protocol
val repoPath = new File(getClass.getResource("/test-repo/http/abc.com").getPath)
check(MavenRepository(repoPath.toURI.toString))
}
'customProtocol - {
* - {
// test the Cache.url method
val shouldFail = Try(Cache.url("notfoundzzzz://foo/bar"))
assert(shouldFail.isFailure)
Cache.url("testprotocol://foo/bar")
}
* - {
// the real custom protocol test
check(MavenRepository(s"${TestprotocolHandler.protocol}://foo/"))
}
}
'httpAuthentication - {
// requires an authenticated HTTP server to be running on localhost:8080 with user 'user'
// and password 'pass'
val address = "localhost:8080"
val user = "user"
val password = "pass"
def printErrorMessage() =
Console.err.println(
Console.RED +
s"HTTP authentication tests require a running HTTP server on $address, requiring " +
s"basic authentication with user '$user' and password '$password', serving the right " +
"files.\n" + Console.RESET +
"Run one from the coursier sources with\n" +
" ./coursier launch -r http://dl.bintray.com/scalaz/releases " +
"io.get-coursier:simple-web-server_2.11:1.0.0-M12 -- " +
"-d tests/jvm/src/test/resources/test-repo/http/abc.com -u user -P pass -r realm -v"
)
* - {
// no authentication -> should fail
val failed = try {
check(
MavenRepository(
s"http://$address"
)
)
printErrorMessage()
false
} catch {
case e: Throwable =>
true
}
assert(failed)
}
* - {
// with authentication -> should work
try {
check(
MavenRepository(
s"http://$address",
authentication = Some(Authentication(user, password))
)
)
} catch {
case e: Throwable =>
printErrorMessage()
throw e
}
}
}
}
}

3
tests/jvm/src/test/scala/coursier/test/ChecksumTests.scala
View File

@ -70,7 +70,8 @@ object ChecksumTests extends TestSuite {
),
Map.empty,
Attributes("jar"),
changing = false
changing = false,
authentication = None
)
val artifacts = Seq(

85
tests/jvm/src/test/scala/coursier/test/CustomProtocolTests.scala
View File

@ -1,85 +0,0 @@
package coursier
package test
import java.io.File
import java.nio.file.Files
import coursier.cache.protocol.TestprotocolHandler
import utest._
import scala.util.Try
object CustomProtocolTests extends TestSuite {
val tests = TestSuite {
def check(extraMavenRepo: String): Unit = {
val tmpDir = Files.createTempDirectory("coursier-protocol-tests").toFile
def cleanTmpDir() = {
def delete(f: File): Boolean =
if (f.isDirectory) {
val removedContent = f.listFiles().map(delete).forall(x => x)
val removedDir = f.delete()
removedContent && removedDir
} else
f.delete()
if (!delete(tmpDir))
Console.err.println(s"Warning: unable to remove temporary directory $tmpDir")
}
val res = try {
val fetch = Fetch.from(
Seq(
MavenRepository(extraMavenRepo),
MavenRepository("https://repo1.maven.org/maven2")
),
Cache.fetch(
tmpDir
)
)
val startRes = Resolution(
Set(
Dependency(
Module("com.github.alexarchambault", "coursier_2.11"), "1.0.0-M9-test"
)
)
)
startRes.process.run(fetch).run
} finally {
cleanTmpDir()
}
val errors = res.errors
assert(errors.isEmpty)
}
// using scala-test would allow to put the below comments in the test names...
* - {
// test that everything's fine with standard protocols
val repoPath = new File(getClass.getResource("/test-repo/http/abc.com").getPath)
check(repoPath.toURI.toString)
}
* - {
// test the Cache.url method
val shouldFail = Try(Cache.url("notfoundzzzz://foo/bar"))
assert(shouldFail.isFailure)
Cache.url("testprotocol://foo/bar")
}
* - {
// the real custom protocol test
check(s"${TestprotocolHandler.protocol}://foo/")
}
}
}