luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

use okhttp to download the artifacts 

This adds a custom URLHandler that internally uses Square OkHttp.

Fixes sbt/librarymanagement#95
This commit is contained in:
Eugene Yokota 2017-05-26 04:13:55 -04:00
parent 4d9013a3a3
commit fd7cec0148
7 changed files with 206 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
build.sbt
View File

@ -61,6 +61,7 @@ lazy val lm = (project in file("librarymanagement"))
scalaReflect.value,
launcherInterface,
gigahorseOkhttp,
okhttpUrlconnection,
sjsonnewScalaJson % Optional),
libraryDependencies ++= scalaXml.value,
resourceGenerators in Compile += Def

19
librarymanagement/src/main/scala/sbt/internal/librarymanagement/Ivy.scala
View File

@ -33,7 +33,7 @@ import org.apache.ivy.plugins.matcher.PatternMatcher
import org.apache.ivy.plugins.resolver.DependencyResolver
import org.apache.ivy.util.{ Message, MessageLogger }
import org.apache.ivy.util.extendable.ExtendableItem
import org.apache.ivy.util.url._
import scala.xml.NodeSeq
import scala.collection.mutable
import sbt.util.Logger
@ -43,7 +43,8 @@ import ivyint.{
CachedResolutionResolveCache,
CachedResolutionResolveEngine,
ParallelResolveEngine,
SbtDefaultDependencyDescriptor
SbtDefaultDependencyDescriptor,
GigahorseUrlHandler
}
final class IvySbt(val configuration: IvyConfiguration) { self =>
@ -72,9 +73,19 @@ final class IvySbt(val configuration: IvyConfiguration) { self =>
case None => action()
}
}
private lazy val settings: IvySettings = {
val is = new IvySettings
private lazy val basicUrlHandler: URLHandler = new BasicURLHandler
private lazy val gigahorseUrlHandler: URLHandler = {
val dispatcher = new URLHandlerDispatcher
val handler = new GigahorseUrlHandler
dispatcher.setDownloader("http", handler)
dispatcher.setDownloader("https", handler)
dispatcher
}
private lazy val settings: IvySettings = {
if (configuration.updateOptions.gigahorse) URLHandlerRegistry.setDefault(gigahorseUrlHandler)
else URLHandlerRegistry.setDefault(basicUrlHandler)
val is = new IvySettings
is.setBaseDir(baseDirectory)
is.setCircularDependencyStrategy(
configuration.updateOptions.circularDependencyLevel.ivyStrategy

2
librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala
View File

@ -24,8 +24,8 @@ trait UpdateOptionsFormat { self: BasicJsonProtocol with ModuleIDFormats with Re
uo.circularDependencyLevel.name,
uo.interProjectFirst,
uo.latestSnapshots,
uo.consolidatedResolution,
uo.cachedResolution,
uo.gigahorse,
uo.moduleResolvers
),
(xs: (String, Boolean, Boolean, Boolean, Boolean, Map[ModuleID, Resolver])) =>

171
librarymanagement/src/main/scala/sbt/internal/librarymanagement/ivyint/GigahorseUrlHandler.scala Normal file
View File

@ -0,0 +1,171 @@
package sbt.internal.librarymanagement
package ivyint
import java.net.{ URL, UnknownHostException, HttpURLConnection }
import java.io.{ File, IOException, InputStream, ByteArrayOutputStream, ByteArrayInputStream }
import org.apache.ivy.util.{ CopyProgressListener, Message, FileUtil }
import org.apache.ivy.util.url.{ URLHandler, AbstractURLHandler, BasicURLHandler }
import org.apache.ivy.util.url.URLHandler._
import sbt.librarymanagement.Http
import sbt.io.{ IO, Using }
// Copied from Ivy's BasicURLHandler.
class GigahorseUrlHandler extends AbstractURLHandler {
private val BUFFER_SIZE = 64 * 1024
/**
* Returns the URLInfo of the given url or a #UNAVAILABLE instance,
* if the url is not reachable.
*/
def getURLInfo(url: URL): URLInfo = getURLInfo(url, 0)
/**
* Returns the URLInfo of the given url or a #UNAVAILABLE instance,
* if the url is not reachable.
*/
def getURLInfo(url0: URL, timeout: Int): URLInfo = {
val url = normalizeToURL(url0)
val con = Http.open(url)
val infoOption = try {
con match {
case httpCon: HttpURLConnection =>
if (getRequestMethod == URLHandler.REQUEST_METHOD_HEAD) {
httpCon.setRequestMethod("HEAD")
}
if (checkStatusCode(url, httpCon)) {
val bodyCharset = BasicURLHandler.getCharSetFromContentType(con.getContentType)
Some(
new SbtUrlInfo(true,
httpCon.getContentLength.toLong,
con.getLastModified(),
bodyCharset))
} else None
case _ =>
val contentLength = con.getContentLength
if (contentLength <= 0) None
else {
// TODO: not HTTP... maybe we *don't* want to default to ISO-8559-1 here?
val bodyCharset = BasicURLHandler.getCharSetFromContentType(con.getContentType)
Some(new SbtUrlInfo(true, contentLength.toLong, con.getLastModified(), bodyCharset))
}
}
} catch {
case e: UnknownHostException =>
Message.warn("Host " + e.getMessage() + " not found. url=" + url)
Message.info(
"You probably access the destination server through "
+ "a proxy server that is not well configured.")
None
case e: IOException =>
Message.error("Server access Error: " + e.getMessage() + " url=" + url)
None
}
infoOption.getOrElse(UNAVAILABLE)
}
def openStream(url0: URL): InputStream = {
val url = normalizeToURL(url0)
val conn = Http.open(url)
conn.setRequestProperty("Accept-Encoding", "gzip,deflate")
conn match {
case httpCon: HttpURLConnection =>
if (!checkStatusCode(url, httpCon)) {
throw new IOException(
"The HTTP response code for " + url + " did not indicate a success."
+ " See log for more detail.")
}
case _ =>
}
val inStream = getDecodingInputStream(conn.getContentEncoding(), conn.getInputStream())
val outStream = new ByteArrayOutputStream()
val buffer = new Array[Byte](BUFFER_SIZE)
var len = 0
while ({
len = inStream.read(buffer)
len > 0
}) {
outStream.write(buffer, 0, len)
}
new ByteArrayInputStream(outStream.toByteArray())
}
def download(src0: URL, dest: File, l: CopyProgressListener): Unit = {
val src = normalizeToURL(src0)
val srcConn = Http.open(src)
srcConn.setRequestProperty("Accept-Encoding", "gzip,deflate")
srcConn match {
case httpCon: HttpURLConnection =>
if (!checkStatusCode(src, httpCon)) {
throw new IOException(
"The HTTP response code for " + src + " did not indicate a success."
+ " See log for more detail.")
}
case _ =>
}
val inStream = getDecodingInputStream(srcConn.getContentEncoding(), srcConn.getInputStream())
FileUtil.copy(inStream, dest, l)
// check content length only if content was not encoded
Option(srcConn.getContentEncoding) match {
case None =>
val contentLength = srcConn.getContentLength
if (contentLength != -1 && dest.length != contentLength) {
IO.delete(dest)
throw new IOException(
"Downloaded file size doesn't match expected Content Length for " + src
+ ". Please retry.")
}
case _ => ()
}
val lastModified = srcConn.getLastModified
if (lastModified > 0) {
dest.setLastModified(lastModified)
()
}
()
}
def upload(source: File, dest0: URL, l: CopyProgressListener): Unit = {
val dest = normalizeToURL(dest0)
val conn = Http.open(dest) match {
case c: HttpURLConnection => c
}
conn.setDoOutput(true)
conn.setRequestMethod("PUT")
conn.setRequestProperty("Content-type", "application/octet-stream")
conn.setRequestProperty("Content-length", source.length.toLong.toString)
conn.setInstanceFollowRedirects(true)
Using.fileInputStream(source) { in =>
val os = conn.getOutputStream
FileUtil.copy(in, os, l)
}
validatePutStatusCode(dest, conn.getResponseCode(), conn.getResponseMessage())
}
def checkStatusCode(url: URL, con: HttpURLConnection): Boolean =
con.getResponseCode match {
case 200 => true
case 204 if "HEAD" == con.getRequestMethod => true
case status =>
Message.debug("HTTP response status: " + status + " url=" + url)
if (status == 407 /* PROXY_AUTHENTICATION_REQUIRED */ ) {
Message.warn("Your proxy requires authentication.");
} else if (String.valueOf(status).startsWith("4")) {
Message.verbose("CLIENT ERROR: " + con.getResponseMessage() + " url=" + url)
} else if (String.valueOf(status).startsWith("5")) {
Message.error("SERVER ERROR: " + con.getResponseMessage() + " url=" + url)
}
false
}
// This is requires to access the constructor of URLInfo.
private[sbt] class SbtUrlInfo(available: Boolean,
contentLength: Long,
lastModified: Long,
bodyCharset: String)
extends URLInfo(available, contentLength, lastModified, bodyCharset) {
def this(available: Boolean, contentLength: Long, lastModified: Long) = {
this(available, contentLength, lastModified, null)
}
}
}

6
librarymanagement/src/main/scala/sbt/librarymanagement/Http.scala
View File

@ -1,7 +1,13 @@
package sbt.librarymanagement
import gigahorse._, support.okhttp.Gigahorse
import okhttp3.{ OkUrlFactory, OkHttpClient }
import java.net.{ URL, HttpURLConnection }
object Http {
lazy val http: HttpClient = Gigahorse.http(Gigahorse.config)
private[sbt] lazy val urlFactory = new OkUrlFactory(http.underlying[OkHttpClient])
private[sbt] def open(url: URL): HttpURLConnection =
urlFactory.open(url)
}

26
librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala
View File

@ -18,10 +18,10 @@ final class UpdateOptions private[sbt] (
val interProjectFirst: Boolean,
// If set to true, check all resolvers for snapshots.
val latestSnapshots: Boolean,
// If set to true, use consolidated resolution.
val consolidatedResolution: Boolean,
// If set to true, use cached resolution.
val cachedResolution: Boolean,
// If set to true, use Gigahorse
val gigahorse: Boolean,
// Extension point for an alternative resolver converter.
val resolverConverter: UpdateOptions.ResolverConverter,
// Map the unique resolver to be checked for the module ID
@ -35,17 +35,11 @@ final class UpdateOptions private[sbt] (
copy(interProjectFirst = interProjectFirst)
def withLatestSnapshots(latestSnapshots: Boolean): UpdateOptions =
copy(latestSnapshots = latestSnapshots)
@deprecated("Use withCachedResolution instead.", "0.13.7")
def withConsolidatedResolution(consolidatedResolution: Boolean): UpdateOptions =
copy(
consolidatedResolution = consolidatedResolution,
cachedResolution = consolidatedResolution
)
def withCachedResolution(cachedResoluton: Boolean): UpdateOptions =
copy(
cachedResolution = cachedResoluton,
consolidatedResolution = cachedResolution
)
copy(cachedResolution = cachedResoluton)
def withGigahorse(gigahorse: Boolean): UpdateOptions =
copy(gigahorse = gigahorse)
/** Extention point for an alternative resolver converter. */
def withResolverConverter(resolverConverter: UpdateOptions.ResolverConverter): UpdateOptions =
@ -58,8 +52,8 @@ final class UpdateOptions private[sbt] (
circularDependencyLevel: CircularDependencyLevel = this.circularDependencyLevel,
interProjectFirst: Boolean = this.interProjectFirst,
latestSnapshots: Boolean = this.latestSnapshots,
consolidatedResolution: Boolean = this.consolidatedResolution,
cachedResolution: Boolean = this.cachedResolution,
gigahorse: Boolean = this.gigahorse,
resolverConverter: UpdateOptions.ResolverConverter = this.resolverConverter,
moduleResolvers: Map[ModuleID, Resolver] = this.moduleResolvers
): UpdateOptions =
@ -67,8 +61,8 @@ final class UpdateOptions private[sbt] (
circularDependencyLevel,
interProjectFirst,
latestSnapshots,
consolidatedResolution,
cachedResolution,
gigahorse,
resolverConverter,
moduleResolvers
)
@ -79,6 +73,7 @@ final class UpdateOptions private[sbt] (
this.interProjectFirst == o.interProjectFirst &&
this.latestSnapshots == o.latestSnapshots &&
this.cachedResolution == o.cachedResolution &&
this.gigahorse == o.gigahorse &&
this.resolverConverter == o.resolverConverter &&
this.moduleResolvers == o.moduleResolvers
case _ => false
@ -90,6 +85,7 @@ final class UpdateOptions private[sbt] (
hash = hash * 31 + this.interProjectFirst.##
hash = hash * 31 + this.latestSnapshots.##
hash = hash * 31 + this.cachedResolution.##
hash = hash * 31 + this.gigahorse.##
hash = hash * 31 + this.resolverConverter.##
hash = hash * 31 + this.moduleResolvers.##
hash
@ -104,8 +100,8 @@ object UpdateOptions {
circularDependencyLevel = CircularDependencyLevel.Warn,
interProjectFirst = true,
latestSnapshots = true,
consolidatedResolution = false,
cachedResolution = false,
gigahorse = true,
resolverConverter = PartialFunction.empty,
moduleResolvers = Map.empty
)

1
project/Dependencies.scala
View File

@ -58,6 +58,7 @@ object Dependencies {
val sjsonnewVersion = "0.7.0"
val sjsonnewScalaJson = "com.eed3si9n" %% "sjson-new-scalajson" % sjsonnewVersion
val gigahorseOkhttp = "com.eed3si9n" %% "gigahorse-okhttp" % "0.3.0"
val okhttpUrlconnection = "com.squareup.okhttp3" % "okhttp-urlconnection" % "3.7.0"
private def scala211Module(name: String, moduleVersion: String) =
Def.setting {