diff --git a/build.sbt b/build.sbt index c5ad63207..97cf4a78e 100644 --- a/build.sbt +++ b/build.sbt @@ -61,6 +61,7 @@ lazy val lm = (project in file("librarymanagement")) scalaReflect.value, launcherInterface, gigahorseOkhttp, + okhttpUrlconnection, sjsonnewScalaJson % Optional), libraryDependencies ++= scalaXml.value, resourceGenerators in Compile += Def diff --git a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/Ivy.scala b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/Ivy.scala index e27eb1907..a6acdbafc 100644 --- a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/Ivy.scala +++ b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/Ivy.scala @@ -33,7 +33,7 @@ import org.apache.ivy.plugins.matcher.PatternMatcher import org.apache.ivy.plugins.resolver.DependencyResolver import org.apache.ivy.util.{ Message, MessageLogger } import org.apache.ivy.util.extendable.ExtendableItem - +import org.apache.ivy.util.url._ import scala.xml.NodeSeq import scala.collection.mutable import sbt.util.Logger @@ -43,7 +43,8 @@ import ivyint.{ CachedResolutionResolveCache, CachedResolutionResolveEngine, ParallelResolveEngine, - SbtDefaultDependencyDescriptor + SbtDefaultDependencyDescriptor, + GigahorseUrlHandler } final class IvySbt(val configuration: IvyConfiguration) { self => @@ -72,9 +73,19 @@ final class IvySbt(val configuration: IvyConfiguration) { self => case None => action() } } - private lazy val settings: IvySettings = { - val is = new IvySettings + private lazy val basicUrlHandler: URLHandler = new BasicURLHandler + private lazy val gigahorseUrlHandler: URLHandler = { + val dispatcher = new URLHandlerDispatcher + val handler = new GigahorseUrlHandler + dispatcher.setDownloader("http", handler) + dispatcher.setDownloader("https", handler) + dispatcher + } + private lazy val settings: IvySettings = { + if (configuration.updateOptions.gigahorse) URLHandlerRegistry.setDefault(gigahorseUrlHandler) + else URLHandlerRegistry.setDefault(basicUrlHandler) + val is = new IvySettings is.setBaseDir(baseDirectory) is.setCircularDependencyStrategy( configuration.updateOptions.circularDependencyLevel.ivyStrategy diff --git a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala index 5074e8a0e..ae080e80e 100644 --- a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala +++ b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala @@ -24,8 +24,8 @@ trait UpdateOptionsFormat { self: BasicJsonProtocol with ModuleIDFormats with Re uo.circularDependencyLevel.name, uo.interProjectFirst, uo.latestSnapshots, - uo.consolidatedResolution, uo.cachedResolution, + uo.gigahorse, uo.moduleResolvers ), (xs: (String, Boolean, Boolean, Boolean, Boolean, Map[ModuleID, Resolver])) => diff --git a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ivyint/GigahorseUrlHandler.scala b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ivyint/GigahorseUrlHandler.scala new file mode 100644 index 000000000..6603919b5 --- /dev/null +++ b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ivyint/GigahorseUrlHandler.scala @@ -0,0 +1,171 @@ +package sbt.internal.librarymanagement +package ivyint + +import java.net.{ URL, UnknownHostException, HttpURLConnection } +import java.io.{ File, IOException, InputStream, ByteArrayOutputStream, ByteArrayInputStream } +import org.apache.ivy.util.{ CopyProgressListener, Message, FileUtil } +import org.apache.ivy.util.url.{ URLHandler, AbstractURLHandler, BasicURLHandler } +import org.apache.ivy.util.url.URLHandler._ +import sbt.librarymanagement.Http +import sbt.io.{ IO, Using } + +// Copied from Ivy's BasicURLHandler. +class GigahorseUrlHandler extends AbstractURLHandler { + private val BUFFER_SIZE = 64 * 1024 + + /** + * Returns the URLInfo of the given url or a #UNAVAILABLE instance, + * if the url is not reachable. + */ + def getURLInfo(url: URL): URLInfo = getURLInfo(url, 0) + + /** + * Returns the URLInfo of the given url or a #UNAVAILABLE instance, + * if the url is not reachable. + */ + def getURLInfo(url0: URL, timeout: Int): URLInfo = { + val url = normalizeToURL(url0) + val con = Http.open(url) + val infoOption = try { + con match { + case httpCon: HttpURLConnection => + if (getRequestMethod == URLHandler.REQUEST_METHOD_HEAD) { + httpCon.setRequestMethod("HEAD") + } + if (checkStatusCode(url, httpCon)) { + val bodyCharset = BasicURLHandler.getCharSetFromContentType(con.getContentType) + Some( + new SbtUrlInfo(true, + httpCon.getContentLength.toLong, + con.getLastModified(), + bodyCharset)) + } else None + + case _ => + val contentLength = con.getContentLength + if (contentLength <= 0) None + else { + // TODO: not HTTP... maybe we *don't* want to default to ISO-8559-1 here? + val bodyCharset = BasicURLHandler.getCharSetFromContentType(con.getContentType) + Some(new SbtUrlInfo(true, contentLength.toLong, con.getLastModified(), bodyCharset)) + } + } + } catch { + case e: UnknownHostException => + Message.warn("Host " + e.getMessage() + " not found. url=" + url) + Message.info( + "You probably access the destination server through " + + "a proxy server that is not well configured.") + None + case e: IOException => + Message.error("Server access Error: " + e.getMessage() + " url=" + url) + None + } + infoOption.getOrElse(UNAVAILABLE) + } + + def openStream(url0: URL): InputStream = { + val url = normalizeToURL(url0) + val conn = Http.open(url) + conn.setRequestProperty("Accept-Encoding", "gzip,deflate") + conn match { + case httpCon: HttpURLConnection => + if (!checkStatusCode(url, httpCon)) { + throw new IOException( + "The HTTP response code for " + url + " did not indicate a success." + + " See log for more detail.") + } + case _ => + } + val inStream = getDecodingInputStream(conn.getContentEncoding(), conn.getInputStream()) + val outStream = new ByteArrayOutputStream() + val buffer = new Array[Byte](BUFFER_SIZE) + var len = 0 + while ({ + len = inStream.read(buffer) + len > 0 + }) { + outStream.write(buffer, 0, len) + } + new ByteArrayInputStream(outStream.toByteArray()) + } + + def download(src0: URL, dest: File, l: CopyProgressListener): Unit = { + val src = normalizeToURL(src0) + val srcConn = Http.open(src) + srcConn.setRequestProperty("Accept-Encoding", "gzip,deflate") + srcConn match { + case httpCon: HttpURLConnection => + if (!checkStatusCode(src, httpCon)) { + throw new IOException( + "The HTTP response code for " + src + " did not indicate a success." + + " See log for more detail.") + } + case _ => + } + val inStream = getDecodingInputStream(srcConn.getContentEncoding(), srcConn.getInputStream()) + FileUtil.copy(inStream, dest, l) + // check content length only if content was not encoded + Option(srcConn.getContentEncoding) match { + case None => + val contentLength = srcConn.getContentLength + if (contentLength != -1 && dest.length != contentLength) { + IO.delete(dest) + throw new IOException( + "Downloaded file size doesn't match expected Content Length for " + src + + ". Please retry.") + } + case _ => () + } + val lastModified = srcConn.getLastModified + if (lastModified > 0) { + dest.setLastModified(lastModified) + () + } + () + } + + def upload(source: File, dest0: URL, l: CopyProgressListener): Unit = { + val dest = normalizeToURL(dest0) + val conn = Http.open(dest) match { + case c: HttpURLConnection => c + } + conn.setDoOutput(true) + conn.setRequestMethod("PUT") + conn.setRequestProperty("Content-type", "application/octet-stream") + conn.setRequestProperty("Content-length", source.length.toLong.toString) + conn.setInstanceFollowRedirects(true) + Using.fileInputStream(source) { in => + val os = conn.getOutputStream + FileUtil.copy(in, os, l) + } + validatePutStatusCode(dest, conn.getResponseCode(), conn.getResponseMessage()) + } + + def checkStatusCode(url: URL, con: HttpURLConnection): Boolean = + con.getResponseCode match { + case 200 => true + case 204 if "HEAD" == con.getRequestMethod => true + case status => + Message.debug("HTTP response status: " + status + " url=" + url) + if (status == 407 /* PROXY_AUTHENTICATION_REQUIRED */ ) { + Message.warn("Your proxy requires authentication."); + } else if (String.valueOf(status).startsWith("4")) { + Message.verbose("CLIENT ERROR: " + con.getResponseMessage() + " url=" + url) + } else if (String.valueOf(status).startsWith("5")) { + Message.error("SERVER ERROR: " + con.getResponseMessage() + " url=" + url) + } + false + } + + // This is requires to access the constructor of URLInfo. + private[sbt] class SbtUrlInfo(available: Boolean, + contentLength: Long, + lastModified: Long, + bodyCharset: String) + extends URLInfo(available, contentLength, lastModified, bodyCharset) { + def this(available: Boolean, contentLength: Long, lastModified: Long) = { + this(available, contentLength, lastModified, null) + } + } +} diff --git a/librarymanagement/src/main/scala/sbt/librarymanagement/Http.scala b/librarymanagement/src/main/scala/sbt/librarymanagement/Http.scala index 9870d9b0f..3fee33b82 100644 --- a/librarymanagement/src/main/scala/sbt/librarymanagement/Http.scala +++ b/librarymanagement/src/main/scala/sbt/librarymanagement/Http.scala @@ -1,7 +1,13 @@ package sbt.librarymanagement import gigahorse._, support.okhttp.Gigahorse +import okhttp3.{ OkUrlFactory, OkHttpClient } +import java.net.{ URL, HttpURLConnection } object Http { lazy val http: HttpClient = Gigahorse.http(Gigahorse.config) + + private[sbt] lazy val urlFactory = new OkUrlFactory(http.underlying[OkHttpClient]) + private[sbt] def open(url: URL): HttpURLConnection = + urlFactory.open(url) } diff --git a/librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala b/librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala index 4c3810055..c0214a6cb 100644 --- a/librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala +++ b/librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala @@ -18,10 +18,10 @@ final class UpdateOptions private[sbt] ( val interProjectFirst: Boolean, // If set to true, check all resolvers for snapshots. val latestSnapshots: Boolean, - // If set to true, use consolidated resolution. - val consolidatedResolution: Boolean, // If set to true, use cached resolution. val cachedResolution: Boolean, + // If set to true, use Gigahorse + val gigahorse: Boolean, // Extension point for an alternative resolver converter. val resolverConverter: UpdateOptions.ResolverConverter, // Map the unique resolver to be checked for the module ID @@ -35,17 +35,11 @@ final class UpdateOptions private[sbt] ( copy(interProjectFirst = interProjectFirst) def withLatestSnapshots(latestSnapshots: Boolean): UpdateOptions = copy(latestSnapshots = latestSnapshots) - @deprecated("Use withCachedResolution instead.", "0.13.7") - def withConsolidatedResolution(consolidatedResolution: Boolean): UpdateOptions = - copy( - consolidatedResolution = consolidatedResolution, - cachedResolution = consolidatedResolution - ) def withCachedResolution(cachedResoluton: Boolean): UpdateOptions = - copy( - cachedResolution = cachedResoluton, - consolidatedResolution = cachedResolution - ) + copy(cachedResolution = cachedResoluton) + + def withGigahorse(gigahorse: Boolean): UpdateOptions = + copy(gigahorse = gigahorse) /** Extention point for an alternative resolver converter. */ def withResolverConverter(resolverConverter: UpdateOptions.ResolverConverter): UpdateOptions = @@ -58,8 +52,8 @@ final class UpdateOptions private[sbt] ( circularDependencyLevel: CircularDependencyLevel = this.circularDependencyLevel, interProjectFirst: Boolean = this.interProjectFirst, latestSnapshots: Boolean = this.latestSnapshots, - consolidatedResolution: Boolean = this.consolidatedResolution, cachedResolution: Boolean = this.cachedResolution, + gigahorse: Boolean = this.gigahorse, resolverConverter: UpdateOptions.ResolverConverter = this.resolverConverter, moduleResolvers: Map[ModuleID, Resolver] = this.moduleResolvers ): UpdateOptions = @@ -67,8 +61,8 @@ final class UpdateOptions private[sbt] ( circularDependencyLevel, interProjectFirst, latestSnapshots, - consolidatedResolution, cachedResolution, + gigahorse, resolverConverter, moduleResolvers ) @@ -79,6 +73,7 @@ final class UpdateOptions private[sbt] ( this.interProjectFirst == o.interProjectFirst && this.latestSnapshots == o.latestSnapshots && this.cachedResolution == o.cachedResolution && + this.gigahorse == o.gigahorse && this.resolverConverter == o.resolverConverter && this.moduleResolvers == o.moduleResolvers case _ => false @@ -90,6 +85,7 @@ final class UpdateOptions private[sbt] ( hash = hash * 31 + this.interProjectFirst.## hash = hash * 31 + this.latestSnapshots.## hash = hash * 31 + this.cachedResolution.## + hash = hash * 31 + this.gigahorse.## hash = hash * 31 + this.resolverConverter.## hash = hash * 31 + this.moduleResolvers.## hash @@ -104,8 +100,8 @@ object UpdateOptions { circularDependencyLevel = CircularDependencyLevel.Warn, interProjectFirst = true, latestSnapshots = true, - consolidatedResolution = false, cachedResolution = false, + gigahorse = true, resolverConverter = PartialFunction.empty, moduleResolvers = Map.empty ) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 2ad1ff99a..e5be5fa54 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -58,6 +58,7 @@ object Dependencies { val sjsonnewVersion = "0.7.0" val sjsonnewScalaJson = "com.eed3si9n" %% "sjson-new-scalajson" % sjsonnewVersion val gigahorseOkhttp = "com.eed3si9n" %% "gigahorse-okhttp" % "0.3.0" + val okhttpUrlconnection = "com.squareup.okhttp3" % "okhttp-urlconnection" % "3.7.0" private def scala211Module(name: String, moduleVersion: String) = Def.setting {