Merge pull request #330 from alexarchambault/topic/develop

Latest developments

cache/src/main/scala/coursier/Cache.scala

@@ -22,6 +22,7 @@ import java.io.{ Serializable => _, _ }
 
 import scala.concurrent.duration.{ Duration, DurationInt }
 import scala.util.Try
+import scala.util.control.NonFatal
 
 trait AuthenticatedURLConnection extends URLConnection {
   def authenticate(authentication: Authentication): Unit
@@ -325,27 +326,41 @@ object Cache {
     def referenceFileExists: Boolean = referenceFileOpt.exists(_.exists())
 
     def urlConn(url0: String) = {
-      val conn = url(url0).openConnection() // FIXME Should this be closed?
+      var conn: URLConnection = null
-      // Dummy user-agent instead of the default "Java/...",
-      // so that we are not returned incomplete/erroneous metadata
-      // (Maven 2 compatibility? - happens for snapshot versioning metadata,
-      // this is SO FSCKING CRAZY)
-      conn.setRequestProperty("User-Agent", "")
 
-      for (auth <- artifact.authentication)
+      try {
-        conn match {
+        conn = url(url0).openConnection() // FIXME Should this be closed?
-          case authenticated: AuthenticatedURLConnection =>
+        // Dummy user-agent instead of the default "Java/...",
-            authenticated.authenticate(auth)
+        // so that we are not returned incomplete/erroneous metadata
-          case conn0: HttpURLConnection =>
+        // (Maven 2 compatibility? - happens for snapshot versioning metadata,
-            conn0.setRequestProperty(
+        // this is SO FSCKING CRAZY)
-              "Authorization",
+        conn.setRequestProperty("User-Agent", "")
-              "Basic " + basicAuthenticationEncode(auth.user, auth.password)
+
-            )
+        for (auth <- artifact.authentication)
-          case _ =>
+          conn match {
+            case authenticated: AuthenticatedURLConnection =>
+              authenticated.authenticate(auth)
+            case conn0: HttpURLConnection =>
+              conn0.setRequestProperty(
+                "Authorization",
+                "Basic " + basicAuthenticationEncode(auth.user, auth.password)
+              )
+            case _ =>
             // FIXME Authentication is ignored
-        }
+          }
 
-      conn
+        conn
+      } catch {
+        case NonFatal(e) =>
+          if (conn != null)
+            conn match {
+              case conn0: HttpURLConnection =>
+                conn0.getInputStream.close()
+                conn0.disconnect()
+              case _ =>
+            }
+          throw e
+      }
     }
 
 
@@ -369,34 +384,47 @@ object Cache {
     ): EitherT[Task, FileError, Option[Long]] =
       EitherT {
         Task {
-          urlConn(url) match {
+          var conn: URLConnection = null
-            case c: HttpURLConnection =>
-              logger.foreach(_.checkingUpdates(url, currentLastModifiedOpt))
 
-              var success = false
+          try {
-              try {
+            conn = urlConn(url)
-                c.setRequestMethod("HEAD")
-                val remoteLastModified = c.getLastModified
 
-                // TODO 404 Not found could be checked here
+            conn match {
+              case c: HttpURLConnection =>
+                logger.foreach(_.checkingUpdates(url, currentLastModifiedOpt))
 
-                val res =
+                var success = false
-                  if (remoteLastModified > 0L)
+                try {
-                    Some(remoteLastModified)
+                  c.setRequestMethod("HEAD")
-                  else
+                  val remoteLastModified = c.getLastModified
-                    None
 
-                success = true
+                  // TODO 404 Not found could be checked here
-                logger.foreach(_.checkingUpdatesResult(url, currentLastModifiedOpt, res))
 
-                res.right
+                  val res =
-              } finally {
+                    if (remoteLastModified > 0L)
-                if (!success)
+                      Some(remoteLastModified)
-                  logger.foreach(_.checkingUpdatesResult(url, currentLastModifiedOpt, None))
+                    else
+                      None
+
+                  success = true
+                  logger.foreach(_.checkingUpdatesResult(url, currentLastModifiedOpt, res))
+
+                  res.right
+                } finally {
+                  if (!success)
+                    logger.foreach(_.checkingUpdatesResult(url, currentLastModifiedOpt, None))
+                }
+
+              case other =>
+                -\/(FileError.DownloadError(s"Cannot do HEAD request with connection $other ($url)"))
+            }
+          } finally {
+            if (conn != null)
+              conn match {
+                case conn0: HttpURLConnection =>
+                  conn0.disconnect()
+                case _ =>
               }
-
-            case other =>
-              -\/(FileError.DownloadError(s"Cannot do HEAD request with connection $other ($url)"))
           }
         }
       }
@@ -512,59 +540,70 @@ object Cache {
 
               val alreadyDownloaded = tmp.length()
 
-              val conn0 = urlConn(url)
+              var conn: URLConnection = null
 
-              val (partialDownload, conn) = conn0 match {
+              try {
-                case conn0: HttpURLConnection if alreadyDownloaded > 0L =>
+                conn = urlConn(url)
-                  conn0.setRequestProperty("Range", s"bytes=$alreadyDownloaded-")
 
-                  if (conn0.getResponseCode == partialContentResponseCode) {
+                val partialDownload = conn match {
-                    val ackRange = Option(conn0.getHeaderField("Content-Range")).getOrElse("")
+                  case conn0: HttpURLConnection if alreadyDownloaded > 0L =>
+                    conn0.setRequestProperty("Range", s"bytes=$alreadyDownloaded-")
 
-                    if (ackRange.startsWith(s"bytes $alreadyDownloaded-"))
+                    (conn0.getResponseCode == partialContentResponseCode) && {
-                      (true, conn0)
+                      val ackRange = Option(conn0.getHeaderField("Content-Range")).getOrElse("")
-                    else
-                      // unrecognized Content-Range header -> start a new connection with no resume
-                      (false, urlConn(url))
-                  } else
-                    (false, conn0)
 
-                case _ => (false, conn0)
+                      ackRange.startsWith(s"bytes $alreadyDownloaded-") || {
-              }
+                        // unrecognized Content-Range header -> start a new connection with no resume
-
+                        conn0.getInputStream.close()
-              if (responseCode(conn) == Some(404))
+                        conn0.disconnect()
-                FileError.NotFound(url, permanent = Some(true)).left
+                        conn = urlConn(url)
-              else if (responseCode(conn) == Some(401))
+                        false
-                FileError.Unauthorized(url, realm = realm(conn)).left
+                      }
-              else {
-                for (len0 <- Option(conn.getContentLengthLong) if len0 >= 0L) {
-                  val len = len0 + (if (partialDownload) alreadyDownloaded else 0L)
-                  logger.foreach(_.downloadLength(url, len, alreadyDownloaded))
-                }
-
-                val in = new BufferedInputStream(conn.getInputStream, bufferSize)
-
-                val result =
-                  try {
-                    val out = withStructureLock(cache) {
-                      tmp.getParentFile.mkdirs()
-                      new FileOutputStream(tmp, partialDownload)
                     }
-                    try readFullyTo(in, out, logger, url, if (partialDownload) alreadyDownloaded else 0L)
+                  case _ => false
-                    finally out.close()
-                  } finally in.close()
-
-                withStructureLock(cache) {
-                  file.getParentFile.mkdirs()
-                  NioFiles.move(tmp.toPath, file.toPath, StandardCopyOption.ATOMIC_MOVE)
                 }
 
-                for (lastModified <- Option(conn.getLastModified) if lastModified > 0L)
+                if (responseCode(conn) == Some(404))
-                  file.setLastModified(lastModified)
+                  FileError.NotFound(url, permanent = Some(true)).left
+                else if (responseCode(conn) == Some(401))
+                  FileError.Unauthorized(url, realm = realm(conn)).left
+                else {
+                  for (len0 <- Option(conn.getContentLengthLong) if len0 >= 0L) {
+                    val len = len0 + (if (partialDownload) alreadyDownloaded else 0L)
+                    logger.foreach(_.downloadLength(url, len, alreadyDownloaded))
+                  }
 
-                doTouchCheckFile(file)
+                  val in = new BufferedInputStream(conn.getInputStream, bufferSize)
 
-                result.right
+                  val result =
+                    try {
+                      val out = withStructureLock(cache) {
+                        tmp.getParentFile.mkdirs()
+                        new FileOutputStream(tmp, partialDownload)
+                      }
+                      try readFullyTo(in, out, logger, url, if (partialDownload) alreadyDownloaded else 0L)
+                      finally out.close()
+                    } finally in.close()
+
+                  withStructureLock(cache) {
+                    file.getParentFile.mkdirs()
+                    NioFiles.move(tmp.toPath, file.toPath, StandardCopyOption.ATOMIC_MOVE)
+                  }
+
+                  for (lastModified <- Option(conn.getLastModified) if lastModified > 0L)
+                    file.setLastModified(lastModified)
+
+                  doTouchCheckFile(file)
+
+                  result.right
+                }
+              } finally {
+                if (conn != null)
+                  conn match {
+                    case conn0: HttpURLConnection =>
+                      conn0.disconnect()
+                    case _ =>
+                  }
               }
             }
           }
@@ -852,16 +891,58 @@ object Cache {
         pool = pool,
         ttl = ttl
       ).leftMap(_.describe).flatMap { f =>
-        val res = if (!f.isDirectory && f.exists()) {
+
-          Try(new String(NioFiles.readAllBytes(f.toPath), "UTF-8")) match {
+        def notFound(f: File) = Left(s"${f.getCanonicalPath} not found")
-            case scala.util.Success(content) =>
+
-              // stripping any UTF-8 BOM if any, see
+        def read(f: File) =
-              // https://github.com/alexarchambault/coursier/issues/316 and the corresponding test
+          try Right(new String(NioFiles.readAllBytes(f.toPath), "UTF-8").stripPrefix(utf8Bom))
-              Right(content.stripPrefix(utf8Bom))
+          catch {
-            case scala.util.Failure(e) =>
+            case NonFatal(e) =>
               Left(s"Could not read (file:${f.getCanonicalPath}): ${e.getMessage}")
           }
-        } else Left(s"Could not read (file:${f.getCanonicalPath}) (isFile:${!f.isDirectory}) (exists:${f.exists()})")
+
+        val res = if (f.exists()) {
+          if (f.isDirectory) {
+            if (artifact.url.startsWith("file:")) {
+
+              val elements = f.listFiles().map { c =>
+                val name = c.getName
+                val name0 = if (c.isDirectory)
+                  name + "/"
+                else
+                  name
+
+                s"""<li><a href="$name0">$name0</a></li>"""
+              }.mkString
+
+              val page =
+                s"""<!DOCTYPE html>
+                   |<html>
+                   |<head></head>
+                   |<body>
+                   |<ul>
+                   |$elements
+                   |</ul>
+                   |</body>
+                   |</html>
+                 """.stripMargin
+
+              Right(page)
+            } else {
+              val f0 = new File(f, ".directory")
+
+              if (f0.exists()) {
+                if (f0.isDirectory)
+                  Left(s"Woops: ${f.getCanonicalPath} is a directory")
+                else
+                  read(f0)
+              } else
+                notFound(f0)
+            }
+          } else
+            read(f)
+        } else
+          notFound(f)
 
         EitherT.fromEither(Task.now[Either[String, String]](res))
       }

core/js/src/main/scala/coursier/core/compatibility/package.scala

@@ -93,7 +93,12 @@ package object compatibility {
   def encodeURIComponent(s: String): String =
     g.encodeURIComponent(s).asInstanceOf[String]
 
-  def listWebPageSubDirectories(page: String): Seq[String] = {
+  def listWebPageSubDirectories(url: String, page: String): Seq[String] = {
+    // TODO
+    ???
+  }
+
+  def listWebPageFiles(url: String, page: String): Seq[String] = {
     // TODO
     ???
   }

core/jvm/src/main/scala/coursier/core/compatibility/package.scala

@@ -56,17 +56,25 @@ package object compatibility {
   def encodeURIComponent(s: String): String =
     new java.net.URI(null, null, null, -1, s, null, null) .toASCIIString
 
-  def listWebPageSubDirectories(page: String): Seq[String] =
+  def listWebPageDirectoryElements(url: String, page: String, directories: Boolean): Seq[String] =
     Jsoup.parse(page)
-      .select("a[href~=[^/]*/]")
+      .select("a")
       .asScala
       .toVector
-      .map { elem =>
+      .map(_.attr("href"))
-        elem
+      .collect {
-          .attr("href")
+        case elem if elem.nonEmpty && elem.endsWith("/") == directories =>
-          .stripPrefix(":") // bintray typically prepends these
+          elem
-          .stripSuffix("/")
+            .stripSuffix("/")
+            .stripPrefix(url)
+            .stripPrefix(":") // bintray typically prepends these
       }
-      .filter(n => n != "." && n != "..")
+      .filter(n => !n.contains("/") && n != "." && n != "..")
+
+  def listWebPageSubDirectories(url: String, page: String): Seq[String] =
+    listWebPageDirectoryElements(url, page, directories = true)
+
+  def listWebPageFiles(url: String, page: String): Seq[String] =
+    listWebPageDirectoryElements(url, page, directories = false)
 
 }

core/shared/src/main/scala/coursier/ivy/IvyRepository.scala

@@ -147,8 +147,8 @@ case class IvyRepository(
                 s"Don't know how to list revisions of ${metadataPattern.string}".left
             }
 
-            def fromWebPage(s: String) = {
+            def fromWebPage(url: String, s: String) = {
-              val subDirs = coursier.core.compatibility.listWebPageSubDirectories(s)
+              val subDirs = coursier.core.compatibility.listWebPageSubDirectories(url, s)
               val versions = subDirs.map(Parse.version).collect { case Some(v) => v }
               val versionsInItv = versions.filter(itv.contains)
 
@@ -173,7 +173,7 @@ case class IvyRepository(
             for {
               url <- EitherT(F.point(listingUrl))
               s <- fetch(artifactFor(url))
-              res <- fromWebPage(s)
+              res <- fromWebPage(url, s)
             } yield res
         }
     }

plugin/src/main/scala-2.10/coursier/FallbackDependenciesRepository.scala

@@ -8,7 +8,7 @@ case class FallbackDependenciesRepository(
   fallbacks: Map[(Module, String), (URL, Boolean)]
 ) extends Repository {
 
-  private val source = new Artifact.Source {
+  private val source: Artifact.Source = new Artifact.Source {
     def artifacts(
       dependency: Dependency,
       project: Project,
@@ -35,22 +35,41 @@ case class FallbackDependenciesRepository(
         EitherT.left(F.point("No fallback URL found"))
 
       case Some((url, _)) =>
-        val proj = Project(
-          module,
-          version,
-          Nil,
-          Map.empty,
-          None,
-          Nil,
-          Nil,
-          Nil,
-          None,
-          None,
-          None,
-          Nil,
-          Info.empty
-        )
 
-        EitherT.right(F.point((source, proj)))
+        val urlStr = url.toExternalForm
+        val idx = urlStr.lastIndexOf('/')
+
+        if (idx < 0 || urlStr.endsWith("/"))
+          EitherT.left(F.point(s"$url doesn't point to a file"))
+        else {
+          val (dirUrlStr, fileName) = urlStr.splitAt(idx + 1)
+
+          fetch(Artifact(dirUrlStr, Map.empty, Map.empty, Attributes("", ""), changing = true, None)).flatMap { listing =>
+
+            val files = coursier.core.compatibility.listWebPageFiles(dirUrlStr, listing)
+
+            if (files.contains(fileName)) {
+
+              val proj = Project(
+                module,
+                version,
+                Nil,
+                Map.empty,
+                None,
+                Nil,
+                Nil,
+                Nil,
+                None,
+                None,
+                None,
+                Nil,
+                Info.empty
+              )
+
+              EitherT.right(F.point((source, proj)))
+            } else
+              EitherT.left(F.point(s"$fileName not found under $dirUrlStr"))
+          }
+        }
     }
 }

plugin/src/main/scala-2.10/coursier/Tasks.scala

@@ -276,9 +276,6 @@ object Tasks {
       Module(scalaOrganization, "scalap") -> scalaVersion
     )
 
-  private def projectDescription(project: Project) =
-    s"${project.module.organization}:${project.module.name}:${project.version}"
-
   private def createLogger() = new TermDisplay(new OutputStreamWriter(System.err))
 
   private lazy val globalPluginPattern = {
@@ -325,6 +322,8 @@ object Tasks {
 
       lazy val cm = coursierSbtClassifiersModule.value
 
+      lazy val projectName = thisProjectRef.value.project
+
       val (currentProject, fallbackDependencies) =
         if (sbtClassifiers) {
           val sv = scalaVersion.value
@@ -614,8 +613,7 @@ object Tasks {
 
           if (verbosityLevel >= 0)
             log.info(
-              s"Updating ${projectDescription(currentProject)}" +
+              s"Updating $projectName" + (if (sbtClassifiers) " (sbt classifiers)" else "")
-                (if (sbtClassifiers) " (sbt classifiers)" else "")
             )
           if (verbosityLevel >= 2)
             for (depRepr <- depsRepr(currentProject.dependencies))
@@ -663,7 +661,7 @@ object Tasks {
           }
 
           if (verbosityLevel >= 0)
-            log.info(s"Resolved ${projectDescription(currentProject)} dependencies")
+            log.info(s"Resolved $projectName dependencies")
 
           res
         } finally {
@@ -705,6 +703,8 @@ object Tasks {
 
       lazy val cm = coursierSbtClassifiersModule.value
 
+      lazy val projectName = thisProjectRef.value.project
+
       val currentProject =
         if (sbtClassifiers) {
           val sv = scalaVersion.value
@@ -834,7 +834,7 @@ object Tasks {
 
           if (verbosityLevel >= 0)
             log.info(
-              s"Fetching artifacts of ${projectDescription(currentProject)}" +
+              s"Fetching artifacts of $projectName" +
                 (if (sbtClassifiers) " (sbt classifiers)" else "")
             )
 
@@ -850,7 +850,7 @@ object Tasks {
 
           if (verbosityLevel >= 0)
             log.info(
-              s"Fetched artifacts of ${projectDescription(currentProject)}" +
+              s"Fetched artifacts of $projectName" +
                 (if (sbtClassifiers) " (sbt classifiers)" else "")
             )
 
@@ -925,6 +925,8 @@ object Tasks {
     ignoreArtifactErrors: Boolean = false
   ) = Def.task {
 
+    lazy val projectName = thisProjectRef.value.project
+
     val currentProject =
       if (sbtClassifiers) {
         val cm = coursierSbtClassifiersModule.value
@@ -953,7 +955,7 @@ object Tasks {
         coursierResolution
     }.value
 
-    val config = classpathConfiguration.value.name
+    val config = configuration.value.name
     val configs = coursierConfigurations.value
 
     val includedConfigs = configs.getOrElse(config, Set.empty) + config
@@ -968,7 +970,7 @@ object Tasks {
 
     // use sbt logging?
     println(
-      projectDescription(currentProject) + "\n" +
+      projectName + "\n" +
       Print.dependencyTree(dependencies0, subRes, printExclusions = true, inverse)
     )
   }

plugin/src/sbt-test/sbt-coursier/from-wrong-url/build.sbt

@@ -0,0 +1,7 @@
+scalaVersion := "2.11.8"
+
+// keeping the default cache policies here
+
+libraryDependencies += "com.chuusai" %% "shapeless" % "2.3.2" from {
+  "https://repo1.maven.org/maven2/com/chuusai/shapeless_2.11/2.3.242/shapeless_2.11-2.3.242.jar"
+}

plugin/src/sbt-test/sbt-coursier/from-wrong-url/project/plugins.sbt

@@ -0,0 +1,11 @@
+{
+  val pluginVersion = sys.props.getOrElse(
+    "plugin.version",
+    throw new RuntimeException(
+      """|The system property 'plugin.version' is not defined.
+         |Specify this property using the scriptedLaunchOpts -D.""".stripMargin
+    )
+  )
+
+  addSbtPlugin("io.get-coursier" % "sbt-coursier" % pluginVersion)
+}

plugin/src/sbt-test/sbt-coursier/from-wrong-url/src/main/scala/Main.scala

@@ -0,0 +1,13 @@
+import java.io.File
+import java.nio.file.Files
+
+import shapeless._
+
+object Main extends App {
+  case class CC(s: String)
+  val cc = CC("OK")
+  val l = Generic[CC].to(cc)
+  val msg = l.head
+
+  Files.write(new File("output").toPath, msg.getBytes("UTF-8"))
+}

plugin/src/sbt-test/sbt-coursier/from-wrong-url/test

@@ -0,0 +1,3 @@
+$ delete output
+> run
+$ exists output