Merge pull request #330 from alexarchambault/topic/develop

Latest developments

cache/src/main/scala/coursier/Cache.scala

@@ -22,6 +22,7 @@ import java.io.{ Serializable => _, _ }
 
 import scala.concurrent.duration.{ Duration, DurationInt }
 import scala.util.Try
+import scala.util.control.NonFatal
 
 trait AuthenticatedURLConnection extends URLConnection {
   def authenticate(authentication: Authentication): Unit
@@ -325,7 +326,10 @@ object Cache {
     def referenceFileExists: Boolean = referenceFileOpt.exists(_.exists())
 
     def urlConn(url0: String) = {
-      val conn = url(url0).openConnection() // FIXME Should this be closed?
+      var conn: URLConnection = null
+
+      try {
+        conn = url(url0).openConnection() // FIXME Should this be closed?
         // Dummy user-agent instead of the default "Java/...",
         // so that we are not returned incomplete/erroneous metadata
         // (Maven 2 compatibility? - happens for snapshot versioning metadata,
@@ -346,6 +350,17 @@ object Cache {
           }
 
         conn
+      } catch {
+        case NonFatal(e) =>
+          if (conn != null)
+            conn match {
+              case conn0: HttpURLConnection =>
+                conn0.getInputStream.close()
+                conn0.disconnect()
+              case _ =>
+            }
+          throw e
+      }
     }
 
 
@@ -369,7 +384,12 @@ object Cache {
     ): EitherT[Task, FileError, Option[Long]] =
       EitherT {
         Task {
-          urlConn(url) match {
+          var conn: URLConnection = null
+
+          try {
+            conn = urlConn(url)
+
+            conn match {
               case c: HttpURLConnection =>
                 logger.foreach(_.checkingUpdates(url, currentLastModifiedOpt))
 
@@ -398,6 +418,14 @@ object Cache {
               case other =>
                 -\/(FileError.DownloadError(s"Cannot do HEAD request with connection $other ($url)"))
             }
+          } finally {
+            if (conn != null)
+              conn match {
+                case conn0: HttpURLConnection =>
+                  conn0.disconnect()
+                case _ =>
+              }
+          }
         }
       }
 
@@ -512,24 +540,27 @@ object Cache {
 
               val alreadyDownloaded = tmp.length()
 
-              val conn0 = urlConn(url)
+              var conn: URLConnection = null
 
-              val (partialDownload, conn) = conn0 match {
+              try {
+                conn = urlConn(url)
+
+                val partialDownload = conn match {
                   case conn0: HttpURLConnection if alreadyDownloaded > 0L =>
                     conn0.setRequestProperty("Range", s"bytes=$alreadyDownloaded-")
 
-                  if (conn0.getResponseCode == partialContentResponseCode) {
+                    (conn0.getResponseCode == partialContentResponseCode) && {
                       val ackRange = Option(conn0.getHeaderField("Content-Range")).getOrElse("")
 
-                    if (ackRange.startsWith(s"bytes $alreadyDownloaded-"))
+                      ackRange.startsWith(s"bytes $alreadyDownloaded-") || {
-                      (true, conn0)
-                    else
                         // unrecognized Content-Range header -> start a new connection with no resume
-                      (false, urlConn(url))
+                        conn0.getInputStream.close()
-                  } else
+                        conn0.disconnect()
-                    (false, conn0)
+                        conn = urlConn(url)
-
+                        false
-                case _ => (false, conn0)
+                      }
+                    }
+                  case _ => false
                 }
 
                 if (responseCode(conn) == Some(404))
@@ -566,6 +597,14 @@ object Cache {
 
                   result.right
                 }
+              } finally {
+                if (conn != null)
+                  conn match {
+                    case conn0: HttpURLConnection =>
+                      conn0.disconnect()
+                    case _ =>
+                  }
+              }
             }
           }
         }
@@ -852,16 +891,58 @@ object Cache {
         pool = pool,
         ttl = ttl
       ).leftMap(_.describe).flatMap { f =>
-        val res = if (!f.isDirectory && f.exists()) {
+
-          Try(new String(NioFiles.readAllBytes(f.toPath), "UTF-8")) match {
+        def notFound(f: File) = Left(s"${f.getCanonicalPath} not found")
-            case scala.util.Success(content) =>
+
-              // stripping any UTF-8 BOM if any, see
+        def read(f: File) =
-              // https://github.com/alexarchambault/coursier/issues/316 and the corresponding test
+          try Right(new String(NioFiles.readAllBytes(f.toPath), "UTF-8").stripPrefix(utf8Bom))
-              Right(content.stripPrefix(utf8Bom))
+          catch {
-            case scala.util.Failure(e) =>
+            case NonFatal(e) =>
               Left(s"Could not read (file:${f.getCanonicalPath}): ${e.getMessage}")
           }
-        } else Left(s"Could not read (file:${f.getCanonicalPath}) (isFile:${!f.isDirectory}) (exists:${f.exists()})")
+
+        val res = if (f.exists()) {
+          if (f.isDirectory) {
+            if (artifact.url.startsWith("file:")) {
+
+              val elements = f.listFiles().map { c =>
+                val name = c.getName
+                val name0 = if (c.isDirectory)
+                  name + "/"
+                else
+                  name
+
+                s"""<li><a href="$name0">$name0</a></li>"""
+              }.mkString
+
+              val page =
+                s"""<!DOCTYPE html>
+                   |<html>
+                   |<head></head>
+                   |<body>
+                   |<ul>
+                   |$elements
+                   |</ul>
+                   |</body>
+                   |</html>
+                 """.stripMargin
+
+              Right(page)
+            } else {
+              val f0 = new File(f, ".directory")
+
+              if (f0.exists()) {
+                if (f0.isDirectory)
+                  Left(s"Woops: ${f.getCanonicalPath} is a directory")
+                else
+                  read(f0)
+              } else
+                notFound(f0)
+            }
+          } else
+            read(f)
+        } else
+          notFound(f)
 
         EitherT.fromEither(Task.now[Either[String, String]](res))
       }

core/js/src/main/scala/coursier/core/compatibility/package.scala

@@ -93,7 +93,12 @@ package object compatibility {
   def encodeURIComponent(s: String): String =
     g.encodeURIComponent(s).asInstanceOf[String]
 
-  def listWebPageSubDirectories(page: String): Seq[String] = {
+  def listWebPageSubDirectories(url: String, page: String): Seq[String] = {
+    // TODO
+    ???
+  }
+
+  def listWebPageFiles(url: String, page: String): Seq[String] = {
     // TODO
     ???
   }

core/jvm/src/main/scala/coursier/core/compatibility/package.scala

@@ -56,17 +56,25 @@ package object compatibility {
   def encodeURIComponent(s: String): String =
     new java.net.URI(null, null, null, -1, s, null, null) .toASCIIString
 
-  def listWebPageSubDirectories(page: String): Seq[String] =
+  def listWebPageDirectoryElements(url: String, page: String, directories: Boolean): Seq[String] =
     Jsoup.parse(page)
-      .select("a[href~=[^/]*/]")
+      .select("a")
       .asScala
       .toVector
-      .map { elem =>
+      .map(_.attr("href"))
+      .collect {
+        case elem if elem.nonEmpty && elem.endsWith("/") == directories =>
           elem
-          .attr("href")
-          .stripPrefix(":") // bintray typically prepends these
             .stripSuffix("/")
+            .stripPrefix(url)
+            .stripPrefix(":") // bintray typically prepends these
       }
-      .filter(n => n != "." && n != "..")
+      .filter(n => !n.contains("/") && n != "." && n != "..")
+
+  def listWebPageSubDirectories(url: String, page: String): Seq[String] =
+    listWebPageDirectoryElements(url, page, directories = true)
+
+  def listWebPageFiles(url: String, page: String): Seq[String] =
+    listWebPageDirectoryElements(url, page, directories = false)
 
 }

core/shared/src/main/scala/coursier/ivy/IvyRepository.scala

@@ -147,8 +147,8 @@ case class IvyRepository(
                 s"Don't know how to list revisions of ${metadataPattern.string}".left
             }
 
-            def fromWebPage(s: String) = {
+            def fromWebPage(url: String, s: String) = {
-              val subDirs = coursier.core.compatibility.listWebPageSubDirectories(s)
+              val subDirs = coursier.core.compatibility.listWebPageSubDirectories(url, s)
               val versions = subDirs.map(Parse.version).collect { case Some(v) => v }
               val versionsInItv = versions.filter(itv.contains)
 
@@ -173,7 +173,7 @@ case class IvyRepository(
             for {
               url <- EitherT(F.point(listingUrl))
               s <- fetch(artifactFor(url))
-              res <- fromWebPage(s)
+              res <- fromWebPage(url, s)
             } yield res
         }
     }

plugin/src/main/scala-2.10/coursier/FallbackDependenciesRepository.scala

@@ -8,7 +8,7 @@ case class FallbackDependenciesRepository(
   fallbacks: Map[(Module, String), (URL, Boolean)]
 ) extends Repository {
 
-  private val source = new Artifact.Source {
+  private val source: Artifact.Source = new Artifact.Source {
     def artifacts(
       dependency: Dependency,
       project: Project,
@@ -35,6 +35,21 @@ case class FallbackDependenciesRepository(
         EitherT.left(F.point("No fallback URL found"))
 
       case Some((url, _)) =>
+
+        val urlStr = url.toExternalForm
+        val idx = urlStr.lastIndexOf('/')
+
+        if (idx < 0 || urlStr.endsWith("/"))
+          EitherT.left(F.point(s"$url doesn't point to a file"))
+        else {
+          val (dirUrlStr, fileName) = urlStr.splitAt(idx + 1)
+
+          fetch(Artifact(dirUrlStr, Map.empty, Map.empty, Attributes("", ""), changing = true, None)).flatMap { listing =>
+
+            val files = coursier.core.compatibility.listWebPageFiles(dirUrlStr, listing)
+
+            if (files.contains(fileName)) {
+
               val proj = Project(
                 module,
                 version,
@@ -52,5 +67,9 @@ case class FallbackDependenciesRepository(
               )
 
               EitherT.right(F.point((source, proj)))
+            } else
+              EitherT.left(F.point(s"$fileName not found under $dirUrlStr"))
+          }
+        }
     }
 }

plugin/src/main/scala-2.10/coursier/Tasks.scala

@@ -276,9 +276,6 @@ object Tasks {
       Module(scalaOrganization, "scalap") -> scalaVersion
     )
 
-  private def projectDescription(project: Project) =
-    s"${project.module.organization}:${project.module.name}:${project.version}"
-
   private def createLogger() = new TermDisplay(new OutputStreamWriter(System.err))
 
   private lazy val globalPluginPattern = {
@@ -325,6 +322,8 @@ object Tasks {
 
       lazy val cm = coursierSbtClassifiersModule.value
 
+      lazy val projectName = thisProjectRef.value.project
+
       val (currentProject, fallbackDependencies) =
         if (sbtClassifiers) {
           val sv = scalaVersion.value
@@ -614,8 +613,7 @@ object Tasks {
 
           if (verbosityLevel >= 0)
             log.info(
-              s"Updating ${projectDescription(currentProject)}" +
+              s"Updating $projectName" + (if (sbtClassifiers) " (sbt classifiers)" else "")
-                (if (sbtClassifiers) " (sbt classifiers)" else "")
             )
           if (verbosityLevel >= 2)
             for (depRepr <- depsRepr(currentProject.dependencies))
@@ -663,7 +661,7 @@ object Tasks {
           }
 
           if (verbosityLevel >= 0)
-            log.info(s"Resolved ${projectDescription(currentProject)} dependencies")
+            log.info(s"Resolved $projectName dependencies")
 
           res
         } finally {
@@ -705,6 +703,8 @@ object Tasks {
 
       lazy val cm = coursierSbtClassifiersModule.value
 
+      lazy val projectName = thisProjectRef.value.project
+
       val currentProject =
         if (sbtClassifiers) {
           val sv = scalaVersion.value
@@ -834,7 +834,7 @@ object Tasks {
 
           if (verbosityLevel >= 0)
             log.info(
-              s"Fetching artifacts of ${projectDescription(currentProject)}" +
+              s"Fetching artifacts of $projectName" +
                 (if (sbtClassifiers) " (sbt classifiers)" else "")
             )
 
@@ -850,7 +850,7 @@ object Tasks {
 
           if (verbosityLevel >= 0)
             log.info(
-              s"Fetched artifacts of ${projectDescription(currentProject)}" +
+              s"Fetched artifacts of $projectName" +
                 (if (sbtClassifiers) " (sbt classifiers)" else "")
             )
 
@@ -925,6 +925,8 @@ object Tasks {
     ignoreArtifactErrors: Boolean = false
   ) = Def.task {
 
+    lazy val projectName = thisProjectRef.value.project
+
     val currentProject =
       if (sbtClassifiers) {
         val cm = coursierSbtClassifiersModule.value
@@ -953,7 +955,7 @@ object Tasks {
         coursierResolution
     }.value
 
-    val config = classpathConfiguration.value.name
+    val config = configuration.value.name
     val configs = coursierConfigurations.value
 
     val includedConfigs = configs.getOrElse(config, Set.empty) + config
@@ -968,7 +970,7 @@ object Tasks {
 
     // use sbt logging?
     println(
-      projectDescription(currentProject) + "\n" +
+      projectName + "\n" +
       Print.dependencyTree(dependencies0, subRes, printExclusions = true, inverse)
     )
   }

plugin/src/sbt-test/sbt-coursier/from-wrong-url/build.sbt

@@ -0,0 +1,7 @@
+scalaVersion := "2.11.8"
+
+// keeping the default cache policies here
+
+libraryDependencies += "com.chuusai" %% "shapeless" % "2.3.2" from {
+  "https://repo1.maven.org/maven2/com/chuusai/shapeless_2.11/2.3.242/shapeless_2.11-2.3.242.jar"
+}

plugin/src/sbt-test/sbt-coursier/from-wrong-url/project/plugins.sbt

@@ -0,0 +1,11 @@
+{
+  val pluginVersion = sys.props.getOrElse(
+    "plugin.version",
+    throw new RuntimeException(
+      """|The system property 'plugin.version' is not defined.
+         |Specify this property using the scriptedLaunchOpts -D.""".stripMargin
+    )
+  )
+
+  addSbtPlugin("io.get-coursier" % "sbt-coursier" % pluginVersion)
+}

plugin/src/sbt-test/sbt-coursier/from-wrong-url/src/main/scala/Main.scala

@@ -0,0 +1,13 @@
+import java.io.File
+import java.nio.file.Files
+
+import shapeless._
+
+object Main extends App {
+  case class CC(s: String)
+  val cc = CC("OK")
+  val l = Generic[CC].to(cc)
+  val msg = l.head
+
+  Files.write(new File("output").toPath, msg.getBytes("UTF-8"))
+}

plugin/src/sbt-test/sbt-coursier/from-wrong-url/test

@@ -0,0 +1,3 @@
+$ delete output
+> run
+$ exists output