From fe529fb9f48aa52e1caedc40bca225c37556ad9e Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 27 Nov 2016 13:44:14 +0100 Subject: [PATCH] Switch to Java 6 --- .ci/java-6-test.sh | 48 +++++++++ .ci/travis.sh | 8 +- .travis.yml | 6 ++ build.sbt | 6 +- cache/src/main/scala/coursier/Cache.scala | 13 +-- .../scala/coursier/internal/FileUtil.scala | 101 ++++++++++++++++++ .../scala-2.11/coursier/cli/Bootstrap.scala | 3 +- .../scala-2.11/coursier/cli/SparkSubmit.scala | 7 +- .../coursier/cli/spark/Assembly.scala | 6 +- .../src/main/scala-2.10/coursier/Tasks.scala | 5 +- .../scala/coursier/test/CacheFetchTests.scala | 5 +- 11 files changed, 183 insertions(+), 25 deletions(-) create mode 100755 .ci/java-6-test.sh create mode 100644 cache/src/main/scala/coursier/internal/FileUtil.scala diff --git a/.ci/java-6-test.sh b/.ci/java-6-test.sh new file mode 100755 index 000000000..a97d3df5d --- /dev/null +++ b/.ci/java-6-test.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -ev + +# We're not using a jdk6 matrix entry with Travis here as some sources of coursier require Java 7 to compile +# (even though it won't try to call Java 7 specific methods if it detects it runs under Java 6). +# The tests here check that coursier is nonetheless fine when run under Java 6. + +if echo "$TRAVIS_SCALA_VERSION" | grep -q "^2\.11"; then + ~/sbt ++${TRAVIS_SCALA_VERSION} cli/pack + docker run -it --rm \ + -v $(pwd)/cli/target/pack:/opt/coursier \ + -e CI=true \ + openjdk:6-jre \ + /opt/coursier/bin/coursier fetch org.scalacheck::scalacheck:1.13.4 + + docker run -it --rm \ + -v $(pwd)/cli/target/pack:/opt/coursier \ + -e CI=true \ + openjdk:6-jre \ + /opt/coursier/bin/coursier launch --help +fi + +function clean_plugin_sbt() { + mv plugins.sbt plugins.sbt0 + grep -v coursier plugins.sbt0 > plugins.sbt || true + echo ' +addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-SNAPSHOT") + ' >> plugins.sbt +} + +if echo "$TRAVIS_SCALA_VERSION" | grep -q "^2\.10"; then + ~/sbt ++${TRAVIS_SCALA_VERSION} publishLocal + git clone https://github.com/alexarchambault/scalacheck-shapeless.git + cd scalacheck-shapeless + cd project + clean_plugin_sbt + cd project + clean_plugin_sbt + cd ../.. + docker run -it --rm \ + -v $HOME/.ivy2/local:/root/.ivy2/local \ + -v $HOME/sbt:/root/sbt \ + -v $(pwd):/root/project \ + -e CI=true \ + openjdk:6-jre \ + /bin/bash -c "cd /root/project && /root/sbt update" + cd .. +fi diff --git a/.ci/travis.sh b/.ci/travis.sh index ad114ebfe..544e3214b 100755 --- a/.ci/travis.sh +++ b/.ci/travis.sh @@ -50,8 +50,12 @@ fi SBT_COMMANDS="$SBT_COMMANDS tut coreJVM/mimaReportBinaryIssues cache/mimaReportBinaryIssues" +~/sbt ++${TRAVIS_SCALA_VERSION} $SBT_COMMANDS + +.ci/java-6-test.sh + if isNotPr && publish && isMaster; then - SBT_COMMANDS="$SBT_COMMANDS publish" + ~/sbt ++${TRAVIS_SCALA_VERSION} publish fi PUSH_GHPAGES=0 @@ -61,6 +65,4 @@ if isNotPr && publish && isMasterOrDevelop; then fi fi -~/sbt ++${TRAVIS_SCALA_VERSION} $SBT_COMMANDS - # [ "$PUSH_GHPAGES" = 0 ] || "$(dirname "$0")/push-gh-pages.sh" "$TRAVIS_SCALA_VERSION" diff --git a/.travis.yml b/.travis.yml index 4baeb92da..427805666 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,9 +18,15 @@ matrix: - env: TRAVIS_SCALA_VERSION=2.11.8 PUBLISH=1 os: linux jdk: oraclejdk8 + sudo: required + services: + - docker - env: TRAVIS_SCALA_VERSION=2.10.6 PUBLISH=1 os: linux jdk: oraclejdk8 + sudo: required + services: + - docker env: global: - COURSIER_NO_TERM=1 diff --git a/build.sbt b/build.sbt index e91b6d279..b55aa8e2f 100644 --- a/build.sbt +++ b/build.sbt @@ -77,7 +77,7 @@ lazy val scalaVersionAgnosticCommonSettings = Seq( scalacOptions ++= { scalaBinaryVersion.value match { case "2.10" | "2.11" => - Seq("-target:jvm-1.7") + Seq("-target:jvm-1.6") case _ => Seq() } @@ -86,8 +86,8 @@ lazy val scalaVersionAgnosticCommonSettings = Seq( scalaBinaryVersion.value match { case "2.10" | "2.11" => Seq( - "-source", "1.7", - "-target", "1.7" + "-source", "1.6", + "-target", "1.6" ) case _ => Seq() diff --git a/cache/src/main/scala/coursier/Cache.scala b/cache/src/main/scala/coursier/Cache.scala index 3e2089474..48f96d3cc 100644 --- a/cache/src/main/scala/coursier/Cache.scala +++ b/cache/src/main/scala/coursier/Cache.scala @@ -3,13 +3,13 @@ package coursier import java.math.BigInteger import java.net.{ HttpURLConnection, URL, URLConnection, URLStreamHandler, URLStreamHandlerFactory } import java.nio.channels.{ OverlappingFileLockException, FileLock } -import java.nio.file.{ StandardCopyOption, Files => NioFiles } import java.security.MessageDigest import java.util.concurrent.{ ConcurrentHashMap, Executors, ExecutorService } import java.util.regex.Pattern import coursier.core.Authentication import coursier.ivy.IvyRepository +import coursier.internal.FileUtil import coursier.util.Base64.Encoder import scala.annotation.tailrec @@ -583,7 +583,8 @@ object Cache { else if (responseCode(conn) == Some(401)) FileError.Unauthorized(url, realm = realm(conn)).left else { - for (len0 <- Option(conn.getContentLengthLong) if len0 >= 0L) { + // TODO Use the safer getContentLengthLong when switching back to Java >= 7 + for (len0 <- Option(conn.getContentLength) if len0 >= 0L) { val len = len0 + (if (partialDownload) alreadyDownloaded else 0L) logger.foreach(_.downloadLength(url, len, alreadyDownloaded)) } @@ -602,7 +603,7 @@ object Cache { withStructureLock(cache) { file.getParentFile.mkdirs() - NioFiles.move(tmp.toPath, file.toPath, StandardCopyOption.ATOMIC_MOVE) + FileUtil.atomicMove(tmp, file) } for (lastModified <- Option(conn.getLastModified) if lastModified > 0L) @@ -641,7 +642,7 @@ object Cache { Task { if (referenceFileExists) { if (!errFile.exists()) - NioFiles.write(errFile.toPath, "".getBytes("UTF-8")) + FileUtil.write(errFile, "".getBytes("UTF-8")) } ().right[FileError] @@ -796,7 +797,7 @@ object Cache { Task { val sumOpt = parseChecksum( - new String(NioFiles.readAllBytes(sumFile.toPath), "UTF-8") + new String(FileUtil.readAllBytes(sumFile), "UTF-8") ) sumOpt match { @@ -910,7 +911,7 @@ object Cache { def notFound(f: File) = Left(s"${f.getCanonicalPath} not found") def read(f: File) = - try Right(new String(NioFiles.readAllBytes(f.toPath), "UTF-8").stripPrefix(utf8Bom)) + try Right(new String(FileUtil.readAllBytes(f), "UTF-8").stripPrefix(utf8Bom)) catch { case NonFatal(e) => Left(s"Could not read (file:${f.getCanonicalPath}): ${e.getMessage}") diff --git a/cache/src/main/scala/coursier/internal/FileUtil.scala b/cache/src/main/scala/coursier/internal/FileUtil.scala new file mode 100644 index 000000000..5b8bcc876 --- /dev/null +++ b/cache/src/main/scala/coursier/internal/FileUtil.scala @@ -0,0 +1,101 @@ +package coursier.internal + +import java.io._ +import java.util.UUID + +/** Java 6-compatible helpers mimicking NIO */ +object FileUtil { + + private object Java7 { + + import java.nio.file.{ Files, StandardCopyOption } + + def atomicMove(from: File, to: File): Unit = + Files.move(from.toPath, to.toPath, StandardCopyOption.ATOMIC_MOVE) + + def createTempDirectory(prefix: String): File = + Files.createTempDirectory(prefix).toFile + } + + private object Java6 { + def move(from: File, to: File): Unit = + if (!from.renameTo(to)) + throw new IOException(s"Cannot move $from to $to") + + def createTempDirectory(prefix: String): File = { + val tmpBaseDir = new File(sys.props("java.io.tmpdir")) + val tmpDir = new File(tmpBaseDir, s"$prefix-${UUID.randomUUID()}") + tmpDir.mkdirs() + tmpDir + } + } + + private def versionGteq(version: String, to: (Int, Int)): Boolean = + version.split('.').take(2).map(s => scala.util.Try(s.toInt).toOption) match { + case Array(Some(major), Some(minor)) => + Ordering[(Int, Int)].gteq((major, minor), (1, 7)) + case _ => false + } + + // Fine if set several times (if java7Available() is initially called concurrently) + @volatile private var java7AvailableOpt = Option.empty[Boolean] + private def java7Available(): Boolean = + java7AvailableOpt.getOrElse { + val available = sys.props.get("java.version").exists { version => + versionGteq(version, (1, 7)) + } + java7AvailableOpt = Some(available) + available + } + + /** Not guaranteed to be atomic on Java 6 */ + def atomicMove(from: File, to: File): Unit = + if (java7Available()) + Java7.atomicMove(from, to) + else + Java6.move(from, to) + + def write(file: File, bytes: Array[Byte]): Unit = { + var fos: FileOutputStream = null + try { + fos = new FileOutputStream(file) + fos.write(bytes) + fos.close() + } finally { + if (fos != null) fos.close() + } + } + + private def readFully(is: InputStream): Array[Byte] = { + val buffer = new ByteArrayOutputStream + val data = Array.ofDim[Byte](16384) + + var nRead = 0 + while ({ + nRead = is.read(data, 0, data.length) + nRead != -1 + }) + buffer.write(data, 0, nRead) + + buffer.flush() + buffer.toByteArray + } + + def readAllBytes(file: File): Array[Byte] = { + var fis: FileInputStream = null + try { + fis = new FileInputStream(file) + readFully(fis) + } finally { + if (fis != null) + fis.close() + } + } + + def createTempDirectory(prefix: String): File = + if (java7Available()) + Java7.createTempDirectory(prefix) + else + Java6.createTempDirectory(prefix) + +} \ No newline at end of file diff --git a/cli/src/main/scala-2.11/coursier/cli/Bootstrap.scala b/cli/src/main/scala-2.11/coursier/cli/Bootstrap.scala index 4c725813c..3ae44f51a 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Bootstrap.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Bootstrap.scala @@ -9,6 +9,7 @@ import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream} import caseapp._ import coursier.cli.util.Zip +import coursier.internal.FileUtil case class Bootstrap( @Recurse @@ -200,7 +201,7 @@ case class Bootstrap( "exec java -jar " + options.javaOpt.map(s => "'" + s.replace("'", "\\'") + "'").mkString(" ") + " \"$0\" \"$@\"" ).mkString("", "\n", "\n") - try Files.write(output0.toPath, shellPreamble.getBytes("UTF-8") ++ buffer.toByteArray) + try FileUtil.write(output0, shellPreamble.getBytes("UTF-8") ++ buffer.toByteArray) catch { case e: IOException => Console.err.println(s"Error while writing $output0${Option(e.getMessage).fold("")(" (" + _ + ")")}") sys.exit(1) diff --git a/cli/src/main/scala-2.11/coursier/cli/SparkSubmit.scala b/cli/src/main/scala-2.11/coursier/cli/SparkSubmit.scala index 9bb408c21..4df696265 100644 --- a/cli/src/main/scala-2.11/coursier/cli/SparkSubmit.scala +++ b/cli/src/main/scala-2.11/coursier/cli/SparkSubmit.scala @@ -2,12 +2,12 @@ package coursier.cli import java.io.{PrintStream, BufferedReader, File, PipedInputStream, PipedOutputStream, InputStream, InputStreamReader} import java.net.URLClassLoader -import java.nio.file.Files import caseapp._ import coursier.{ Attributes, Dependency } import coursier.cli.spark.{ Assembly, Submit } +import coursier.internal.FileUtil import coursier.util.Parse import scala.util.control.NonFatal @@ -273,9 +273,8 @@ object OutputHelper { lock.synchronized { if (!written) { println(s"Detected YARN app ID $id") - val path = yarnAppFile.toPath - Option(path.getParent).foreach(_.toFile.mkdirs()) - Files.write(path, id.getBytes("UTF-8")) + Option(yarnAppFile.getParentFile).foreach(_.mkdirs()) + FileUtil.write(yarnAppFile, id.getBytes("UTF-8")) written = true } } diff --git a/cli/src/main/scala-2.11/coursier/cli/spark/Assembly.scala b/cli/src/main/scala-2.11/coursier/cli/spark/Assembly.scala index 9f6131d07..6cc1a59fc 100644 --- a/cli/src/main/scala-2.11/coursier/cli/spark/Assembly.scala +++ b/cli/src/main/scala-2.11/coursier/cli/spark/Assembly.scala @@ -2,7 +2,6 @@ package coursier.cli.spark import java.io.{File, FileInputStream, FileOutputStream} import java.math.BigInteger -import java.nio.file.{Files, StandardCopyOption} import java.security.MessageDigest import java.util.jar.{Attributes, JarFile, JarOutputStream, Manifest} import java.util.regex.Pattern @@ -11,6 +10,7 @@ import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream} import coursier.Cache import coursier.cli.{CommonOptions, Helper} import coursier.cli.util.Zip +import coursier.internal.FileUtil import scala.collection.mutable import scalaz.\/- @@ -224,7 +224,7 @@ object Assembly { } val sumOpt = Cache.parseChecksum( - new String(Files.readAllBytes(f.toPath), "UTF-8") + new String(FileUtil.readAllBytes(f), "UTF-8") ) sumOpt match { @@ -273,7 +273,7 @@ object Assembly { val tmpDest = new File(dest.getParentFile, s".${dest.getName}.part") // FIXME Acquire lock on tmpDest Assembly.make(jars, tmpDest, assemblyRules) - Files.move(tmpDest.toPath, dest.toPath, StandardCopyOption.ATOMIC_MOVE) + FileUtil.atomicMove(tmpDest, dest) \/-((dest, jars)) }.leftMap(_.describe).toEither } diff --git a/plugin/src/main/scala-2.10/coursier/Tasks.scala b/plugin/src/main/scala-2.10/coursier/Tasks.scala index 4b5413bdd..0b082a731 100644 --- a/plugin/src/main/scala-2.10/coursier/Tasks.scala +++ b/plugin/src/main/scala-2.10/coursier/Tasks.scala @@ -8,6 +8,7 @@ import coursier.core.{ Authentication, Publication } import coursier.ivy.{ IvyRepository, PropertiesPattern } import coursier.Keys._ import coursier.Structure._ +import coursier.internal.FileUtil import coursier.util.{ Config, Print } import org.apache.ivy.core.module.id.ModuleRevisionId @@ -730,11 +731,11 @@ object Tasks { b += '\n' b ++= printer.format(MakeIvyXml(currentProject)) cacheIvyFile.getParentFile.mkdirs() - Files.write(cacheIvyFile.toPath, b.result().getBytes("UTF-8")) + FileUtil.write(cacheIvyFile, b.result().getBytes("UTF-8")) // Just writing an empty file here... Are these only used? cacheIvyPropertiesFile.getParentFile.mkdirs() - Files.write(cacheIvyPropertiesFile.toPath, "".getBytes("UTF-8")) + FileUtil.write(cacheIvyPropertiesFile, "".getBytes("UTF-8")) } val res = { diff --git a/tests/jvm/src/test/scala/coursier/test/CacheFetchTests.scala b/tests/jvm/src/test/scala/coursier/test/CacheFetchTests.scala index 6525a3c5a..b38b97a37 100644 --- a/tests/jvm/src/test/scala/coursier/test/CacheFetchTests.scala +++ b/tests/jvm/src/test/scala/coursier/test/CacheFetchTests.scala @@ -2,10 +2,9 @@ package coursier package test import java.io.File -import java.nio.file.Files import coursier.cache.protocol.TestprotocolHandler -import coursier.core.Authentication +import coursier.internal.FileUtil import utest._ @@ -15,7 +14,7 @@ object CacheFetchTests extends TestSuite { def check(extraRepo: Repository): Unit = { - val tmpDir = Files.createTempDirectory("coursier-cache-fetch-tests").toFile + val tmpDir = FileUtil.createTempDirectory("coursier-cache-fetch-tests") def cleanTmpDir() = { def delete(f: File): Boolean =