diff --git a/cache/src/main/scala/coursier/Cache.scala b/cache/src/main/scala/coursier/Cache.scala index 76153fde3..3652a0970 100644 --- a/cache/src/main/scala/coursier/Cache.scala +++ b/cache/src/main/scala/coursier/Cache.scala @@ -442,6 +442,16 @@ object Cache { Nondeterminism[Task].gather(tasks) } + def parseChecksum(content: String): Option[String] = { + // matches md5 or sha1 + val pattern = "^[0-9a-f]{32}([0-9a-f]{8})?" + content + .linesIterator + .toStream + .map(_.toLowerCase.replaceAll("\\s", "")) + .find(_.matches(pattern)) + } + def validateChecksum( artifact: Artifact, sumType: String, @@ -459,12 +469,9 @@ object Cache { artifact0.checksumUrls.get(sumType) match { case Some(sumFile) => Task { - val sum = new String(NioFiles.readAllBytes(new File(sumFile).toPath), "UTF-8") - .linesIterator - .toStream - .headOption - .mkString - .takeWhile(!_.isSpaceChar) + val sum = parseChecksum( + new String(NioFiles.readAllBytes(new File(sumFile).toPath), "UTF-8")) + .getOrElse("") val f = new File(artifact0.url) val md = MessageDigest.getInstance(sumType) @@ -489,7 +496,7 @@ object Cache { res.flatMap { _ => val digest = md.digest() - val calculatedSum = f"${BigInt(1, digest)}%040x" + val calculatedSum = f"${BigInt(1, digest)}%x" if (sum == calculatedSum) \/-(()) diff --git a/tests/jvm/src/test/resources/checksums/abc.com/test.jar b/tests/jvm/src/test/resources/checksums/abc.com/test.jar new file mode 100644 index 000000000..48b9ad31d --- /dev/null +++ b/tests/jvm/src/test/resources/checksums/abc.com/test.jar @@ -0,0 +1,582 @@ + + + + spark-parent + org.apache.spark + 1.2.0 + + 4.0.0 + org.apache.spark + spark-core_2.11 + Spark Project Core + http://spark.apache.org/ + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes + + + src/main/resources + + + ../python + + pyspark/*.py + + + + ../python/build + + py4j/*.py + + + + + + org.scalatest + scalatest-maven-plugin + + + test + + test + + + + + + maven-antrun-plugin + + + generate-resources + + run + + + + + + + + + + + maven-clean-plugin + + + + ${basedir}/../python/build + + + true + + + + maven-shade-plugin + + + package + + shade + + + false + + + com.google.guava:guava + + + + + com.google.guava:guava + + com/google/common/base/Absent* + com/google/common/base/Optional* + com/google/common/base/Present* + + + + + + + + + maven-dependency-plugin + + + copy-dependencies + package + + copy-dependencies + + + ${project.build.directory} + false + false + true + true + guava + true + + + + + + + + + com.twitter + chill_2.11 + 0.5.0 + compile + + + asm + org.ow2.asm + + + asm-commons + org.ow2.asm + + + + + com.twitter + chill-java + 0.5.0 + compile + + + asm + org.ow2.asm + + + asm-commons + org.ow2.asm + + + + + org.apache.hadoop + hadoop-client + 2.2.0 + compile + + + servlet-api + javax.servlet + + + + + org.apache.spark + spark-network-common_2.11 + 1.2.0 + compile + + + org.apache.spark + spark-network-shuffle_2.11 + 1.2.0 + compile + + + net.java.dev.jets3t + jets3t + 0.7.1 + compile + + + commons-logging + commons-logging + + + + + org.apache.curator + curator-recipes + 2.4.0 + compile + + + netty + org.jboss.netty + + + + + org.eclipse.jetty + jetty-plus + 8.1.14.v20131031 + compile + + + org.eclipse.jetty + jetty-security + 8.1.14.v20131031 + compile + + + org.eclipse.jetty + jetty-util + 8.1.14.v20131031 + compile + + + org.eclipse.jetty + jetty-server + 8.1.14.v20131031 + compile + + + org.apache.commons + commons-lang3 + 3.3.2 + compile + + + org.apache.commons + commons-math3 + 3.1.1 + compile + + + com.google.code.findbugs + jsr305 + 1.3.9 + compile + + + org.slf4j + slf4j-api + 1.7.5 + compile + + + org.slf4j + jul-to-slf4j + 1.7.5 + compile + + + org.slf4j + jcl-over-slf4j + 1.7.5 + compile + + + log4j + log4j + 1.2.17 + compile + + + org.slf4j + slf4j-log4j12 + 1.7.5 + compile + + + com.ning + compress-lzf + 1.0.0 + compile + + + org.xerial.snappy + snappy-java + 1.1.1.6 + compile + + + net.jpountz.lz4 + lz4 + 1.2.0 + compile + + + org.roaringbitmap + RoaringBitmap + 0.4.5 + compile + + + commons-net + commons-net + 2.2 + compile + + + org.spark-project.akka + akka-remote_2.11 + 2.3.4-spark + compile + + + org.spark-project.akka + akka-slf4j_2.11 + 2.3.4-spark + compile + + + org.spark-project.akka + akka-testkit_2.11 + 2.3.4-spark + test + + + org.scala-lang + scala-library + 2.11.2 + compile + + + org.json4s + json4s-jackson_2.11 + 3.2.10 + compile + + + org.apache.mesos + mesos + 0.18.1 + shaded-protobuf + compile + + + protobuf-java + com.google.protobuf + + + + + io.netty + netty-all + 4.0.23.Final + compile + + + com.clearspring.analytics + stream + 2.7.0 + compile + + + fastutil + it.unimi.dsi + + + + + com.codahale.metrics + metrics-core + 3.0.0 + compile + + + com.codahale.metrics + metrics-jvm + 3.0.0 + compile + + + com.codahale.metrics + metrics-json + 3.0.0 + compile + + + com.codahale.metrics + metrics-graphite + 3.0.0 + compile + + + org.apache.derby + derby + 10.10.1.1 + test + + + org.tachyonproject + tachyon-client + 0.5.0 + compile + + + hadoop-client + org.apache.hadoop + + + curator-recipes + org.apache.curator + + + jetty-jsp + org.eclipse.jetty + + + jetty-webapp + org.eclipse.jetty + + + jetty-server + org.eclipse.jetty + + + jetty-servlet + org.eclipse.jetty + + + junit + junit + + + powermock-module-junit4 + org.powermock + + + powermock-api-mockito + org.powermock + + + curator-test + org.apache.curator + + + + + org.seleniumhq.selenium + selenium-java + 2.42.2 + test + + + selenium-chrome-driver + org.seleniumhq.selenium + + + selenium-htmlunit-driver + org.seleniumhq.selenium + + + selenium-firefox-driver + org.seleniumhq.selenium + + + selenium-ie-driver + org.seleniumhq.selenium + + + selenium-safari-driver + org.seleniumhq.selenium + + + selenium-support + org.seleniumhq.selenium + + + webbit + org.webbitserver + + + + + org.scalatest + scalatest_2.11 + 2.2.1 + test + + + org.mockito + mockito-all + 1.9.0 + test + + + org.scalacheck + scalacheck_2.11 + 1.11.3 + test + + + test-interface + org.scala-sbt + + + + + org.easymock + easymockclassextension + 3.1 + test + + + easymock + org.easymock + + + + + asm + asm + 3.3.1 + test + + + junit + junit + 4.10 + test + + + hamcrest-core + org.hamcrest + + + + + com.novocode + junit-interface + 0.10 + test + + + junit-dep + junit + + + test-interface + org.scala-tools.testing + + + + + org.spark-project + pyrolite + 2.0.1 + compile + + + net.sf.py4j + py4j + 0.8.2.1 + compile + + + org.spark-project.spark + unused + 1.0.0 + compile + + + org.codehaus.groovy + groovy-all + 2.3.7 + provided + + + + core + + + diff --git a/tests/jvm/src/test/resources/checksums/abc.com/test.jar.md5 b/tests/jvm/src/test/resources/checksums/abc.com/test.jar.md5 new file mode 100644 index 000000000..d07fafcf3 --- /dev/null +++ b/tests/jvm/src/test/resources/checksums/abc.com/test.jar.md5 @@ -0,0 +1 @@ +4c4b130f5f49e2a81c6c44d658c21d9b diff --git a/tests/jvm/src/test/resources/checksums/abc.com/test.jar.sha1 b/tests/jvm/src/test/resources/checksums/abc.com/test.jar.sha1 new file mode 100644 index 000000000..cb8874abb --- /dev/null +++ b/tests/jvm/src/test/resources/checksums/abc.com/test.jar.sha1 @@ -0,0 +1 @@ +563042a54b97e31af4529ea0db79ba2c4c2bb6cc \ No newline at end of file diff --git a/tests/jvm/src/test/scala/coursier/test/ChecksumTests.scala b/tests/jvm/src/test/scala/coursier/test/ChecksumTests.scala new file mode 100644 index 000000000..d542aaf16 --- /dev/null +++ b/tests/jvm/src/test/scala/coursier/test/ChecksumTests.scala @@ -0,0 +1,73 @@ +package coursier +package test + +import java.io.File +import java.util.concurrent.Executors + +import utest._ + + +object ChecksumTests extends TestSuite { + val tests = TestSuite { + + val junkSha1 = + """./spark-core_2.11/1.2.0/spark-core_2.11-1.2.0.pom: +5630 42A5 4B97 E31A F452 9EA0 DB79 BA2C 4C2B B6CC + """.stripMargin + + val cleanSha1 = + """563042a54b97e31af4529ea0db79ba2c4c2bb6cc + """.stripMargin + + val checksum = Some("563042a54b97e31af4529ea0db79ba2c4c2bb6cc") + + 'parseJunkChecksum{ + assert(Cache.parseChecksum(junkSha1) == checksum) + } + + 'parseCleanChecksum{ + assert(Cache.parseChecksum(cleanSha1) == checksum) + } + + + val artifact = Artifact( + "http://abc.com/test.jar", + Map ( + "MD5" -> "http://abc.com/test.jar.md5", + "SHA-1" -> "http://abc.com/test.jar.sha1" + ), + Map.empty, + Attributes("jar"), + changing = false + ) + + val pool = Executors.newFixedThreadPool(1) + val cachePath = getClass.getResource("/checksums").getPath + + val cache = Seq( + "http://" -> new File(cachePath), + "https://" -> new File(cachePath) + ) + + 'sha1{ + val res = Cache.validateChecksum( + artifact, + "SHA-1", + cache, + pool + ) + assert(res.run.run.isRight) + } + + 'md5{ + val res = Cache.validateChecksum( + artifact, + "MD5", + cache, + pool + ) + + assert(res.run.run.isRight) + } + } +} \ No newline at end of file