Update gradle-download-plugin and use some of its new features

We now download and verify checksums as a single task, rather than as two separate tasks. This simplifies other task dependencies, since we no longer have a checksum-verified "stamp" file separate from the download itself. Unfortunately the combined task now has a significant amount of repeated boilerplate. I'm hoping to refactor that all out into a custom task class, but haven't yet figured out the details: <michel-kraemer/gradle-download-task#108>. We now also use ETags to be smarter about when a fresh download is or is not actually needed. I think there are still opportunities for improved caching here, but this is a step in the right direction.
liblit · Mar 18, 2018 · 010e4e1 · 010e4e1
1 parent 5c3149c
commit 010e4e1
Show file tree

Hide file tree

Showing 9 changed files with 173 additions and 146 deletions.
diff --git a/build.gradle b/build.gradle
@@ -6,7 +6,7 @@
 plugins {
 	id 'com.diffplug.gradle.p2.asmaven' version '3.10.0'
 	id 'com.github.hauner.jarTest' version '1.0.1' apply false
-	id 'de.undercouch.download' apply false
+	id 'de.undercouch.download'
 	id 'nebula.lint' version '8.3.1'
 	id "nebula.source-jar" version '7.0.1' apply false
 }

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
@@ -3,5 +3,5 @@ repositories {
 }
 
 dependencies {
-	compile 'de.undercouch:gradle-download-task:3.3.0'
+	compile 'de.undercouch:gradle-download-task:3.4.1'
 }
diff --git a/buildSrc/src/main/groovy/stamp.groovy b/buildSrc/src/main/groovy/stamp.groovy
diff --git a/com.ibm.wala.cast.java.test.data/build.gradle b/com.ibm.wala.cast.java.test.data/build.gradle
@@ -6,23 +6,26 @@ sourceSets.test.java.srcDirs = ['src']
 //  download JLex
 //
 
-def versionedArchive = 'ajaxslt-0.8.1'
-def packedArchive = "${versionedArchive}.tar.gz"
-
-task downloadJLex(type: de.undercouch.gradle.tasks.download.Download) {
-	src 'http://www.cs.princeton.edu/~appel/modern/java/JLex/current/Main.java'
-	dest 'src/JLex/Main.java'
-	overwrite false
+task downloadJLex {
+	outputs.file 'src/JLex/Main.java'
+	doLast {
+		download {
+			src 'http://www.cs.princeton.edu/~appel/modern/java/JLex/current/Main.java'
+			dest outputs.files.singleFile
+			overwrite true
+			onlyIfModified true
+			useETag true
+		}
+		verifyChecksum {
+			src outputs.files.singleFile
+			checksum 'fe0cff5db3e2f0f5d67a153cf6c783af'
+		}
+	}
 }
 
 task cleanDownloadJLex(type: Delete) {
-	delete downloadJLex.dest.parent
-}
-
-task verifyJLex(type: VerifyWithStamp, dependsOn: downloadJLex) {
-	src downloadJLex.dest
-	checksum 'fe0cff5db3e2f0f5d67a153cf6c783af'
+	delete files(downloadJLex).singleFile.parent
 }
 
-compileTestJava.dependsOn verifyJLex
+compileTestJava.dependsOn downloadJLex
 clean.dependsOn cleanDownloadJLex
diff --git a/com.ibm.wala.cast.js.nodejs/build.gradle b/com.ibm.wala.cast.js.nodejs/build.gradle
@@ -10,19 +10,25 @@ dependencies {
 		)
 }
 
-task downloadNodeJS(type: de.undercouch.gradle.tasks.download.Download) {
-	src 'https://api.github.com/repos/nodejs/node/zipball/0a604e92e258c5ee2752d763e50721e35053f135'
-	dest new File(temporaryDir, 'nodejs.zip')
-	overwrite false
-}
-
-task verifyNodeJS(type: VerifyWithStamp, dependsOn: downloadNodeJS) {
-	src files(downloadNodeJS).singleFile
-	checksum '33c5ba7a5d45644e70d268d8ad3e57df'
+task downloadNodeJS {
+	outputs.file "$temporaryDir/nodejs.zip"
+	doLast {
+		download {
+			src 'https://api.github.com/repos/nodejs/node/zipball/0a604e92e258c5ee2752d763e50721e35053f135'
+			dest outputs.files.singleFile
+			overwrite true
+			onlyIfModified true
+			useETag true
+		}
+		verifyChecksum {
+			src outputs.files.singleFile
+			checksum '33c5ba7a5d45644e70d268d8ad3e57df'
+		}
+	}
 }
 
-task unpackNodeJSLib(type: Copy, dependsOn: verifyNodeJS) {
-	from(zipTree(verifyNodeJS.src)) {
+task unpackNodeJSLib(type: Copy, dependsOn: downloadNodeJS) {
+	from(zipTree(files(downloadNodeJS).singleFile)) {
 		include 'nodejs-node-0a604e9/lib/*.js'
 		eachFile {
 			relativePath new RelativePath(!directory, relativePath.lastName)

diff --git a/com.ibm.wala.cast.js.test.data/build.gradle b/com.ibm.wala.cast.js.test.data/build.gradle
@@ -1,22 +1,26 @@
 apply plugin: 'base'
 
-task downloadAjaxslt(type: de.undercouch.gradle.tasks.download.Download) {
+task downloadAjaxslt {
 	def version = '0.8.1'
-	ext {
-		versionedArchive = "ajaxslt-$version"
+	def versionedArchive = "ajaxslt-${version}.tar.gz"
+	outputs.file "$temporaryDir/$versionedArchive"
+	doLast {
+		download {
+			src "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/ajaxslt/$versionedArchive"
+			dest outputs.files.singleFile
+			overwrite true
+			onlyIfModified true
+			useETag true
+		}
+		verifyChecksum {
+			src outputs.files.singleFile
+			checksum 'c995abe3310a401bb4db7f28a6409756'
+		}
 	}
-	src "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/ajaxslt/${versionedArchive}.tar.gz"
-	dest temporaryDir
-	overwrite false
-}
-
-task verifyAjaxslt(type: VerifyWithStamp, dependsOn: downloadAjaxslt) {
-	src files(downloadAjaxslt).singleFile
-	checksum 'c995abe3310a401bb4db7f28a6409756'
 }
 
-task unpackAjaxslt(type: Sync, dependsOn: verifyAjaxslt) {
-	from(tarTree(verifyAjaxslt.src)) {
+task unpackAjaxslt(type: Sync, dependsOn: downloadAjaxslt) {
+	from(tarTree(files(downloadAjaxslt).singleFile)) {
 		eachFile {
 			def newSegments = relativePath.segments[1 .. -1] as String[]
 			relativePath new RelativePath(!directory, newSegments)

diff --git a/com.ibm.wala.core.testdata/build.gradle b/com.ibm.wala.core.testdata/build.gradle
@@ -5,40 +5,42 @@ eclipse.project.natures 'org.eclipse.pde.PluginNature'
 sourceSets.test.java.srcDirs = ['src']
 
 
-////////////////////////////////////////////////////////////////////////
-//
-//  download and/or create extra bundled jar archives
-//
-
-import de.undercouch.gradle.tasks.download.*
-
-
 ////////////////////////////////////////////////////////////////////////
 //
 //  download and extract "bcel-5.2.jar"
 //
 
-task downloadBcel(type: Download) {
-	src 'http://archive.apache.org/dist/jakarta/bcel/binaries/bcel-5.2.tar.gz'
-	dest temporaryDir
-	overwrite false
-}
-
-task verifyBcel(type: VerifyWithStamp, dependsOn: downloadBcel) {
-	src files(downloadBcel).singleFile
-	checksum '19bffd7f217b0eae415f1ef87af2f0bc'
+task downloadBcel {
+	ext {
+		version = '5.2'
+	}
+	def archive = "bcel-${version}.tar.gz"
+	outputs.file "$temporaryDir/$archive"
+	doLast {
+		download {
+			src "http://archive.apache.org/dist/jakarta/bcel/binaries/$archive"
+			dest outputs.files.singleFile
+			overwrite true
+			onlyIfModified true
+			useETag false // server does not provide ETags
+		}
+		verifyChecksum {
+			src outputs.files.singleFile
+			checksum '19bffd7f217b0eae415f1ef87af2f0bc'
+		}
+	}
 }
 
-task extractBcel(type: Copy, dependsOn: verifyBcel) {
-	from(tarTree(verifyBcel.src)) {
-		include 'bcel-5.2/bcel-5.2.jar'
+task extractBcel(type: Copy, dependsOn: downloadBcel) {
+	from(tarTree(files(downloadBcel).singleFile)) {
+		include "bcel-${downloadBcel.version}/bcel-${downloadBcel.version}.jar"
 		eachFile {
 			relativePath new RelativePath(!directory, relativePath.lastName)
 		}
 	}
 	into projectDir
 	includeEmptyDirs false
-	outputs.files 'bcel-5.2.jar'
+	outputs.file "bcel-${downloadBcel.version}.jar"
 }
 
 task cleanExtractBcel(type: Delete) {
@@ -54,21 +56,28 @@ clean.dependsOn cleanExtractBcel
 //  download "java-cup-11a.jar"
 //
 
-task downloadJavaCup(type: Download) {
-	src 'http://www2.cs.tum.edu/projects/cup/java-cup-11a.jar'
-	dest projectDir
-	overwrite false
+task downloadJavaCup {
+	def archive = 'java-cup-11a.jar'
+	outputs.file "$projectDir/$archive"
+	doLast {
+		download {
+			src "http://www2.cs.tum.edu/projects/cup/$archive"
+			dest outputs.files.singleFile
+			overwrite true
+			onlyIfModified true
+			useETag true
+		}
+		verifyChecksum {
+			src outputs.files.singleFile
+			checksum '2bda8c40abd0cbc295d3038643d6e4ec'
+		}
+	}
 }
 
 task cleanDownloadJavaCup(type: Delete) {
 	delete downloadJavaCup
 }
 
-task verifyJavaCup(type: VerifyWithStamp, dependsOn: downloadJavaCup) {
-	src files(downloadJavaCup).singleFile
-	checksum '2bda8c40abd0cbc295d3038643d6e4ec'
-}
-
 clean.dependsOn cleanDownloadJavaCup
 
 
@@ -97,23 +106,28 @@ clean.dependsOn cleanCollectJLex
 //  generate "hello_hash.jar"
 //
 
-task downloadOcamlJava(type: Download) {
+task downloadOcamlJava {
 	def version = '2.0-alpha1'
-	ext {
-		basename = "ocamljava-$version"
+	ext.basename = "ocamljava-$version"
+	def archive = "${basename}.tar.gz"
+	outputs.file "$temporaryDir/$archive"
+	doLast {
+		download {
+			src "http://www.ocamljava.org/downloads/download.php?version=$version-bin"
+			dest outputs.files.singleFile
+			overwrite true
+			onlyIfModified true
+			useETag true
+		}
+		verifyChecksum {
+			src outputs.files.singleFile
+			checksum '45feec6e3889f5073a39c2c4c84878d1'
+		}
 	}
-	src "http://www.ocamljava.org/downloads/download.php?version=$version-bin"
-	dest new File(temporaryDir, "${basename}.tar.gz")
-	overwrite false
-}
-
-task verifyOcamlJava(type: VerifyWithStamp, dependsOn: downloadOcamlJava) {
-	src files(downloadOcamlJava).singleFile
-	checksum '45feec6e3889f5073a39c2c4c84878d1'
 }
 
-task unpackOcamlJava(type: Sync, dependsOn: verifyOcamlJava) {
-	from tarTree(verifyOcamlJava.src)
+task unpackOcamlJava(type: Sync, dependsOn: downloadOcamlJava) {
+	from tarTree(files(downloadOcamlJava).singleFile)
 	into temporaryDir
 }
 
@@ -191,8 +205,8 @@ afterEclipseBuildshipImport {
 		collectJLex,
 		collectTestData,
 		collectTestDataA,
+		downloadJavaCup,
 		extractBcel,
 		generateHelloHash,
-		verifyJavaCup,
 		)
 }
diff --git a/com.ibm.wala.core.tests/build.gradle b/com.ibm.wala.core.tests/build.gradle
@@ -25,7 +25,6 @@ processTestResources {
 	def testdata = project(':com.ibm.wala.core.testdata')
 	dependsOn testdata.compileTestJava
 	dependsOn testdata.extractBcel
-	dependsOn testdata.verifyJavaCup
 
 	from testdata.collectJLex
 	from testdata.collectTestData