ENH: Update SampleData to use checksum

This commit ensures a file is re-download if a previous download is incomplete or if the the expected checksum is updated. See https://discourse.slicer.org/t/improving-testing-data-management-for-self-test/5014 svn-url: http://viewvc.slicer.org/viewvc.cgi/Slicer4?view=revision&revision=28290 git-svn-id: http://svn.slicer.org/Slicer4/trunk@28290 3bd1e089-480b-0410-8dfb-8563597acbee
Slicer · Jun 12, 2019 · 80870c3 · 80870c3
1 parent 1df415f
commit 80870c3
Show file tree

Hide file tree

Showing 25 changed files with 302 additions and 90 deletions.
diff --git a/Applications/SlicerApp/Testing/Python/AtlasTests.py b/Applications/SlicerApp/Testing/Python/AtlasTests.py
@@ -154,7 +154,8 @@ def test_AbdominalAtlasTest(self):
     downloads = {
       'fileNames': 'Abdominal_Atlas_2012.mrb',
       'loadFiles': True,
-      'uris': 'http://slicer.kitware.com/midas3/download?items=8301'
+      'uris': 'http://slicer.kitware.com/midas3/download?items=8301',
+      'checksums': 'SHA256:5d315abf7d303326669c6075f9eea927eeda2e531a5b1662cfa505806cb498ea',
       }
     self.perform_AtlasTest(downloads,'I')
 
@@ -163,7 +164,8 @@ def test_BrainAtlasTest(self):
     downloads = {
       'fileNames': 'BrainAtlas2012.mrb',
       'loadFiles': True,
-      'uris': 'http://slicer.kitware.com/midas3/download?items=10937'
+      'uris': 'http://slicer.kitware.com/midas3/download?items=10937',
+      'checksums': 'SHA256:688ebcc6f45989795be2bcdc6b8b5bfc461f1656d677ed3ddef8c313532687f1',
       }
     self.perform_AtlasTest(downloads,'A1_grayT1')
 
@@ -172,7 +174,8 @@ def test_KneeAtlasTest(self):
     downloads = {
       'fileNames': 'KneeAtlas2012.mrb',
       'loadFiles': True,
-      'uris': 'http://slicer.kitware.com/midas3/download?items=9912'
+      'uris': 'http://slicer.kitware.com/midas3/download?items=9912',
+      'checksums': 'SHA256:5d5506c07c238918d0c892e7b04c26ad7f43684d89580780bb207d1d860b0b33',
       }
     self.perform_AtlasTest(downloads,'I')
 

diff --git a/Applications/SlicerApp/Testing/Python/CMakeLists.txt b/Applications/SlicerApp/Testing/Python/CMakeLists.txt
@@ -247,6 +247,12 @@ slicer_add_python_unittest(
   TESTNAME_PREFIX nomainwindow_
   )
 
+slicer_add_python_unittest(
+  SCRIPT ${Slicer_SOURCE_DIR}/Base/Python/slicer/tests/test_slicer_util_without_modules.py
+  SLICER_ARGS --no-main-window  --disable-modules
+  TESTNAME_PREFIX nomainwindow_
+  )
+
 slicer_add_python_unittest(
   SCRIPT ${Slicer_SOURCE_DIR}/Base/Python/slicer/tests/test_slicer_util_save.py
   SLICER_ARGS --no-main-window --disable-cli-modules --disable-scripted-loadable-modules DATA{${INPUT}/MR-head.nrrd}

diff --git a/Applications/SlicerApp/Testing/Python/DICOMReaders.py b/Applications/SlicerApp/Testing/Python/DICOMReaders.py
@@ -68,6 +68,7 @@ def test_AlternateReaders(self):
 
     referenceData = [
       { "url": "http://slicer.kitware.com/midas3/download?items=292839",
+        "checksum": "SHA256:3450ef9372a3460a2f181c8d3bb35a74b4f0acb10c6e18cfcf7804e1d99bf843",
         "fileName": "Mouse-MR-example-where-GDCM_fails.zip",
         "name": "Mouse-MR-example-where-GDCM_fails",
         "seriesUID": "1.3.6.1.4.1.9590.100.1.2.366426457713813178933224342280246227461",
@@ -76,6 +77,7 @@ def test_AlternateReaders(self):
         "voxelValueUnits": "(1, UCUM, \"no units\")"
       },
       { "url": "http://slicer.kitware.com/midas3/download?items=294857",
+        "checksum": "SHA256:899f3f8617ca53bad7dca0b2908478319e708b48ff41dfa64b6bac1d76529928",
         "fileName": "deidentifiedMRHead-dcm-one-series.zip",
         "name": "deidentifiedMRHead-dcm-one-series",
         "seriesUID": "1.3.6.1.4.1.5962.99.1.3814087073.479799962.1489872804257.270.0",
@@ -103,7 +105,7 @@ def test_AlternateReaders(self):
       try:
         import SampleData
         dicomFilesDirectory = SampleData.downloadFromURL(
-          fileNames=dataset['fileName'], uris=dataset['url'])[0]
+          fileNames=dataset['fileName'], uris=dataset['url'], checksums=dataset['checksum'])[0]
         self.delayDisplay('Finished with download')
 
         #
@@ -208,7 +210,8 @@ def test_MissingSlices(self):
     import SampleData
     dicomFilesDirectory = SampleData.downloadFromURL(
       fileNames='deidentifiedMRHead-dcm-one-series.zip',
-      uris='http://slicer.kitware.com/midas3/download?items=294857')[0]
+      uris='http://slicer.kitware.com/midas3/download?items=294857',
+      checksums='SHA256:899f3f8617ca53bad7dca0b2908478319e708b48ff41dfa64b6bac1d76529928')[0]
     self.delayDisplay('Finished with download\n')
 
     seriesUID = "1.3.6.1.4.1.5962.99.1.3814087073.479799962.1489872804257.270.0"

diff --git a/Applications/SlicerApp/Testing/Python/JRC2013Vis.py b/Applications/SlicerApp/Testing/Python/JRC2013Vis.py
@@ -95,7 +95,8 @@ def onStartStopDicomPeer(self,flag):
         import SampleData
         SampleData.downloadFromURL(
           fileNames='Dcmtk-db.zip',
-          uris='http://slicer.kitware.com/midas3/download?items=18822')[0]
+          uris='http://slicer.kitware.com/midas3/download?items=18822',
+          checksums='SHA256:6bfb01cf5ffb8e3af9b1c0c9556f0c6b45f0ec40305a9539ed7a9f0dcfe378e3')[0]
 
       import subprocess
       dcmqrscpExeOptions = (
@@ -176,7 +177,8 @@ def test_Part1DICOM(self):
     import SampleData
     SampleData.downloadFromURL(
       fileNames='Dcmtk-db.zip',
-      uris='http://slicer.kitware.com/midas3/download?items=18822')[0]
+      uris='http://slicer.kitware.com/midas3/download?items=18822',
+      checksums='SHA256:6bfb01cf5ffb8e3af9b1c0c9556f0c6b45f0ec40305a9539ed7a9f0dcfe378e3')[0]
 
     try:
       self.delayDisplay("Switching to temp database directory")
@@ -299,7 +301,8 @@ def test_Part2Head(self):
     SampleData.downloadFromURL(
       fileNames='3DHeadData.mrb',
       loadFiles=True,
-      uris='http://slicer.kitware.com/midas3/download?items=8609')
+      uris='http://slicer.kitware.com/midas3/download?items=8609',
+      checksums='SHA256:e2c7944095dd92be7961bed37f3c8f49e6f40c7f31d4fe865753b6efddae7993')
     self.delayDisplay('Finished with download and loading\n')
 
     try:
@@ -372,7 +375,8 @@ def test_Part3Liver(self):
     SampleData.downloadFromURL(
       fileNames='LiverData.mrb',
       loadFiles=True,
-      uris='http://slicer.kitware.com/midas3/download?items=8611')
+      uris='http://slicer.kitware.com/midas3/download?items=8611',
+      checksums='SHA256:a39075d3e87f80bbf8eba1e0222ee68c60036e57c3db830db08f3022f424e221')
     self.delayDisplay('Finished with download and loading\n')
 
     try:
@@ -440,7 +444,8 @@ def test_Part4Lung(self):
     SampleData.downloadFromURL(
       fileNames='LungData.mrb',
       loadFiles=True,
-      uris='http://slicer.kitware.com/midas3/download?items=8612')
+      uris='http://slicer.kitware.com/midas3/download?items=8612',
+      checksums='SHA256:9da091065aa42edbba2d436a2ef21a093792e8a76455c28e5b80590b04f5a73e')
     self.delayDisplay('Finished with download and loading\n')
 
     try:

diff --git a/Applications/SlicerApp/Testing/Python/RSNA2012ProstateDemo.py b/Applications/SlicerApp/Testing/Python/RSNA2012ProstateDemo.py
@@ -69,7 +69,8 @@ def test_RSNA2012ProstateDemo(self):
     SampleData.downloadFromURL(
       fileNames='RSNA2012ProstateDemo.mrb',
       loadFiles=True,
-      uris='http://slicer.kitware.com/midas3/download?items=10697')
+      uris='http://slicer.kitware.com/midas3/download?items=10697',
+      checksums='SHA256:2627388ee213564f8783d0242993212ba01189f4c6640d57c4cde4e28fc5f97b')
 
     # get all scene view nodes and test switching
     svns = slicer.util.getNodes('vtkMRMLSceneViewNode*')

diff --git a/Applications/SlicerApp/Testing/Python/RSNAQuantTutorial.py b/Applications/SlicerApp/Testing/Python/RSNAQuantTutorial.py
@@ -220,7 +220,8 @@ def test_Part3PETCT(self,enableScreenshotsFlag=0,screenshotScaleFactor=1):
     import SampleData
     extractPath = SampleData.downloadFromURL(
       fileNames='dataset3_PETCT.zip',
-      uris='http://slicer.kitware.com/midas3/download?items=124185')[0]
+      uris='http://slicer.kitware.com/midas3/download?items=124185',
+      checksums='SHA256:11e81af3462076f4ca371b632e03ed435240042915c2daf07f80059b3f78f88d')[0]
 
     self.delayDisplay("Loading PET_CT_pre-treatment.mrb")
     preTreatmentPath = extractPath + '/PET_CT_pre-treatment.mrb'
@@ -323,7 +324,8 @@ def test_Part2ChangeTracker(self,enableScreenshotsFlag=0,screenshotScaleFactor=1
     SampleData.downloadFromURL(
       fileNames='ChangeTrackerScene.mrb',
       loadFiles=True,
-      uris='http://slicer.kitware.com/midas3/download?items=124184')
+      uris='http://slicer.kitware.com/midas3/download?items=124184',
+      checksums='SHA256:64734cbbf8ebafe4a52f551d1510a8f6f3d0625eb5b6c1e328be117c48e2c653')
     logic.takeScreenshot('ChangeTracker-Loaded','Finished with download and loading',-1)
 
     try:

diff --git a/Applications/SlicerApp/Testing/Python/RSNAVisTutorial.py b/Applications/SlicerApp/Testing/Python/RSNAVisTutorial.py
@@ -186,7 +186,8 @@ def test_Part1DICOM(self,enableScreenshotsFlag=0,screenshotScaleFactor=1):
     import SampleData
     dicomFilesDirectory = SampleData.downloadFromURL(
       fileNames='dataset1_Thorax_Abdomen.zip',
-      uris='http://slicer.kitware.com/midas3/download?items=124183')[0]
+      uris='http://slicer.kitware.com/midas3/download?items=124183',
+      checksums='SHA256:17a4199aad03a373dab27dc17e5bfcf84fc194d0a30975b4073e5b595d43a56a')[0]
 
     try:
       self.delayDisplay("Switching to temp database directory")
@@ -310,7 +311,8 @@ def test_Part2Head(self,enableScreenshotsFlag=0,screenshotScaleFactor=1):
     SampleData.downloadFromURL(
       fileNames='Head_Scene.mrb',
       loadFiles=True,
-      uris='http://slicer.kitware.com/midas3/download?items=124180')
+      uris='http://slicer.kitware.com/midas3/download?items=124180',
+      checksums='SHA256:6785e481925c912a5a3940e9c9b71935df93a78a871e10f66ab71f8478229e68')
 
     logic.takeScreenshot('Head-Downloaded','Finished with download and loading',-1)
 
@@ -410,7 +412,8 @@ def test_Part3Liver(self,enableScreenshotsFlag=0,screenshotScaleFactor=1):
     SampleData.downloadFromURL(
       fileNames='LiverSegments_Scene.mrb',
       loadFiles=True,
-      uris='http://slicer.kitware.com/midas3/download?items=124181')
+      uris='http://slicer.kitware.com/midas3/download?items=124181',
+      checksums='SHA256:ff797140c13a5988a7b72920adf0d2dab390a9babeab9161d5c52613328249f7')
 
     logic.takeScreenshot('Liver-Loaded','Loaded Liver scene',-1)
 
@@ -483,7 +486,8 @@ def test_Part4Lung(self,enableScreenshotsFlag=0,screenshotScaleFactor=1):
     SampleData.downloadFromURL(
       fileNames='LungSegments_Scene.mrb',
       loadFiles=True,
-      uris='http://slicer.kitware.com/midas3/download?items=124182')
+      uris='http://slicer.kitware.com/midas3/download?items=124182',
+      checksums='SHA256:89ffc6cabd76a17dfa6beb404a5901a4b4e4b4f2f4ee46c2d5f4d34459f554a1')
 
     logic.takeScreenshot('Lung-Loaded','Finished with download and loading',-1)
 

diff --git a/Applications/SlicerApp/Testing/Python/ScenePerformance.py b/Applications/SlicerApp/Testing/Python/ScenePerformance.py
@@ -112,17 +112,12 @@ def findWidget(self, widget, objectName):
 #
 class ScenePerformanceLogic(ScriptedLoadableModuleLogic):
 
-  def downloadFile(self, downloadURL, downloadFileName):
-    downloads = (
-        (downloadURL, downloadFileName),
-        )
-
-    import urllib.request, urllib.parse, urllib.error
-    for url,name in downloads:
-      filePath = slicer.app.temporaryPath + '/' + name
-      if not os.path.exists(filePath) or os.stat(filePath).st_size == 0:
-        urllib.request.urlretrieve(url, filePath)
-    return filePath
+  def downloadFile(self, downloadURL, downloadFileName, downloadFileChecksum=None):
+    import SampleData
+    return SampleData.downloadFromURL(
+      fileNames=downloadFileName,
+      uris=downloadURL,
+      checksums=downloadFileChecksum)[0]
 
   def startTiming(self):
     self.Timer = qt.QTime()
@@ -149,7 +144,7 @@ def runTest(self):
   def testAll(self):
     self.setUp()
 
-    self.addURLData('http://slicer.kitware.com/midas3/download?items=10937', 'BrainAtlas2012.mrb')
+    self.addURLData('http://slicer.kitware.com/midas3/download?items=10937', 'BrainAtlas2012.mrb', 'SHA256:688ebcc6f45989795be2bcdc6b8b5bfc461f1656d677ed3ddef8c313532687f1')
     self.modifyNodeByID('vtkMRMLScalarVolumeNode1')
     self.modifyNodeByID('vtkMRMLScalarVolumeNode2')
     self.modifyNodeByID('vtkMRMLScalarVolumeNode3')
@@ -180,9 +175,9 @@ def displayPerformance(self, action, property, time):
     self.delayDisplay(message)
     return message
 
-  def addURLData(self, url, file):
+  def addURLData(self, url, file, checksum):
     logic = ScenePerformanceLogic()
-    file = logic.downloadFile(url, file)
+    file = logic.downloadFile(url, file, checksum)
     self.addData(file)
 
   def addData(self, file):

diff --git a/Applications/SlicerApp/Testing/Python/SliceLinkLogic.py b/Applications/SlicerApp/Testing/Python/SliceLinkLogic.py
@@ -127,7 +127,8 @@ def test_SliceLinkLogic1(self):
     SampleData.downloadFromURL(
       nodeNames='FA',
       fileNames='FA.nrrd',
-      uris='http://slicer.kitware.com/midas3/download?items=5767')
+      uris='http://slicer.kitware.com/midas3/download?items=5767',
+      checksums='SHA256:12d17fba4f2e1f1a843f0757366f28c3f3e1a8bb38836f0de2a32bb1cd476560')
     self.delayDisplay('Finished with download and loading')
     print('')
 

diff --git a/Applications/SlicerApp/Testing/Python/Slicer4Minute.py b/Applications/SlicerApp/Testing/Python/Slicer4Minute.py
@@ -124,7 +124,8 @@ def test_Slicer4Minute1(self):
     SampleData.downloadFromURL(
       fileNames='slicer4minute.mrb',
       loadFiles=True,
-      uris='http://slicer.kitware.com/midas3/download?items=8466')
+      uris='http://slicer.kitware.com/midas3/download?items=8466',
+      checksums='SHA256:5a1c78c3347f77970b1a29e718bfa10e5376214692d55a7320af94b9d8d592b8')
     self.delayDisplay('Finished with download and loading')
 
     # Testing "Part 2" of Tutorial

diff --git a/Applications/SlicerApp/Testing/Python/SlicerRestoreSceneViewCrashIssue3445.py b/Applications/SlicerApp/Testing/Python/SlicerRestoreSceneViewCrashIssue3445.py
@@ -97,7 +97,8 @@ def test_SlicerRestoreSceneViewCrashIssue3445(self):
     filePath = SampleData.downloadFromURL(
       fileNames='BrainAtlas2012.mrb',
       loadFiles=True,
-      uris='http://slicer.kitware.com/midas3/download?items=10937')[0]
+      uris='http://slicer.kitware.com/midas3/download?items=10937',
+      checksums='SHA256:688ebcc6f45989795be2bcdc6b8b5bfc461f1656d677ed3ddef8c313532687f1')[0]
 
     self.delayDisplay('Finished with download')
 

diff --git a/Applications/SlicerApp/Testing/Python/SubjectHierarchyGenericSelfTest.py b/Applications/SlicerApp/Testing/Python/SubjectHierarchyGenericSelfTest.py
@@ -119,6 +119,7 @@ def section_SetupPathsAndNames(self):
 
     self.dicomDatabaseDir = subjectHierarchyGenericSelfTestDir + '/CtkDicomDatabase'
     self.dicomZipFileUrl = 'http://slicer.kitware.com/midas3/download/item/137843/TestDicomCT.zip'
+    self.dicomZipChecksum = 'SHA256:1aa0bb177bbf6471ca5f2192340a6cecdedb81b33506b03ff316c6b5f624e863'
     self.dicomZipFilePath = subjectHierarchyGenericSelfTestDir + '/TestDicomCT.zip'
     self.expectedNumOfFilesInDicomDataDir = 10
     self.tempDir = subjectHierarchyGenericSelfTestDir + '/Temp'
@@ -174,7 +175,7 @@ def section_LoadDicomData(self):
         with DICOMUtils.LoadDICOMFilesToDatabase( \
             self.dicomZipFileUrl, self.dicomZipFilePath, \
             self.dicomDataDir, self.expectedNumOfFilesInDicomDataDir, \
-            {}, loadedNodes) as success:
+            {}, loadedNodes, checksum=self.dicomZipChecksum) as success:
           self.assertTrue(success)
 
       self.assertEqual( len( slicer.util.getNodes('vtkMRMLSubjectHierarchyNode*') ), 1 )

diff --git a/Applications/SlicerApp/Testing/Python/slicerCloseCrashBug2590.py b/Applications/SlicerApp/Testing/Python/slicerCloseCrashBug2590.py
@@ -91,7 +91,8 @@ def test_Part3ChangeTracker(self):
     import SampleData
     SampleData.downloadFromURL(
       fileNames='RSNA2011_ChangeTracker_data.zip',
-      uris='http://slicer.kitware.com/midas3/download?items=8986')
+      uris='http://slicer.kitware.com/midas3/download?items=8986',
+      checksums='SHA256:256bf00a83884fab173edc9f83c028f654bd5eb44aeed28d2203ec76fab941ce')
 
     try:
       mainWindow = slicer.util.mainWindow()

diff --git a/Base/Python/slicer/tests/compute-checksum.txt b/Base/Python/slicer/tests/compute-checksum.txt
@@ -0,0 +1 @@
+This is a text file!
diff --git a/Base/Python/slicer/tests/test_slicer_util_getNodes.py b/Base/Python/slicer/tests/test_slicer_util_getNodes.py
@@ -3,7 +3,7 @@
 import slicer.util
 import vtk
 
-class SlicerUtilTest(unittest.TestCase):
+class SlicerUtilGetNodeTest(unittest.TestCase):
 
     def setUp(self):
         slicer.mrmlScene.Clear(0)

diff --git a/Base/Python/slicer/tests/test_slicer_util_without_modules.py b/Base/Python/slicer/tests/test_slicer_util_without_modules.py
@@ -0,0 +1,31 @@
+import os
+import unittest
+import slicer.util
+
+
+class SlicerUtilWithoutModulesTest(unittest.TestCase):
+
+  def setUp(self):
+    pass
+
+  def test_computeChecksum(self):
+    with self.assertRaises(IOError):
+      slicer.util.computeChecksum('SHA256', 'compute-checksum-nonexistent.txt')
+
+    input_file = os.path.join(os.path.dirname(__file__), 'compute-checksum.txt')
+    self.assertEqual(slicer.util.computeChecksum('SHA256', input_file), '4a57f3207b97f26a6061f86948483c00b03893ddfef9e82b639ebe66e3aba338')
+    self.assertEqual(slicer.util.computeChecksum('SHA512', input_file), '5080ee92e951c5f8336053f11d278c23f5d26b5eb78805c952960eac0194f357f98b0e350611ce081d4a1e28dd8ea182d3a276c99b1752e0def2de0f47b8b27b')
+
+    with self.assertRaises(ValueError):
+      slicer.util.computeChecksum('SHAINVALID', input_file)
+
+  def test_extractAlgoAndDigest(self):
+    with self.assertRaises(ValueError):
+      slicer.util.extractAlgoAndDigest('4a57f3207b97f26a6061f86948483c00b03893ddfef9e82b639ebe66e3aba338')
+
+    with self.assertRaises(ValueError):
+      slicer.util.extractAlgoAndDigest('SHAINVALID:4a57f3207b97f26a6061f86948483c00b03893ddfef9e82b639ebe66e3aba338')
+
+    self.assertEqual(
+      slicer.util.extractAlgoAndDigest('SHA256:4a57f3207b97f26a6061f86948483c00b03893ddfef9e82b639ebe66e3aba338'),
+      ('SHA256', '4a57f3207b97f26a6061f86948483c00b03893ddfef9e82b639ebe66e3aba338'))