fix up some python version problems

weaverba137 · Aug 10, 2017 · 3c0a184 · 3c0a184
1 parent d978f62
commit 3c0a184
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 7 deletions.
diff --git a/hpsspy/scan.py b/hpsspy/scan.py
@@ -40,7 +40,8 @@ def validate_configuration(config):
             except ValueError:
                 logger.critical("%s is not valid JSON.", config)
                 return 1
-    except FileNotFoundError:
+    # except FileNotFoundError:
+    except OSError:
         logger.critical("%s does not exist. Try again.", config)
         return 1
     if 'config' in json_data:
@@ -120,11 +121,11 @@ def files_to_hpss(hpss_map_cache, release):
         with open(hpss_map_cache) as t:
             hpss_map = json.load(t)
     else:
-        if resource_exists('hpsspy.data', hpss_map_cache):
+        if resource_exists('hpsspy', 'data/' + hpss_map_cache):
             logger.info("Reading from file %s in the hpsspy distribution.",
                         hpss_map_cache)
-            t = resource_stream('hpsspy.data', hpss_map_cache)
-            hpss_map = json.load(t)
+            t = resource_stream('hpsspy', 'data/' + hpss_map_cache)
+            hpss_map = json.loads(t.read().decode())
             t.close()
         else:
             logger.warning("Returning empty map file!")
@@ -356,7 +357,7 @@ def process_missing(missing_cache, disk_root, hpss_root, dirmode='2770',
                     makedirs(dirname(h_file), mode=dirmode)
                 created_directories.add(dirname(h_file))
             logger.info("hsi('put', '%s', ':', '%s')",
-                         join(disk_root, missing[h]['files'][0]), h_file)
+                        join(disk_root, missing[h]['files'][0]), h_file)
             if test:
                 out = "Test mode, skipping hsi command."
             else:

diff --git a/hpsspy/test/t/test_scan.json b/hpsspy/test/t/test_scan.json
@@ -8,7 +8,7 @@
         "exclude":["README.html"],
         "d1":{
             "d1/batch/.*$":"d1/batch.tar",
-            "d1/([^/]+\\.txt)":"d1/\\1",
+            "d1/([^/]+\\.txt)$":"d1/\\1",
             "d1/templates/[^/]+$":"d1/templates/templates_files.tar"
         },
         "d2":{

diff --git a/hpsspy/test/test_scan.py b/hpsspy/test/test_scan.py
@@ -19,7 +19,8 @@
 import logging
 from logging.handlers import MemoryHandler
 from pkg_resources import resource_filename, resource_stream
-from ..scan import compile_map, physical_disks, validate_configuration
+from ..scan import (compile_map, files_to_hpss, physical_disks,
+                    validate_configuration)
 
 
 class TestHandler(MemoryHandler):
@@ -103,6 +104,25 @@ def test_compile_map(self):
             new_map = compile_map(self.config, 'redux')
             self.assertEqual(err.colno, 8)
 
+    def test_files_to_hpss(self):
+        """Test conversion of JSON files to directory dictionary.
+        """
+        hpss_map, config = files_to_hpss(self.config_name, 'data')
+        self.assertEqual(config['root'], '/temporary')
+        for key in hpss_map['d2']:
+            self.assertIn(key[0].pattern, self.config['data']['d2'])
+            self.assertEqual(key[1], self.config['data']['d2'][key[0].pattern])
+        hpss_map, config = files_to_hpss('desi.json', 'datachallenge')
+        desi_map = {"dc2/batch/.*$": "dc2/batch.tar",
+                    "dc2/([^/]+\\.txt)$": "dc2/\\1",
+                    "dc2/templates/[^/]+$": "dc2/templates/templates_files.tar"
+                   }
+        for key in hpss_map['dc2']:
+            self.assertIn(key[0].pattern, desi_map)
+            self.assertEqual(key[1], desi_map[key[0].pattern])
+        hpss_map, config = files_to_hpss('foo.json', 'dr8')
+        self.assertIn('casload', hpss_map)
+
     def test_physical_disks(self):
         """Test physical disk path setup.
         """