Merge 743bc34 into 4ef15a9

.travis.yml

@@ -1,21 +1,22 @@
-dist: xenial
+dist: bionic
 language: python
 python:
   - "3.6"
   - "3.7"
   - "3.8"
+  - "3.9"
 # Install sub-module using public HTTPS not SSH 
 git:
   submodules: false
 before_install:
   - sed -i 's/git@github.com:/https:\/\/github.com\//' .gitmodules
   - git submodule update --init --recursive
 install:
-  - pip install coveralls pycodestyle pep257
+  - pip install bagit  #pylint is unhappy with bagit install via setup.py, not sure why
+  - pip install coveralls pylint
   - python setup.py install
 script:
-  - pycodestyle --ignore=E501,W503 ocfl/*.py tests/*.py *.py
-  - pep257 ocfl/*.py tests/*.py *.py
+  - pylint -d W0511,W0622,W0707,C0301,C0103,R0913,R0902,R0911,R0912,R0915,R1714,R0914,R1702 *.py ocfl tests
   - python setup.py test
 after_success:
   - python setup.py coverage

extract_codes.py

@@ -1,12 +1,11 @@
 #!/usr/bin/env python
 """Extract list of currently implemented error and warning codes."""
-
 import datetime
-import fs
 import logging
 import os.path
 import re
 import requests
+import fs
 
 from ocfl.validation_logger import ValidationLogger
 
@@ -16,7 +15,7 @@
 VALIDATION_STATUS_MD = 'docs/validation_status.md'
 
 
-class Code(object):
+class Code():
     """Class for details of one error or warning code."""
 
     def __init__(self, code, desc):
@@ -43,7 +42,7 @@ def desc_links(self, suffix):
         else:
             links = "_Not implemented_"
         desc = self.suffixes[suffix]['desc'] or "**Missing description**"
-        return("%s \\[%s\\]" % (desc, links))
+        return "%s \\[%s\\]" % (desc, links)
 
     def as_str(self):
         """String output for markdown."""
@@ -63,10 +62,10 @@ def as_str(self):
             if suffix == '':
                 continue
             s += '| | %s%s | %s |\n' % (self.code, suffix, self.desc_links(suffix))
-        return(s)
+        return s
 
 
-class Codes(object):
+class Codes():
     """Class for the complete set of error and warning codes."""
 
     def __init__(self):
@@ -83,64 +82,68 @@ def add_impl(self, code, suffix, link=None, desc=None):
             self.codes[code] = Code(code, None)
         self.codes[code].add_suffix(suffix, link=link, desc=desc)
 
-    def as_str(self, filter=''):
+    def as_str(self, exclude=''):
         """String output for markdown."""
         s = ''
         for code in sorted(self.codes.keys()):
-            if code.startswith(filter):
+            if code.startswith(exclude):
                 s += self.codes[code].as_str()
-        return(s)
-
-
-# 0. Assemble all data in codes
-codes = Codes()
-
-# 1. Get validation codes from github
-md = requests.get(VALIDATION_CODES_URL).text
-for line in md.split('\n'):
-    m = re.match(r'''\|\s*([EW]\d\d\d)\s*\|\s*([^\|]+)\|''', line)
-    if m:
-        code = m.group(1)
-        desc = m.group(2).rstrip()
-        codes.add_spec(code, desc)
-
-# 2. Get validation codes and messages from strings file
-vl = ValidationLogger()
-for code_suffix in vl.validation_codes:
-    try:
-        desc = vl.validation_codes[code_suffix]['description']['en']
-    except KeyError:
-        desc = "MISSING ENGLISH DESCRIPTION"
-    m = re.match(r'''([EW]\d\d\d)(\w?)$''', code_suffix)
-    if m:
-        codes.add_impl(m.group(1), m.group(2), desc=desc)
-    else:
-        logging.error("Bad entry for code+suffix '%s' in strings file" % (code_suffix))
-
-# 3. Get validation codes from ocfl-py Python codes
-code_fs = fs.open_fs('ocfl')
-for file in code_fs.walk.files(filter=['*.py']):
-    with code_fs.open(file) as fh:
-        n = 0
-        for line in fh:
-            n += 1
-            m = re.search(r'''(["'])([EW]\d\d\d)(\w)?\1''', line)
-            if m:
-                file_line = 'ocfl%s#L%d' % (file, n)
-                link = '[' + file_line + '](' + GITHUB_REPO + '/blob/main/' + file_line + ')'
-                codes.add_impl(m.group(2), m.group(3), link=link)
-
-# 4. Write table of what is implemented and raise warnings
-logging.info("Writing summary to %s" % (VALIDATION_STATUS_MD))
-with open(VALIDATION_STATUS_MD, "w") as fh:
-    fh.write("# Implementation status for errors and warnings\n\n")
-    fh.write("The following tables show the implementation status of all errors and warnings in the OCFL v1.0 specification, with links to the specification and into the code repository.\n\n")
-    fh.write("## Errors\n\n")
-    fh.write("| Code | Specification text (or suffixed code) | Implementation status and message/links |\n")
-    fh.write("| --- | --- | --- |\n")
-    fh.write(codes.as_str(filter='E') + "\n")
-    fh.write("## Warnings\n\n")
-    fh.write("| Code | Specification text (or suffixed code) | Implementation status and message/links |\n")
-    fh.write("| --- | --- | --- |\n")
-    fh.write(codes.as_str(filter='W') + "\n")
-    fh.write("_Generated by `%s` at %s_" % (os.path.basename(__file__), datetime.datetime.now()))
+        return s
+
+def main():
+    """Run from command line."""
+    # 0. Assemble all data in codes
+    codes = Codes()
+
+    # 1. Get validation codes from github
+    md = requests.get(VALIDATION_CODES_URL).text
+    for line in md.split('\n'):
+        m = re.match(r'''\|\s*([EW]\d\d\d)\s*\|\s*([^\|]+)\|''', line)
+        if m:
+            code = m.group(1)
+            desc = m.group(2).rstrip()
+            codes.add_spec(code, desc)
+
+    # 2. Get validation codes and messages from strings file
+    vl = ValidationLogger()
+    for code_suffix in vl.validation_codes:
+        try:
+            desc = vl.validation_codes[code_suffix]['description']['en']
+        except KeyError:
+            desc = "MISSING ENGLISH DESCRIPTION"
+        m = re.match(r'''([EW]\d\d\d)(\w?)$''', code_suffix)
+        if m:
+            codes.add_impl(m.group(1), m.group(2), desc=desc)
+        else:
+            logging.error("Bad entry for code+suffix '%s' in strings file", code_suffix)
+
+    # 3. Get validation codes from ocfl-py Python codes
+    code_fs = fs.open_fs('ocfl')
+    for file in code_fs.walk.files(filter=['*.py']):
+        with code_fs.open(file) as fh:
+            n = 0
+            for line in fh:
+                n += 1
+                m = re.search(r'''(["'])([EW]\d\d\d)(\w)?\1''', line)
+                if m:
+                    file_line = 'ocfl%s#L%d' % (file, n)
+                    link = '[' + file_line + '](' + GITHUB_REPO + '/blob/main/' + file_line + ')'
+                    codes.add_impl(m.group(2), m.group(3), link=link)
+
+    # 4. Write table of what is implemented and raise warnings
+    logging.info("Writing summary to %s", VALIDATION_STATUS_MD)
+    with open(VALIDATION_STATUS_MD, "w") as fh:
+        fh.write("# Implementation status for errors and warnings\n\n")
+        fh.write("The following tables show the implementation status of all errors and warnings in the OCFL v1.0 specification, with links to the specification and into the code repository.\n\n")
+        fh.write("## Errors\n\n")
+        fh.write("| Code | Specification text (or suffixed code) | Implementation status and message/links |\n")
+        fh.write("| --- | --- | --- |\n")
+        fh.write(codes.as_str(exclude='E') + "\n")
+        fh.write("## Warnings\n\n")
+        fh.write("| Code | Specification text (or suffixed code) | Implementation status and message/links |\n")
+        fh.write("| --- | --- | --- |\n")
+        fh.write(codes.as_str(exclude='W') + "\n")
+        fh.write("_Generated by `%s` at %s_" % (os.path.basename(__file__), datetime.datetime.now()))
+
+if __name__ == "__main__":
+    main()

ocfl-object.py

@@ -2,15 +2,13 @@
 """OCFL Object and Inventory Builder."""
 import argparse
 import logging
-import ocfl
 import sys
 
+import ocfl
 
 class FatalError(Exception):
     """Exception class for conditions that should abort with message."""
 
-    pass
-
 
 def parse_arguments():
     """Parse command line arguments."""
@@ -77,7 +75,7 @@ def parse_arguments():
 
 def do_object_operation(args):
     """Implement object operations in a way that can be reused by ocfl-store.py."""
-    obj = ocfl.Object(id=args.id,
+    obj = ocfl.Object(identifier=args.id,
                       digest_algorithm=args.digest,
                       filepath_normalization=args.normalization,
                       forward_delta=not args.no_forward_delta,
@@ -140,9 +138,9 @@ def do_object_operation(args):
 
 if __name__ == "__main__":
     try:
-        args = parse_arguments()
-        logging.basicConfig(level=logging.INFO if args.verbose else logging.WARN)
-        do_object_operation(args)
+        aargs = parse_arguments()
+        logging.basicConfig(level=logging.INFO if aargs.verbose else logging.WARN)
+        do_object_operation(aargs)
     except (FatalError, ocfl.ObjectException) as e:
         # Show message but otherwise exit quietly
         print('Error - ' + str(e))

ocfl-sidecar.py

@@ -2,9 +2,10 @@
 """OCFL inventory sidecar generator and updater."""
 import argparse
 import logging
-import ocfl
 import os.path
 
+import ocfl
+
 INVENTORY_NAME = "inventory.json"
 
 
@@ -22,36 +23,39 @@ def parse_arguments():
     return args
 
 
-def create_sidecar(dir):
+def create_sidecar(args, directory):
     """Create sidecar for inventory in dir."""
-    inventory_path = os.path.join(dir, INVENTORY_NAME)
+    inventory_path = os.path.join(directory, INVENTORY_NAME)
     if not os.path.isfile(inventory_path):
-        logging.error("Ignoring path %s because there is no inventory file %s." % (dir, inventory_path))
+        logging.error("Ignoring path %s because there is no inventory file %s.", directory, inventory_path)
     else:
-        object = ocfl.Object(path=dir)
+        obj = ocfl.Object(path=directory)
         if args.digest is not None:
-            object.digest_algorithm = args.digest
+            obj.digest_algorithm = args.digest
         else:  # Read inventory in the hope of setting digest_algoritm
             try:
-                object.parse_inventory()
+                obj.parse_inventory()
             except ocfl.ObjectException as e:
-                logging.warning("Failed to read inventory in directory %s (%s)" % (dir, str(e)))
-        sidecar = object.write_inventory_sidecar()
-        logging.info("Written sidecar file %s" % (sidecar))
-
+                logging.warning("Failed to read inventory in directory %s (%s)", directory, e)
+        sidecar = obj.write_inventory_sidecar()
+        logging.info("Written sidecar file %s", sidecar)
 
-if __name__ == "__main__":
+def main():
+    """Run from command line."""
     args = parse_arguments()
     logging.basicConfig(level=logging.INFO if args.verbose else logging.WARN)
     paths = ["."] if len(args.path) == 0 else args.path
     for path in paths:
-        logging.info("Looking at path %s" % (path))
+        logging.info("Looking at path %s", path)
         if os.path.isdir(path):
-            create_sidecar(path)
+            create_sidecar(args, path)
         else:
-            (dir, filename) = os.path.split(path)
+            (directory, filename) = os.path.split(path)
             if filename == INVENTORY_NAME:
-                create_sidecar(dir)
+                create_sidecar(args, directory)
             else:
                 logging.error("Ignoring path %s with filename that is not inventory.json")
+
+if __name__ == "__main__":
+    main()
     print("Done.")

ocfl-store.py

@@ -2,9 +2,10 @@
 """OCFL Storage Root Tool."""
 import argparse
 import logging
-import ocfl
 import sys
 
+import ocfl
+
 parser = argparse.ArgumentParser(description='Manpulate or validate an OCFL Storage Root.',
                                  formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 parser.add_argument('--root', required=True,
@@ -83,7 +84,7 @@
         else:
             logging.error("create/build/validate not implemented")
     else:
-        logging.warn("Nuttin' happenin' 'round 'ere.")
+        logging.warning("No command, nothing to do.")
 except (ocfl.StoreException, ocfl.ObjectException) as e:
     logging.error(str(e))
     sys.exit(1)

ocfl-validate.py

@@ -2,9 +2,10 @@
 """Validate an OCFL Object."""
 import argparse
 import logging
-import ocfl
 import sys
 
+import ocfl
+
 parser = argparse.ArgumentParser(
     description='Validate one or more OCFL objects, storage roots or standalone '
     'inventory files. By default shows any errors or warnings, and final '
@@ -40,31 +41,31 @@
     num += 1
     path_type = ocfl.find_path_type(path)
     if path_type == 'object':
-        log.info("Validating OCFL Object at " + path)
+        log.info("Validating OCFL Object at %s", path)
         obj = ocfl.Object(lax_digests=args.lax_digests)
         if obj.validate(path,
                         show_warnings=show_warnings,
                         show_errors=not args.very_quiet,
                         check_digests=not args.no_check_digests):
             num_good += 1
     elif path_type == 'root':
-        log.info("Validating OCFL Storage Root at " + path)
+        log.info("Validating OCFL Storage Root at %s", path)
         store = ocfl.Store(root=path,
                            lax_digests=args.lax_digests)
         if store.validate(show_warnings=show_warnings,
                           show_errors=not args.very_quiet,
                           check_digests=not args.no_check_digests):
             num_good += 1
     elif path_type == 'file':
-        log.info("Validating separate OCFL Inventory at " + path)
+        log.info("Validating separate OCFL Inventory at %s", path)
         obj = ocfl.Object(lax_digests=args.lax_digests)
         if obj.validate_inventory(path,
                                   show_warnings=show_warnings,
                                   show_errors=not args.very_quiet):
             num_good += 1
     else:
-        log.error("Bad path %s (%s)" % (path, path_type))
+        log.error("Bad path %s (%s)", path, path_type)
     if num_paths > 1:
-        log.info(" [%d / %d paths validated, %d / %d VALID]\n" % (num, num_paths, num_good, num))
+        log.info(" [%d / %d paths validated, %d / %d VALID]\n", num, num_paths, num_good, num)
 if num_good != num:
     sys.exit(1)

ocfl/bagger.py

@@ -15,15 +15,13 @@
 suited to transferring content that might be used to update an OCLF object
 or disseminate a particular version.
 """
-import bagit
 import os.path
+import bagit
 
 
 class BaggerError(Exception):
     """Exception class for conditions that should abort with message."""
 
-    pass
-
 
 def bag_as_source(srcbag, metadata):
     """Validate and read metadata from srcbag as input.
@@ -76,4 +74,4 @@ def bag_extracted_version(dst, metadata):
         tags['Contact-Name'] = metadata.name
     if metadata.address and metadata.address.startswith('mailto:'):
         tags['Contact-Email'] = metadata.address[7:]
-    bag = bagit.make_bag(dst, bag_info=tags, checksums=['sha512'])
+    bagit.make_bag(dst, bag_info=tags, checksums=['sha512'])