Merge 9b1a4a6 into 8450271

.flake8

@@ -0,0 +1,35 @@
+[flake8]
+max-line-length = 120
+ignore =
+    # ==============================================================
+    # =========  Default list of things Flake8 would ignore ========
+    # ==============================================================
+    # E121: Continuation line under-indented for hanging indent
+    #       https://www.flake8rules.com/rules/E121.html
+    E121,
+    # E123: Closing bracket does not match indentation of opening bracket's line
+    #       https://www.flake8rules.com/rules/E123.html
+    E123,
+    # E126: Continuation line over-indented for hanging indent
+    #       https://www.flake8rules.com/rules/E126.html
+    E126,
+    # E226: Missing whitespace around arithmetic operator
+    #       https://www.flake8rules.com/rules/E226.html
+    E226,
+    # E24: ??? Hard to find doc on why this undocumented code's in default list
+    # E24,
+    # E704: Multiple statements on one line (def)
+    #       https://www.flake8rules.com/rules/E704.html
+    E704,
+    # W503: Line break occurred before a binary operator
+    #       https://www.flake8rules.com/rules/W503.html
+    W503,
+    # W504: Line break occurred after a binary operator
+    #       https://www.flake8rules.com/rules/W503.html
+    W504,
+    # ==============================================================
+    # =====  Other things we think it shouldn't complain about =====
+    # ==============================================================
+    # F541: f-string is missing placeholders
+    #       https://flake8.pycqa.org/en/latest/user/error-codes.html
+    F541

.gitignore

@@ -26,3 +26,6 @@ Data_Files/MicroscopyCalibration/Files/
 
 # PyCharm metadata
 .idea/
+
+# Environments
+*env/

Makefile

@@ -1,18 +1,27 @@
 .PHONY: test
 
+clear-poetry-cache:  # clear poetry/pypi cache. for user to do explicitly, never automatic
+	poetry cache clear pypi --all
+
 configure:  # does any pre-requisite installs
-	pip install poetry
+	pip install poetry==1.3.2
+
+lint:
+	flake8 wranglertools
 
 build:  # builds
 	make configure
 	poetry install
 
 test:
-	pytest -vv
+	pytest -vv -m 'not ftp'
 
 update:  # updates dependencies
 	poetry update
 
+tag-and-push:  # tags the branch and pushes it
+	@scripts/tag-and-push
+
 publish:
 	scripts/publish
 
@@ -22,7 +31,9 @@ help:
 info:
 	@: $(info Here are some 'make' options:)
 	   $(info - Use 'make configure' to install poetry, though 'make build' will do it automatically.)
+	   $(info - Use 'make lint' to check style with flake8.)
 	   $(info - Use 'make build' to install dependencies using poetry.)
+	   $(info - Use 'make clear-poetry-cache' to clear the poetry pypi cache if in a bad state. (Safe, but later recaching can be slow.))
 	   $(info - Use 'make publish' to publish this library, but only if auto-publishing has failed.)
 	   $(info - Use 'make test' to run tests with the normal options we use on travis)
 	   $(info - Use 'make update' to update dependencies (and the lock file))

pyproject.toml

@@ -13,22 +13,23 @@ packages = [
 
 [tool.poetry.dependencies]
 python = ">=3.7.0,<3.10"
-python-magic = ">=0.4.12"
-attrs = ">=21.4"
-openpyxl = "^3.0.9"
-dcicutils = ">=4.0"
+python-magic = ">=0.4.12,<1"
+attrs = "^22.2"
+openpyxl = "^3.1.2"
+dcicutils = "^6.10.1"
 # awscli is not directly imported but is required for aws cp operation
-awscli = "^1.22.88"
+awscli = "^1.27"
 
 [tool.poetry.dev-dependencies]
-pytest = "3.0.1"
-pytest-cov = "2.3.1"
-pytest-mock = ">=1.11.2, <=2.0.0"
-
-[build-system]
-requires = ["poetry>=0.12"]
-build-backend = "poetry.masonry.api"
+flake8 = ">=3.9.2"
+pytest = ">=7.2.2"
+pytest-cov = ">=4.0.0"
+pytest-mock = ">=3.10.0"
 
 [tool.poetry.scripts]
 import_data = "wranglertools.import_data:main"
 get_field_info = "wranglertools.get_field_info:main"
+
+[build-system]
+requires = ["poetry_core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

pytest.ini

@@ -0,0 +1,8 @@
+[pytest]
+markers =
+    file_operation: test involves file operations
+    ftp: test uses ftp
+    webtest: test uses web addresses
+norecursedirs = *env site-packages .cache .git .idea *.egg-info
+testpaths =
+    tests

scripts/tag-and-push

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+dirty=`git describe --all --dirty | rev | cut -d '-' -f 1 | rev`
+if [ "$dirty" = "dirty" ]; then
+    echo "This branch is dirty. That would create an ambiguity in what to tag."
+    exit 1
+fi
+
+if [ ! -e "pyproject.toml" ]; then
+    echo "There is no pyproject.toml"
+    exit 1
+fi
+
+version=`grep '^version = ' pyproject.toml | sed -E "s/^version = ['\"]([^'\"]+)['\"].*/\1/g"`
+
+if [ -z "${version}" ]; then
+    echo "pyproject.toml has no 'version = \"...\"' line."
+    exit 1
+fi
+
+new_tag=v${version}
+
+git tag ${new_tag}
+
+if [ $? -ne 0 ]; then
+    echo "Adding tag ${new_tag} failed."
+    exit 1
+else
+    echo "Added tag ${new_tag}."
+fi
+
+git push origin ${new_tag}
+
+if [ $? -ne 0 ]; then
+    echo "Pushing tag ${new_tag} failed."
+    exit 1
+else
+    echo "Pushed tag ${new_tag}."
+fi

tests/test_import_data.py

@@ -1,3 +1,4 @@
+from dcicutils.misc_utils import get_error_message
 import wranglertools.import_data as imp
 import pytest
 import pathlib as pp
@@ -7,7 +8,13 @@
 # @pytest.mark.file_operation
 @pytest.mark.ftp
 def test_attachment_from_ftp():
-    attach = imp.attachment("ftp://speedtest.tele2.net/1KB.zip")
+    try:
+        attach = imp.attachment("ftp://speedtest.tele2.net/1KB.zip")
+    except Exception as e:
+        msg = get_error_message(e)
+        if 'timeout' in msg.lower():
+            pytest.xfail(f"Integration test failure. {msg}")
+        raise
     assert attach
 
 

wranglertools/get_field_info.py

@@ -12,15 +12,15 @@
 EPILOG = '''
     To create an excel workbook file with sheets to be filled use the examples below and modify to your needs.
     It will accept the following optional parameters.
-        --keyfile        the path to the file where you have stored your access key info (default ~/keypairs.json)
-        --key            the name of the key identifier for the access key and secret in your keys file (default=default)
-        --type           use for each sheet that you want to add to the excel workbook
-        --nodesc         do not add the descriptions in the second line (by default they are added)
-        --noenums        do not add the list of options for a field if they are specified (by default they are added)
-        --comments       adds any (usually internal) comments together with enums (by default False)
-        --outfile        change the default file name "fields.xlsx" to a specified one
-        --debug          to add more debugging output
-        --noadmin        if you have admin access to 4DN this option lets you generate the sheet as a non-admin user
+        --keyfile      the path to the file where you have stored your access key info (default ~/keypairs.json)
+        --key          the name of the key identifier for the access key and secret in your keys file (default=default)
+        --type         use for each sheet that you want to add to the excel workbook
+        --nodesc       do not add the descriptions in the second line (by default they are added)
+        --noenums      do not add the list of options for a field if they are specified (by default they are added)
+        --comments     adds any (usually internal) comments together with enums (by default False)
+        --outfile      change the default file name "fields.xlsx" to a specified one
+        --debug        to add more debugging output
+        --noadmin      if you have admin access to 4DN this option lets you generate the sheet as a non-admin user
 
 
     This program graphs uploadable fields (i.e. not calculated properties)
@@ -44,9 +44,9 @@
 
 
 def _remove_all_from_types(args):
-    ''' helper method to remove the default 'all' argument that is automatically
+    """ helper method to remove the default 'all' argument that is automatically
         add by having a default with the append action for types option
-    '''
+    """
     if len(args.type) > 1:
         types = args.type
         types.remove('all')
@@ -144,7 +144,7 @@ def __init__(self, key4dn):
             self.email = me_page['email']
             self.check = True
             self.admin = True if 'admin' in me_page.get('groups', []) else False
-        except:
+        except Exception:
             print('Can not establish connection, please check your keys')
             me_page = {}
         if not me_page:
@@ -162,11 +162,11 @@ def __init__(self, key4dn):
             self.award = None
 
     def set_award(self, lab, dontPrompt=False):
-        '''Sets the award for the connection for use in import_data
+        """Sets the award for the connection for use in import_data
            if dontPrompt is False will ask the User to choose if there
            are more than one award for the connection.lab otherwise
            the first award for the lab will be used
-        '''
+        """
         self.award = None
         if lab is not None:
             labjson = ff_utils.get_metadata(lab, key=self.key)
@@ -195,10 +195,10 @@ def set_award(self, lab, dontPrompt=False):
         return
 
     def prompt_for_lab_award(self, lab=None, award=None):
-        '''Check to see if user submits_for multiple labs or the lab
+        """Check to see if user submits_for multiple labs or the lab
             has multiple awards and if so prompts for the one to set
             for the connection
-        '''
+        """
         if lab:
             if not award:
                 self.set_award(self.lab)
@@ -283,7 +283,7 @@ def is_subobject(field):
         return True
     try:
         return field['items']['type'] == 'object'
-    except:
+    except Exception:
         return False
 
 
@@ -303,7 +303,7 @@ def build_field_list(properties, required_fields=None, no_description=False,
             continue
         if 'submit4dn' in props.get('exclude_from', []):
             continue
-        if ('import_items' in props.get('permission', []) and not admin):
+        if 'import_items' in props.get('permission', []) and not admin:
             continue
         if is_subobject(props) and name != 'attachment':
             if get_field_type(props).startswith('array'):
@@ -391,12 +391,12 @@ def get_uploadable_fields(connection, types, no_description=False,
 
 
 def create_excel(all_fields, filename):
-    '''
+    """
     all_fields being a dictionary of sheet/Item names -> list of FieldInfo(objects)
     create one sheet per dictionary item, that inserts 4 commented header rows for each column
     that corresponds to one of the FieldInfo objects in the list
     header rows are for fieldname, fieldtype, description and comments/enums
-    '''
+    """
     wb = openpyxl.Workbook()
     wb.remove(wb.active)  # removes the by default created empty sheet named Sheet
     # order sheets