Merge f4280ef into e32e64b

dcicutils/deployment_utils.py

@@ -18,20 +18,45 @@ def main():
 
 """
 
+import argparse
 import datetime
 import glob
 import io
 import json
+import logging
 import os
 import pkg_resources
 import re
 import subprocess
 import sys
 import toml
-import argparse
 
-from dcicutils.env_utils import get_standard_mirror_env, data_set_for_env, get_bucket_env, INDEXER_ENVS
-from dcicutils.misc_utils import PRINT
+from .env_utils import get_standard_mirror_env, data_set_for_env, get_bucket_env, INDEXER_ENVS
+from .misc_utils import PRINT
+
+
+def boolean_setting(settings, key, default=None):
+    """
+    Given a setting from a .ini file and returns a boolean interpretation of it.
+    - Treats an actual python boolean as itself
+    - Using case-insensitive comparison, treats the string 'true' as True and both 'false' and '' as False.
+    - If an element is missing,
+    - Treats None the same as the option being missing, so returns the given default.
+    - Raises an error for any non-standard value. This is harsh, but it should not happen.
+    """
+    if key not in settings:
+        return default
+    setting = settings[key]
+    if isinstance(setting, str):
+        setting_lower = setting.lower()
+        if setting_lower in ("", "false"):
+            return False
+        elif setting_lower == "true":
+            return True
+        else:
+            return setting
+    else:  # booleans, None, and odd types (though we might want to consider making other types an error).
+        return setting
 
 
 class Deployer:
@@ -287,11 +312,12 @@ def main(cls):
                                 help="an ElasticSearch namespace",
                                 default=None)
             parser.add_argument("--indexer",
-                                help="whether this system does indexing",
-                                action='store_true',
-                                default=False)
+                                help="whether this server does indexing at all",
+                                choices=["true", "false"],
+                                default=None)
             parser.add_argument("--index_server",
-                                help="whether this is a standalone indexing server",
+                                help="whether this is a standalone indexing server, only doing indexing",
+                                choices=["true", "false"],
                                 default=None)
             args = parser.parse_args()
             template_file_name = (cls.any_environment_template_filename()

dcicutils/lang_utils.py

@@ -1,6 +1,8 @@
-from .misc_utils import ignored
+import datetime
 import re
 
+from .misc_utils import ignored
+
 
 class EnglishUtils:
     """
@@ -49,6 +51,12 @@ def _adjust_ending(cls, word, strip_chars, add_suffix):
 
     @classmethod
     def string_pluralize(cls, word: str) -> str:
+        """
+        Returns the probable plural of the given word.
+        This is an ad hoc string pluralizer intended for situations where being mostly right is good enough.
+        e.g., string_pluralize('sample') => 'sample'
+              string_pluralize('community') => 'communities'
+        """
         charn = word[-1]
         char1 = word[0]
         capitalize = char1.isupper()
@@ -92,15 +100,31 @@ def string_pluralize(cls, word: str) -> str:
 
     @classmethod
     def select_a_or_an(cls, word):
+        """
+        Uses a heuristic to try to select the appropriate article ('a' or 'an') for a given English noun.
+        select_a_or_an("gene") => 'a'
+        select_a_or_an("accession") => 'an'
+        """
+
         return "a" if cls._PREFIX_PATTERN_FOR_A.match(word) else "an"
 
     @classmethod
     def a_or_an(cls, word):
+        """
+        Heuristically attaches either "a" or "an" to a given English noun.
+        a_or_an("gene") => "a gene"
+        a_or_an("accession") => "an accession"
+        """
         article = cls.select_a_or_an(word)
         return "%s %s" % (article, word)
 
     @classmethod
     def n_of(cls, n, thing, num_format=None):
+        """
+        Given a number and a noun, returns the name for that many of that noun.
+        e.g., n_of(7, "variant") => "7 variants"
+              n_of(1, "accession") => "1 accession"
+        """
         display_n = n
         if num_format:
             res = num_format(n, thing)
@@ -118,7 +142,13 @@ def _time_count_formatter(cls, n, unit):
 
     @classmethod
     def relative_time_string(cls, seconds, detailed=True):
+        """
+        Given a number of seconds, expresses that number of seconds in English.
+        The seconds can be expressed either as a number or a datetime.timedelta.
+        """
         result = []
+        if isinstance(seconds, datetime.timedelta):
+            seconds = seconds.total_seconds()
         remaining_seconds = seconds
         units_seen = False
         for unit_info in cls._TIME_UNITS:

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "0.21.0"
+version = "0.22.0"
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

test/test_deployment_utils.py

@@ -1,16 +1,16 @@
 import datetime
 import io
 import os
-
 import pytest
 import re
 import subprocess
+import sys
 
 from contextlib import contextmanager
 from io import StringIO
 from unittest import mock
 
-from dcicutils.deployment_utils import Deployer
+from dcicutils.deployment_utils import Deployer, boolean_setting
 from dcicutils.env_utils import is_cgap_env
 from dcicutils.misc_utils import ignored
 from dcicutils.qa_utils import override_environ
@@ -684,3 +684,70 @@ def check(self, line):
                                es_server="search-fourfront-webdev-5uqlmdvvshqew46o46kcc2lxmy.%s" % us_east,
                                line_checker=Checker(expect_indexer=None,
                                                     expect_index_server="true"))
+
+
+def test_deployment_utils_main():
+
+    fake_template = "something.ini"
+    with override_environ(ENV_NAME='fourfront-foo'):
+        with mock.patch.object(Deployer, "build_ini_file_from_template") as mock_build:
+            with mock.patch.object(Deployer, "environment_template_filename", return_value=fake_template):
+                with mock.patch.object(Deployer, "template_environment_names", return_value=["something, foo"]):
+
+                    def check_for_mocked_build(expected_kwargs=None, expected_code=0):
+                        def mocked_build(*args, **kwargs):
+                            assert args == (fake_template, 'production.ini')
+                            assert kwargs == (expected_kwargs or {})
+                        mock_build.side_effect = mocked_build
+                        try:
+                            Deployer.main()
+                        except SystemExit as e:
+                            assert e.code == expected_code
+
+                    with mock.patch.object(sys, "argv", ['']):
+                        check_for_mocked_build({
+                            'bs_env': None,
+                            'bs_mirror_env': None,
+                            'data_set': None,
+                            'es_namespace': None,
+                            'es_server': None,
+                            'index_server': None,
+                            'indexer': None,
+                            's3_bucket_env': None
+                        })
+
+                    with mock.patch.object(sys, "argv", ['', '--indexer', 'false', '--index_server', 'true']):
+                        check_for_mocked_build({
+                            'bs_env': None,
+                            'bs_mirror_env': None,
+                            'data_set': None,
+                            'es_namespace': None,
+                            'es_server': None,
+                            'index_server': 'true',
+                            'indexer': 'false',
+                            's3_bucket_env': None
+                        })
+
+                    with mock.patch.object(sys, "argv", ['', '--indexer', 'foo']):
+                        with pytest.raises(Exception):
+                            check_for_mocked_build({
+                                'bs_env': None,
+                                'bs_mirror_env': None,
+                                'data_set': None,
+                                'es_namespace': None,
+                                'es_server': None,
+                                'index_server': 'true',
+                                'indexer': 'false',
+                                's3_bucket_env': None
+                            })
+
+
+def test_deployment_utils_boolean_setting():
+
+    assert boolean_setting({'foo': 'true'}, 'foo') == True
+    assert boolean_setting({'foo': 'false'}, 'foo') == False
+    assert boolean_setting({'foo': ''}, 'foo') == False
+    assert boolean_setting({'foo': None}, 'foo') == None
+    assert boolean_setting({'foo': 'maybe'}, 'foo') == 'maybe'
+    assert boolean_setting({}, 'foo') == None
+    assert boolean_setting({}, 'foo', default='surprise') == 'surprise'

test/test_lang_utils.py

@@ -1,3 +1,5 @@
+import datetime
+
 from dcicutils.lang_utils import EnglishUtils, a_or_an, select_a_or_an, string_pluralize
 
 
@@ -92,6 +94,12 @@ def test(seconds, long_string, short_string):
     test(4 * DAY + 1 * HOUR + 5.2 * SECOND, "4 days, 1 hour, 5.2 seconds", "4 days, 1 hour")
     test(5.2 * SECOND, "5.2 seconds", "5.2 seconds")
 
+    relative_time = datetime.timedelta(hours=1, seconds=3)
+    test(relative_time, "1 hour, 3 seconds", "1 hour")
+    t1 = datetime.datetime.now()
+    t2 = t1 + relative_time
+    test(t2 - t1, "1 hour, 3 seconds", "1 hour")
+
 
 def test_time_count_formatter():