Merge d2e3310 into 928b038

dcicutils/beanstalk_utils.py

@@ -8,6 +8,7 @@
 import os
 import json
 import requests
+import sys
 import time
 from datetime import datetime
 from dcicutils import ff_utils
@@ -17,11 +18,19 @@
 logger = logging.getLogger('logger')
 logger.setLevel(logging.INFO)
 
-# input vs. raw_input for python 2/3
-try:
-    use_input = raw_input
-except NameError:
-    use_input = input
+# In Python 2, the safe 'input' function was called 'raw_input'.  Also in Python 2, there was a function
+# named 'input' that did eval(raw_input(...)).  Python 3 made an incompatible change, renaming 'raw_input'
+# to 'input', and it no longer has a function that does an unsafe eval.  When we supported both Python 2 & 3,
+# use a 'try' expression to sort things out and call the safe function 'use_input' to avoid confusion.
+# But PyCharm found that 'try' expression confusing, so now that we are Python 3 only, we're phasing that
+# out. For a time, we'll retain the transitional naming, though, along with an affirmative error check, so
+# we don't open any security holes. We can remove this naming and check once we're we're only using Python 3.
+# -kmp 27-Mar-2020
+
+_python_major_version = sys.version_info[0]
+if _python_major_version < 3:
+    raise EnvironmentError("The 'dcicutils.beanstalk_utils' package only works in Python 3.")
+use_input = input  # In Python 3, this does 'safe' input reading.
 
 FOURSIGHT_URL = 'https://foursight.4dnucleome.org/'
 # magic CNAME corresponds to data.4dnucleome
@@ -650,6 +659,28 @@ def create_bs(envname, load_prod, db_endpoint, es_url, for_indexing=False):
     return res
 
 
+# location of environment variables on elasticbeanstalk
+BEANSTALK_ENV_PATH = "/opt/python/current/env"
+
+
+def source_beanstalk_env_vars(config_file=BEANSTALK_ENV_PATH):
+    """
+    set environment variables if we are on Elastic Beanstalk
+    AWS_ACCESS_KEY_ID is indicative of whether or not env vars are sourced
+
+    Args:
+        config_file (str): filepath to load env vars from
+    """
+    if os.path.exists(config_file) and not os.environ.get("AWS_ACCESS_KEY_ID"):
+        command = ['bash', '-c', 'source ' + config_file + ' && env']
+        proc = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True)
+        for line in proc.stdout:
+            key, _, value = line.partition("=")
+            print("key=", key, "value=", value)
+            os.environ[key] = value[:-1]
+        proc.communicate()
+
+
 def log_to_foursight(event, lambda_name='', overrides=None):
     """
     Use Foursight as a logging tool within in a lambda function by doing a PUT

dcicutils/env_utils.py

@@ -1,8 +1,60 @@
+import os
+
+
 FOURFRONT_STG_OR_PRD_TOKENS = ['webprod', 'blue', 'green']
 FOURFRONT_STG_OR_PRD_NAMES = ['staging', 'stagging', 'data']
 CGAP_STG_OR_PRD_TOKENS = []
 CGAP_STG_OR_PRD_NAMES = ['fourfront-cgap', 'fourfront-cgap-green', 'fourfront-cgap-blue']
 
+FF_ENV_HOTSEAT = 'fourfront-hotseat'
+FF_ENV_MASTERTEST = 'fourfront-mastertest'
+FF_ENV_PRODUCTION_BLUE = 'fourfront-blue'
+FF_ENV_PRODUCTION_GREEN = 'fourfront-green'
+FF_ENV_STAGING = 'fourfront-staging'
+FF_ENV_WEBDEV = 'fourfront-webdev'
+FF_ENV_WEBPROD = 'fourfront-webprod'
+FF_ENV_WEBPROD2 = 'fourfront-webprod2'
+FF_ENV_WOLF = 'fourfront-wolf'
+
+CGAP_ENV_HOTSEAT = 'fourfront-cgaphot'
+CGAP_ENV_MASTERTEST = 'fourfront-cgaptest'
+CGAP_ENV_PRODUCTION_BLUE = 'fourfront-cgapblue'
+CGAP_ENV_PRODUCTION_GREEN = 'fourfront-cgapgreen'
+CGAP_ENV_STAGING = 'fourfront-cgapstaging'
+CGAP_ENV_WEBDEV = 'fourfront-cgapdev'
+CGAP_ENV_WEBPROD = 'fourfront-cgap'
+# CGAP_ENV_WEBPROD2 doesn't have meaning in old CGAP naming. See ENV_STAGING.
+CGAP_ENV_WOLF = 'fourfront-cgapwolf'
+
+
+# These operate as pairs. Don't add extras.
+BEANSTALK_PROD_MIRRORS = {
+
+    FF_ENV_PRODUCTION_BLUE: FF_ENV_PRODUCTION_GREEN,
+    FF_ENV_PRODUCTION_GREEN: FF_ENV_PRODUCTION_BLUE,
+    FF_ENV_WEBPROD: FF_ENV_WEBPROD2,
+    FF_ENV_WEBPROD2: FF_ENV_WEBPROD,
+
+    CGAP_ENV_PRODUCTION_BLUE: CGAP_ENV_PRODUCTION_GREEN,
+    CGAP_ENV_PRODUCTION_GREEN: CGAP_ENV_PRODUCTION_BLUE,
+    CGAP_ENV_WEBPROD: None,
+
+}
+
+BEANSTALK_TEST_ENVS = [
+
+    FF_ENV_HOTSEAT,
+    FF_ENV_MASTERTEST,
+    FF_ENV_WEBDEV,
+    FF_ENV_WOLF,
+
+    CGAP_ENV_HOTSEAT,
+    CGAP_ENV_MASTERTEST,
+    CGAP_ENV_WEBDEV,
+    CGAP_ENV_WOLF,
+
+]
+
 
 def blue_green_mirror_env(envname):
     """
@@ -47,3 +99,48 @@ def is_stg_or_prd_env(envname):
     elif any(token in envname for token in stg_or_prd_tokens):
         return True
     return False
+
+
+def is_test_env(envname):
+    return envname in BEANSTALK_TEST_ENVS
+
+
+def is_hotseat_env(envname):
+    return 'hot' in envname
+
+
+def get_env_from_context(settings, allow_environ=False):
+    if allow_environ:
+        environ_env_name = os.environ.get('ENV_NAME')
+        if environ_env_name:
+            return environ_env_name
+    return settings.get('env.name')
+
+
+def get_mirror_env_from_context(settings, allow_environ=False, allow_guess=True, ):
+    # TODO: I added allow_environ featurism here but did not yet enable it.
+    #       Want to talk to Will about whether we should consider that a compatible or breaking hcange.
+    #       -kmp 27-Mar-2020
+    """
+    Figures out who the mirror beanstalk Env is if applicable
+    This is important in our production environment because in our
+    blue-green deployment we maintain two elasticsearch intances that
+    must be up to date with each other.
+    """
+    if allow_environ:
+        environ_mirror_env_name = os.environ.get('MIRROR_ENV_NAME')
+        if environ_mirror_env_name:
+            return environ_mirror_env_name
+    declared = settings.get('mirror.env.name', '')
+    if declared:
+        return declared
+    elif allow_guess:
+        who_i_am = get_env_from_context(settings, allow_environ=allow_environ)
+        return guess_mirror_env(who_i_am)
+    else:
+        return None
+
+
+def guess_mirror_env(envname):
+    # TODO: Should this be BEANSTALK_PROD_MIRRORS.get(envname) or blue_green_mirror_env(envname)
+    return BEANSTALK_PROD_MIRRORS.get(envname)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "0.11.0"
+version = "0.12.0b1"
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["William Ronchetti <william_ronchetti@hms.harvard.edu>"]
 license = "MIT"

test/test_beanstalk_utils.py

@@ -1,4 +1,6 @@
-from dcicutils import beanstalk_utils as bs
+import io
+import os
+from dcicutils import beanstalk_utils as bs, source_beanstalk_env_vars
 from unittest import mock
 
 
@@ -21,3 +23,56 @@ def test_get_beanstalk_normal_url():
         man_not_hot.return_value = {'CNAME': 'take-of-your-jacket'}
         url = bs.get_beanstalk_real_url('take-of-your-jacket')
         assert url == 'http://take-of-your-jacket'
+
+
+def _mock_not_called(name):
+    def mock_not_called(*args, **kwargs):
+        raise AssertionError("%s was called where not expected." % name)
+    return mock_not_called
+
+
+def test_source_beanstalk_env_vars_no_config_file():
+    # subprocess.Popen gets called only if config file exists and AWS_ACCESS_KEY_ID environment variable does not.
+    # This tests that if config file does not exist and AWS_ACCESS_KEY_ID does not, it doesn't get called.
+    with mock.patch("os.path.exists") as mock_exists:
+        with mock.patch.object(os, "environ", {}):
+            with mock.patch("subprocess.Popen") as mock_popen:
+                mock_exists.return_value = False
+                mock_popen = _mock_not_called("subprocess.Popen")
+                source_beanstalk_env_vars()
+
+
+def test_source_beanstalk_env_vars_aws_access_key_id():
+    # subprocess.Popen gets called only if config file exists and AWS_ACCESS_KEY_ID environment variable does not.
+    # This tests that if config file exists and AWS_ACCESS_KEY_ID does, it doesn't get called.
+    with mock.patch("os.path.exists") as mock_exists:
+        with mock.patch.object(os, "environ", {"AWS_ACCESS_KEY_ID": "something"}):
+            with mock.patch("subprocess.Popen") as mock_popen:
+                mock_exists.return_value = True
+                mock_popen.side_effect = _mock_not_called("subprocess.Popen")
+                source_beanstalk_env_vars()
+
+
+def test_source_beanstalk_env_vars_normal():
+    # subprocess.Popen gets called only if config file exists and AWS_ACCESS_KEY_ID environment variable does not.
+    # In the normal case, both of those conditions are true, and so it opens the file and parses it,
+    # setting os.environ to hold the relevant values.
+    with mock.patch("os.path.exists") as mock_exists:
+        fake_env = {}
+        with mock.patch.object(os, "environ", fake_env):
+            with mock.patch("subprocess.Popen") as mock_popen:
+                mock_exists.return_value = True
+                class FakeSubprocessPipe:
+                    def __init__(self, *args, **kwargs):
+                        self.stdout = io.StringIO(
+                            'AWS_ACCESS_KEY_ID=12345\n'
+                            'AWS_FAKE_SECRET=amazon\n'
+                        )
+                    def communicate(self):
+                        pass
+                mock_popen.side_effect = FakeSubprocessPipe
+                source_beanstalk_env_vars()
+                assert fake_env == {
+                    'AWS_ACCESS_KEY_ID': '12345',
+                    'AWS_FAKE_SECRET': 'amazon'
+                }