Improve environment/config customization (#121)

* use PYPI_INDEX_URL  env var or pypi_index_url in project config

* enable pip/setuptools specific versions in project configration

cob/bootstrapping.py

@@ -7,11 +7,12 @@
 import logbook
 import yaml
 
-from .defs import COB_CONFIG_FILE_NAME, PYPI_INDEX_ENV_VAR
+from .defs import COB_CONFIG_FILE_NAME
 from .utils.develop import is_develop, cob_root
 from .project import get_project
 
 _logger = logbook.Logger(__name__)
+config = get_project().config
 
 _PREVENT_REENTRY_ENV_VAR = 'COB_NO_REENTRY'
 _USE_PRE_ENV_VAR = 'COB_USE_PRE'
@@ -62,21 +63,15 @@ def _ensure_virtualenv():
         else:
             _virtualenv_pip_install([sdist_path])
     else:
-        _logger.trace('Installing cob form Pypi')
+        _logger.trace('Installing cob from Pypi')
         args = ['-U', 'cob']
         if os.environ.get(_COB_VERSION_ENV_VAR):
             version = os.environ[_COB_VERSION_ENV_VAR]
             args[-1] += f'=={version}'
-        if os.environ.get(_USE_PRE_ENV_VAR):
-            args.append('--pre')
-        if PYPI_INDEX_ENV_VAR in os.environ:
-            args.extend(['-i'], os.environ[PYPI_INDEX_ENV_VAR])
+
         _virtualenv_pip_install(args)
 
-    pypi_index_url = get_project().get_pypi_index_url()
     deps = sorted(get_project().get_deps())
-    if pypi_index_url:
-        deps.extend(['-i', pypi_index_url])
     _logger.trace('Installing dependencies: {}', deps)
     if deps:
         _virtualenv_pip_install(['-U', *deps])
@@ -92,6 +87,14 @@ def _create_virtualenv(path):
         interpreter = sys.executable
 
     _execute_long_command([interpreter, '-m', 'virtualenv', path], 'Creating virtualenv')
+    _alter_virtualenv_if_needed()
+
+
+def _alter_virtualenv_if_needed():
+    '''In case a specific pip/setuptools or other basic virtualenv pkgs are needed inside the virtualenv '''
+    if config.get('specific_virtualenv_pkgs'):
+        click.echo("Altering Virtualenv")
+        _virtualenv_pip_install(['-U', *config['specific_virtualenv_pkgs'].split()])
 
 def _locate_original_interpreter():
     if not os.environ.get('COB_FORCE_CURRENT_INTERPRETER'):
@@ -151,7 +154,15 @@ def _execute_long_command(cmd, message):
 
 def _virtualenv_pip_install(argv):
     msg = 'Installing {} in virtualenv'.format(', '.join(arg for arg in argv if not arg.startswith('-')))
-    _execute_long_command([os.path.join(_VIRTUALENV_PATH, 'bin', 'python'), '-m', 'pip', 'install', *argv], msg)
+    _command_lst = [os.path.join(_VIRTUALENV_PATH, 'bin', 'python'), '-m', 'pip', 'install', *argv]
+    pypi_index_url = get_project().get_pypi_index_url()
+    if pypi_index_url:
+        msg += f' - Using PYPI INDEX {pypi_index_url}'
+        _command_lst.extend(['-i', pypi_index_url])
+    if os.environ.get(_USE_PRE_ENV_VAR):
+        msg += f' - Looking for pre-release/development versions'
+        _command_lst.append('--pre')
+    _execute_long_command(_command_lst, msg)
 
 def _reenter():
     if _is_in_project_virtualenv():

cob/cli/docker_cli.py

@@ -57,11 +57,14 @@ def generate_dockerfile():
     else:
         sdist_file_name = None
 
+    _specific_vers = proj.config.get('specific_virtualenv_pkgs', 'pip setuptools')
+
     with open(".Dockerfile", "w") as f:
         f.write(template.render(
             project=proj,
             deployment_base_image='python:3.6-jessie',
             python_version='3.6',
+            specific_vers=_specific_vers,
             is_develop=is_develop(),
             cob_sdist_filename=os.path.basename(sdist_file_name) if sdist_file_name else None,
             cob_root=cob_root() if is_develop() else None,

cob/defs.py

@@ -1,2 +1,2 @@
 COB_CONFIG_FILE_NAME = '.cob-project.yml'
-PYPI_INDEX_ENV_VAR = 'COB_INDEX_URL'
+PYPI_INDEX_ENV_VAR = 'PYPI_INDEX_URL'

cob/project.py

@@ -76,7 +76,10 @@ def get_deps(self):
         return set(self.config.get('deps') or ())
 
     def get_pypi_index_url(self):
-        return self.config.get('pypi_index_url') or os.environ.get(PYPI_INDEX_ENV_VAR)
+        _index = self.config.get(PYPI_INDEX_ENV_VAR.lower()) or os.environ.get(PYPI_INDEX_ENV_VAR)
+        if _index:
+            _logger.trace(f'Using PYPI INDEX URL {_index}')
+        return _index
 
     def initialize(self):
         if self._initialized:

cob/templates/Dockerfile.j2

@@ -21,8 +21,7 @@ RUN echo "deb-src http://nginx.org/packages/mainline/debian/ jessie nginx" | tee
 RUN apt-get update
 RUN apt-get install -y nginx
 
-
-RUN $PYTHON_EXECUTABLE -m pip install -U virtualenv pip setuptools
+RUN $PYTHON_EXECUTABLE -m pip install -U virtualenv {{specific_vers}}
 
 ADD . /app
 RUN rm -rf /app/.cob

doc/configuration.rst

@@ -24,6 +24,11 @@ You can access ``some_value`` through the ``config`` project attribute:
 
 .. note:: Some keys in the project config have special meaning, like the project name stored in the ``name`` key or ``flask_config`` described below. To avoid name clashes, it is wise to store all of your specific configuration under a specific key as a nested structure, such as ``config`` or ``project-config``
 
+Cob Config Options
+------------------
+pypi_index_url - For use with pypi other than https://pypi.org/simple
+specific_virtualenv_pkgs - a string of the form 'pip==19.0.1 setuptools==40.6.3'. this will tell cob to udpate pip/setuptools versions inside cob's virtualenv before installing dependencies.
+
 
 Managing Dependencies
 ---------------------

doc/environment.rst

@@ -28,3 +28,10 @@ You can install additional dependencies through the ``deps`` section of the ``.c
   deps:
     - Flask-Security
     - Flask-SQLAlchemy>=0.1.0
+
+Environment Variables
+---------------------
+PYPI_INDEX_URL - Makes cob use an alternate pypi registry/index.
+COB_VERSION - Allows setting a specific version of Cob to be used.
+COB_USE_PRE - make cob run `pip install --pre`. affects testserver virtualenv, docker build during deployment.
+COB_DEVELOP - Indicates a development version of cob is used, affecting bootstrapping, dockerfile.