Add reticulate support for conda

R/bundle.R

@@ -171,7 +171,8 @@ bundleFiles <- function(appDir) {
 }
 
 bundleApp <- function(appName, appDir, appFiles, appPrimaryDoc, assetTypeName,
-                      contentCategory, verbose = FALSE, python = NULL) {
+                      contentCategory, verbose = FALSE, python = NULL,
+                      compatibilityMode = F, forceGenerate = F) {
   logger <- verboseLogger(verbose)
 
   logger("Inferring App mode and parameters")
@@ -216,6 +217,8 @@ bundleApp <- function(appName, appDir, appFiles, appPrimaryDoc, assetTypeName,
       appPrimaryDoc = appPrimaryDoc,
       assetTypeName = assetTypeName,
       users = users,
+      compatibilityMode = compatibilityMode,
+      forceGenerate = forceGenerate,
       python = python,
       hasPythonRmd = hasPythonRmd)
   manifestJson <- enc2utf8(toJSON(manifest, pretty = TRUE))
@@ -263,12 +266,23 @@ bundleApp <- function(appName, appDir, appFiles, appPrimaryDoc, assetTypeName,
 #'   If python = NULL, and RETICULATE_PYTHON is set in the environment,
 #'   its value will be used.
 #'
+#' @param forceGeneratePythonEnvironment Optional. If an existing
+#'   `requirements.txt` or `environment.yml` file is found, it will
+#'   be overwritten when this argument is `TRUE`.
+#'
+#' @param forceRequirementsTxtEnvironment Optional. If rsconnect
+#'   detects you are running in a conda environment, it will write
+#'   `requirements.txt` instead of `environment.yml` when this
+#'   argument is `TRUE`.
+#'
 #' @export
 writeManifest <- function(appDir = getwd(),
                           appFiles = NULL,
                           appPrimaryDoc = NULL,
                           contentCategory = NULL,
-                          python = NULL) {
+                          python = NULL,
+                          forceGeneratePythonEnvironment = F,
+                          forceRequirementsTxtEnvironment = F) {
   if (is.null(appFiles)) {
     appFiles <- bundleFiles(appDir)
   } else {
@@ -310,6 +324,8 @@ writeManifest <- function(appDir = getwd(),
       appPrimaryDoc = appPrimaryDoc,
       assetTypeName = "content",
       users = NULL,
+      compatibilityMode = forceRequirementsTxtEnvironment,
+      forceGenerate = forceGeneratePythonEnvironment,
       python = python,
       hasPythonRmd = hasPythonRmd)
   manifestJson <- enc2utf8(toJSON(manifest, pretty = TRUE))
@@ -503,10 +519,15 @@ inferDependencies <- function(appMode, hasParameters, python, hasPythonRmd) {
   unique(deps)
 }
 
-inferPythonEnv <- function(workdir, python) {
+inferPythonEnv <- function(workdir, python, compatibilityMode, forceGenerate) {
   # run the python introspection script
   env_py <- system.file("resources/environment.py", package = "rsconnect")
-  args <- c(shQuote(env_py), shQuote(workdir))
+  args <- c(shQuote(env_py))
+  if (compatibilityMode || forceGenerate) {
+    flags <- paste('-', ifelse(compatibilityMode, 'c', ''), ifelse(forceGenerate, 'f', ''), sep = '')
+    args <- c(args, flags)
+  }
+  args <- c(args, shQuote(workdir))
 
   tryCatch({
     output <- system2(command = python, args = args, stdout = TRUE, stderr = NULL, wait = TRUE)
@@ -530,8 +551,8 @@ inferPythonEnv <- function(workdir, python) {
 }
 
 createAppManifest <- function(appDir, appMode, contentCategory, hasParameters,
-                              appPrimaryDoc, assetTypeName, users, python = NULL,
-                              hasPythonRmd = FALSE) {
+                              appPrimaryDoc, assetTypeName, users, compatibilityMode,
+                              forceGenerate, python = NULL, hasPythonRmd = FALSE) {
 
   # provide package entries for all dependencies
   packages <- list()
@@ -551,9 +572,9 @@ createAppManifest <- function(appDir, appMode, contentCategory, hasParameters,
       name <- deps[i, "Package"]
 
       if (name == "reticulate" && !is.null(python)) {
-        pyInfo <- inferPythonEnv(appDir, python)
+        pyInfo <- inferPythonEnv(appDir, python, compatibilityMode, forceGenerate)
         if (is.null(pyInfo$error)) {
-          # write the package list into requirements.txt file in the bundle dir
+          # write the package list into requirements.txt/environment.yml file in the bundle dir
           packageFile <- file.path(appDir, pyInfo$package_manager$package_file)
           cat(pyInfo$package_manager$contents, file=packageFile, sep="\n")
           pyInfo$package_manager$contents <- NULL

R/deployApp.R

@@ -64,6 +64,13 @@
 #'   If python = NULL, and RETICULATE_PYTHON is set in the environment, its
 #'   value will be used. The specified python binary will be invoked to determine
 #'   its version and to list the python packages installed in the environment.
+#' @param forceGeneratePythonEnvironment Optional. If an existing
+#'   `requirements.txt` or `environment.yml` file is found, it will
+#'   be overwritten when this argument is `TRUE`.
+#' @param forceRequirementsTxtEnvironment Optional. If rsconnect
+#'   detects you are running in a conda environment, it will write
+#'   `requirements.txt` instead of `environment.yml` when this
+#'   argument is `TRUE`.
 #' @examples
 #' \dontrun{
 #'
@@ -109,7 +116,9 @@ deployApp <- function(appDir = getwd(),
                       metadata = list(),
                       forceUpdate = getOption("rsconnect.force.update.apps", FALSE),
                       python = NULL,
-                      on.failure = NULL) {
+                      on.failure = NULL,
+                      forceGeneratePythonEnvironment = F,
+                      forceRequirementsTxtEnvironment = F) {
 
   if (!isStringParam(appDir))
     stop(stringParamErrorMessage("appDir"))
@@ -354,7 +363,8 @@ deployApp <- function(appDir = getwd(),
       # python is enabled on Connect but not on Shinyapps
       python <- getPythonForTarget(python, accountDetails)
       bundlePath <- bundleApp(target$appName, appDir, appFiles,
-                              appPrimaryDoc, assetTypeName, contentCategory, verbose, python)
+                              appPrimaryDoc, assetTypeName, contentCategory, verbose, python,
+                              forceRequirementsTxtEnvironment, forceGeneratePythonEnvironment)
 
       if (isShinyapps(accountDetails)) {
 

inst/resources/environment.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python
+import datetime
 import json
 import locale
 import os
 import re
 import subprocess
 import sys
 
-
 version_re = re.compile(r'\d+\.\d+(\.\d+)?')
 exec_dir = os.path.dirname(sys.executable)
 
@@ -15,33 +15,78 @@ class EnvironmentException(Exception):
     pass
 
 
-def detect_environment(dirname):
+def detect_environment(dirname, force_generate=False, compatibility_mode=False, conda=None):
     """Determine the python dependencies in the environment.
 
     `pip freeze` will be used to introspect the environment.
 
     Returns a dictionary containing the package spec filename
     and contents if successful, or a dictionary containing 'error'
     on failure.
+    :param: dirname Directory name
+    :param: force_generate Force the generation of an environment
+    :param: compatibility_mode Force the usage of `pip freeze` for older
+    connect versions which do not support conda.
     """
-    result = (output_file(dirname, 'requirements.txt', 'pip') or
-              pip_freeze(dirname))
+    if not compatibility_mode:
+        conda = get_conda(conda)
+    if conda:
+        if force_generate:
+            result = conda_env_export(conda)
+        else:
+            result = (output_file(dirname, 'environment.yml', 'conda')
+                      or conda_env_export(conda))
+    else:
+        if force_generate:
+            result = pip_freeze()
+        else:
+            result = (output_file(dirname, 'requirements.txt', 'pip')
+                      or pip_freeze())
 
     if result is not None:
         result['python'] = get_python_version()
         result['pip'] = get_version('pip')
+        if conda:
+            result['conda'] = get_conda_version(conda)
         result['locale'] = get_default_locale()
 
     return result
 
 
+def get_conda(conda=None):
+    """get_conda tries to find the conda executable if we're in
+    a conda environment. If not, or if we cannot find the executable,
+    return None.
+    :returns: conda string path to conda or None.
+    """
+    if os.environ.get('CONDA_PREFIX', None) is None and conda is None:
+        return None
+    else:
+        return conda or os.environ.get('CONDA_EXE', None)
+
+
 def get_python_version():
     v = sys.version_info
     return "%d.%d.%d" % (v[0], v[1], v[2])
 
 
-def get_default_locale():
-    return '.'.join(locale.getdefaultlocale())
+def get_conda_version(conda):
+    try:
+        args = [conda, '-V']
+        proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+        stdout, stderr = proc.communicate()
+        match = version_re.search(stdout)
+        if match:
+            return match.group()
+        msg = "Failed to get version of conda from the output of: %s" % (' '.join(args))
+        raise EnvironmentException(msg)
+    except Exception as exception:
+        raise EnvironmentException("Error getting conda version: %s" % str(exception))
+
+
+def get_default_locale(locale_source=locale.getdefaultlocale):
+    result = '.'.join([item or '' for item in locale_source()])
+    return '' if result == '.' else result
 
 
 def get_version(module):
@@ -55,8 +100,8 @@ def get_version(module):
 
         msg = "Failed to get version of '%s' from the output of: %s" % (module, ' '.join(args))
         raise EnvironmentException(msg)
-    except Exception as exc:
-        raise EnvironmentException("Error getting '%s' version: %s" % (module, str(exc)))
+    except Exception as exception:
+        raise EnvironmentException("Error getting '%s' version: %s" % (module, str(exception)))
 
 
 def output_file(dirname, filename, package_manager):
@@ -75,19 +120,19 @@ def output_file(dirname, filename, package_manager):
             data = f.read()
 
         data = '\n'.join([line for line in data.split('\n')
-                                if 'rsconnect' not in line])
+                          if 'rsconnect' not in line])
 
         return {
             'filename': filename,
             'contents': data,
             'source': 'file',
             'package_manager': package_manager,
         }
-    except Exception as exc:
-        raise EnvironmentException('Error reading %s: %s' % (filename, str(exc)))
+    except Exception as exception:
+        raise EnvironmentException('Error reading %s: %s' % (filename, str(exception)))
 
 
-def pip_freeze(dirname):
+def pip_freeze():
     """Inspect the environment using `pip freeze`.
 
     Returns a dictionary containing the filename
@@ -101,8 +146,8 @@ def pip_freeze(dirname):
 
         pip_stdout, pip_stderr = proc.communicate()
         pip_status = proc.returncode
-    except Exception as exc:
-        raise EnvironmentException('Error during pip freeze: %s' % str(exc))
+    except Exception as exception:
+        raise EnvironmentException('Error during pip freeze: %s' % str(exception))
 
     if pip_status != 0:
         msg = pip_stderr or ('exited with code %d' % pip_status)
@@ -111,6 +156,8 @@ def pip_freeze(dirname):
     pip_stdout = '\n'.join([line for line in pip_stdout.split('\n')
                             if 'rsconnect' not in line])
 
+    pip_stdout = '# requirements.txt generated by rsconnect-python on '+str(datetime.datetime.utcnow())+'\n'+pip_stdout
+
     return {
         'filename': 'requirements.txt',
         'contents': pip_stdout,
@@ -119,13 +166,52 @@ def pip_freeze(dirname):
     }
 
 
-if __name__ == '__main__':
+def conda_env_export(conda):
+    """Inspect the environment using `conda env export`
+    :param: conda path to the `conda` tool
+    :return: dictionary containing the key "environment.yml" and the data inside
+    """
     try:
-        if len(sys.argv) < 2:
-            raise EnvironmentException('Usage: %s DIRECTORY' % sys.argv[0])
+        proc = subprocess.Popen(
+            [conda, 'env', 'export'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+        conda_stdout, conda_stderr = proc.communicate()
+        conda_status = proc.returncode
+    except Exception as exception:
+        raise EnvironmentException('Error during conda env export: %s' % str(exception))
+
+    if conda_status != 0:
+        msg = conda_stderr or ('exited with code %d' % conda_status)
+        raise EnvironmentException('Error during conda env export: %s' % msg)
+
+    return {
+        'filename': 'environment.yml',
+        'contents': conda_stdout,
+        'source': 'conda_env_export',
+        'package_manager': 'conda'
+    }
+
 
-        result = detect_environment(sys.argv[1])
-    except EnvironmentException as exc:
-        result = dict(error=str(exc))
+def main():
+    try:
+        if len(sys.argv) < 2:
+            raise EnvironmentException('Usage: %s [-fc] DIRECTORY' % sys.argv[0])
+        # directory is always the last argument
+        directory = sys.argv[len(sys.argv)-1]
+        flags = ''
+        force_generate = False
+        compatibility_mode = False
+        if len(sys.argv) > 2:
+            flags = sys.argv[1]
+        if 'f' in flags:
+            force_generate = True
+        if 'c' in flags:
+            compatibility_mode = True
+        result = detect_environment(directory, force_generate, compatibility_mode)
+    except EnvironmentException as exception:
+        result = dict(error=str(exception))
 
     json.dump(result, sys.stdout, indent=4)
+
+
+if __name__ == '__main__':
+    main()