Merge pull request #594 from rstudio/update-v2.16

Updates for TF v2.16 and keras3

.github/ISSUE_TEMPLATE/installation-issue.md

@@ -14,7 +14,7 @@ install.packages("pak")
 pak::pak(sprintf("rstudio/%s", c("reticulate", "tensorflow", "keras")))
 if (is.null(reticulate::virtualenv_starter()))
    reticulate::install_python()
-keras::install_keras()
+tensorflow::install_tensorflow()
 ```
 
 Test to see if installation was successful.

.github/workflows/R-CMD-check.yaml

@@ -33,18 +33,18 @@ jobs:
           - {os: 'ubuntu-latest' , tf: 'default', r: 'oldrel'}
           - {os: 'ubuntu-latest' , tf: 'default', r: 'oldrel-1'}
           - {os: 'ubuntu-latest' , tf: 'default', r: 'oldrel-2'}
-          - {os: 'ubuntu-20.04'  , tf: 'default', r: '3.6'} # default R in ubuntu-20.04
+          - {os: 'ubuntu-latest' , tf: 'default', r: 'oldrel-3'}
 
           ## release keras/tf version (if different from 'default')
           # - {os: 'ubuntu-latest' , tf: 'release', r: 'release'}
           # - {os: 'windows-latest', tf: 'release', r: 'release'}
           # - {os: 'macOS-latest'  , tf: 'release', r: 'release'}
 
           ## old keras/tf versions
+          - {os: 'ubuntu-latest', tf: '2.15', r: 'release'}
           - {os: 'ubuntu-latest', tf: '2.14', r: 'release'}
           - {os: 'ubuntu-latest', tf: '2.13', r: 'release'}
           - {os: 'ubuntu-latest', tf: '2.12', r: 'release'}
-          - {os: 'ubuntu-latest', tf: '2.11', r: 'release'}
           - {os: 'ubuntu-latest', tf: '2.9' , r: 'release'}
 
           # these are allowed to fail
@@ -73,24 +73,24 @@ jobs:
           use-public-rspm: true
           Ncpus: '2L'
 
-      - name: Get Date
-        id: get-date
-        shell: bash
-        run: |
-          echo "::set-output name=year-week::$(date -u "+%Y-%U")"
-          echo "::set-output name=date::$(date -u "+%F")"
-
-      - name: Restore R package cache
-        uses: actions/cache@v2
-        id: r-package-cache
-        with:
-          path: ${{ env.R_LIBS_USER }}
-          key: ${{ matrix.os }}-${{ steps.setup-r.outputs.installed-r-version }}-${{ steps.get-date.outputs.year-week }}-2
+      # - name: Get Date
+      #   id: get-date
+      #   shell: bash
+      #   run: |
+      #     echo "::set-output name=year-week::$(date -u "+%Y-%U")"
+      #     echo "::set-output name=date::$(date -u "+%F")"
+      #
+      # - name: Restore R package cache
+      #   uses: actions/cache@v2
+      #   id: r-package-cache
+      #   with:
+      #     path: ${{ env.R_LIBS_USER }}
+      #     key: ${{ matrix.os }}-${{ steps.setup-r.outputs.installed-r-version }}-${{ steps.get-date.outputs.year-week }}-2
 
       - uses: r-lib/actions/setup-r-dependencies@v2
         with:
           extra-packages: any::rcmdcheck local::.
-          cache-version: 1
+          cache-version: 4
           upgrade: 'TRUE'
 
       - name: Install TensorFlow

DESCRIPTION

@@ -50,7 +50,7 @@ Imports:
 Roxygen: list(markdown = TRUE)
 Suggests:
     testthat (>= 2.1.0),
-    keras,
+    keras3,
     pillar,
     withr,
     callr

NAMESPACE

@@ -103,6 +103,9 @@ S3method(print,tensorflow.tensor)
 S3method(print,tensorflow_config)
 S3method(prod,tensorflow.tensor)
 S3method(py_str,tensorflow.python.framework.tensor_shape.TensorShape)
+S3method(py_to_r,keras.src.utils.tracking.TrackedDict)
+S3method(py_to_r,keras.src.utils.tracking.TrackedList)
+S3method(py_to_r,keras.src.utils.tracking.TrackedSet)
 S3method(py_to_r,tensorflow.python.trackable.data_structures.ListWrapper)
 S3method(py_to_r,tensorflow.python.trackable.data_structures._DictWrapper)
 S3method(py_to_r,tensorflow.python.training.tracking.data_structures.ListWrapper)

NEWS.md

@@ -1,5 +1,11 @@
 # tensorflow (development version)
 
+- The package now Suggest 'keras3' instead of 'keras'
+- `install_tensorflow()` installs TensorFlow v2.16 by default.
+- If `install_tensorflow()` detects a GPU on Linux, it will automatically 
+  install the cuda package and configure required symlinks for cudnn.
+
+
 # tensorflow 2.15.0
 
 - `install_tensorflow()` installs TensorFlow v2.15 by default

R/estimator-generics.R

@@ -20,14 +20,14 @@ train <- function(object, ...) {
 #' Evaluate a Model
 #'
 #' Evaluate a model object. See implementations in the
-#' [keras][keras::evaluate.keras.engine.training.Model()] package.
+#' [keras3][keras3::evaluate.keras.src.models.model.Model()] package.
 #'
 #' @param object An evaluatable \R object.
 #' @param ... Optional arguments passed on to implementing methods.
 #'
 #' @section Implementations:
 #'
-#'   - [keras][keras::evaluate.keras.engine.training.Model()]
+#'   - [keras3][keras3::evaluate.keras.src.models.model.Model()]
 #'
 #' @export
 evaluate <- function(object, ...) {
@@ -54,7 +54,7 @@ train_and_evaluate <- function(object, ...) {
 #' Export a Saved Model
 #'
 #' Serialize a model to disk. See implementations in the
-#' [keras][keras::export_savedmodel.keras.engine.training.Model()]
+#' [keras3][keras3::export_savedmodel.keras.src.models.model.Model()]
 #' package.
 #'
 #' @param object An \R object.
@@ -66,7 +66,7 @@ train_and_evaluate <- function(object, ...) {
 #'
 #' @section Implementations:
 #'
-#'   - [keras][keras::export_savedmodel.keras.engine.training.Model()]
+#'   - [keras3][keras3::export_savedmodel.keras.src.models.model.Model]
 #'
 #' @keywords internal
 #' @export

R/generics.R

@@ -863,3 +863,23 @@ py_to_r.tensorflow.python.training.tracking.data_structures.ListWrapper <-
 #' @export
 py_to_r.tensorflow.python.training.tracking.data_structures._DictWrapper <-
   py_to_r.tensorflow.python.trackable.data_structures._DictWrapper
+
+
+
+# For tf_version() >= "2.16"
+## Conditionally export these generics, if keras3 hasn't already exported them.
+## We do this to keep keras3 and tensorflow decoupled, but to avoid
+## "S3 method overwritten" warnings if both packages are loaded.
+##
+## Note, we still may need to revisit this; either to disable it, or export a custom $<- method
+## for base classes like Layer, so that compound assignment expressions aren't a
+## problem.
+
+#' @export
+py_to_r.keras.src.utils.tracking.TrackedDict <- function(x) import("builtins")$dict(x)
+
+#' @export
+py_to_r.keras.src.utils.tracking.TrackedList <- function(x) import("builtins")$list(x)
+
+#' @export
+py_to_r.keras.src.utils.tracking.TrackedSet <- function(x) import("builtins")$list(x)

R/install.R

@@ -2,7 +2,7 @@
 #'
 #' `install_tensorflow()` installs just the tensorflow python package and it's
 #' direct dependencies. For a more complete installation that includes
-#' additional optional dependencies, use [`keras::install_keras()`].
+#' additional optional dependencies, use [`keras3::install_keras()`].
 #'
 #' @details You may be prompted to download and install miniconda if reticulate
 #'   did not find a non-system installation of python. Miniconda is the
@@ -16,7 +16,7 @@
 #'   manually install miniconda by running [`reticulate::install_miniconda()`].
 #'
 #' @section Custom Installation: `install_tensorflow()` or
-#'   `keras::install_keras()` isn't required to use tensorflow with the package.
+#'   `keras3::install_keras()` isn't required to use tensorflow with the package.
 #'   If you manually configure a python environment with the required
 #'   dependencies, you can tell R to use it by pointing reticulate at it,
 #'   commonly by setting an environment variable:
@@ -34,10 +34,10 @@
 #'
 #'   ```` R
 #'   x <- array(runif(64*64), c(1, 64, 64))
-#'   keras::layer_random_rotation(x, .5)  # Error:
+#'   keras3::layer_random_rotation(x, .5)  # Error:
 #'   # No registered 'RngReadAndSkip' OpKernel for 'GPU' devices
 #'   # Pin the operation to the CPU to avoid the error
-#'   with(tf$device("CPU"), keras::layer_random_rotation(x, .5) ) # No Error
+#'   with(tf$device("CPU"), keras3::layer_random_rotation(x, .5) ) # No Error
 #'   ````
 #'
 #' @section Additional Packages:
@@ -47,7 +47,7 @@
 #'   argument of `install_tensorflow()` or `install_keras()`, or alternatively
 #'   install them into an existing environment using the
 #'   [reticulate::py_install()] function. Note that `install_keras()` includes a
-#'   set of additional python packages by default, see `?keras::install_keras`
+#'   set of additional python packages by default, see `?keras3::install_keras`
 #'   for details.
 #'
 #' @md
@@ -103,7 +103,7 @@
 #'   [`reticulate::virtualenv_install()`], depending on the `method` used.
 #'
 #' @seealso
-#' -  [`keras::install_keras()`]
+#' -  [`keras3::install_keras()`]
 #' -  <https://tensorflow.rstudio.com/reference/tensorflow/install_tensorflow>
 #'
 #' @export
@@ -117,7 +117,9 @@ function(method = c("auto", "virtualenv", "conda"),
          conda_python_version = NULL,
          ...,
          cuda = NULL,
-         metal = is_mac_arm64(),
+         # tensorflow-metal broken w/ TF v2.16 and default keras:
+         # https://github.com/tensorflow/tensorflow/issues/63854#issuecomment-2011725507
+         metal = FALSE, #is_mac_arm64(),
          pip_ignore_installed = FALSE,
          new_env = identical(envname, "r-tensorflow"),
          python_version = NULL) {
@@ -257,6 +259,10 @@ function(method = c("auto", "virtualenv", "conda"),
              })
   }
 
+  if(cuda && is_linux()) {
+    configure_cudnn_symlinks(envname = envname)
+  }
+
   cat("\nInstallation complete.\n\n")
 
   if (restart_session &&
@@ -267,7 +273,7 @@ function(method = c("auto", "virtualenv", "conda"),
   invisible(NULL)
 }
 
-default_version <- numeric_version("2.15")
+default_version <- numeric_version("2.16")
 
 parse_tensorflow_version <- function(version) {
   # returns unquoted string directly passable to pip, e.g 'tensorflow==2.5.*'
@@ -315,3 +321,64 @@ extract_numeric_version <- function(x, strict = FALSE) {
   x <- sub("\\.+$", "", x)
   numeric_version(x, strict = strict)
 }
+
+
+
+get_cudnn_path <- function(python) {
+
+  # For TF 2.13, this assumes that someone already has cudn 11-8 installed,
+  # e.g., on ubuntu:
+  # sudo apt install cuda-toolkit-11-8
+  # also, that `python -m pip install 'nvidia-cudnn-cu11==8.6.*'`
+
+  force(python)
+  cudnn_module_path <- suppressWarnings(system2(
+    python, c("-c", shQuote("import nvidia.cudnn;print(nvidia.cudnn.__file__)")),
+    stdout = TRUE, stderr = TRUE))
+  if (!is.null(attr(cudnn_module_path, "status")) ||
+      !is_string(cudnn_module_path) ||
+      !file.exists(cudnn_module_path))
+    return()
+
+  dirname(cudnn_module_path)
+
+}
+
+configure_cudnn_symlinks <- function(envname) {
+  if(!is_linux()) return()
+  python <- reticulate::virtualenv_python(envname)
+
+  cudnn_path <- get_cudnn_path(python)
+  if(is.null(cudnn_path)) return()
+  # "~/.virtualenvs/r-tensorflow/lib/python3.9/site-packages/nvidia/cudnn"
+
+  cudnn_sos <- Sys.glob(paste0(cudnn_path, "/lib/*.so*"))
+  if(!length(cudnn_sos)) return()
+  # [1] "~/.virtualenvs/r-tensorflow/lib/python3.9/site-packages/nvidia/cudnn/lib/libcudnn_adv_infer.so.8"
+  # [2] "~/.virtualenvs/r-tensorflow/lib/python3.9/site-packages/nvidia/cudnn/lib/libcudnn_adv_train.so.8"
+  # [3] "~/.virtualenvs/r-tensorflow/lib/python3.9/site-packages/nvidia/cudnn/lib/libcudnn_cnn_infer.so.8"
+  # [4] "~/.virtualenvs/r-tensorflow/lib/python3.9/site-packages/nvidia/cudnn/lib/libcudnn_cnn_train.so.8"
+  # [5] "~/.virtualenvs/r-tensorflow/lib/python3.9/site-packages/nvidia/cudnn/lib/libcudnn_ops_infer.so.8"
+  # [6] "~/.virtualenvs/r-tensorflow/lib/python3.9/site-packages/nvidia/cudnn/lib/libcudnn_ops_train.so.8"
+  # [7] "~/.virtualenvs/r-tensorflow/lib/python3.9/site-packages/nvidia/cudnn/lib/libcudnn.so.8"
+
+  # "~/.virtualenvs/r-tensorflow/lib/python3.9/site-packages/tensorflow/__init__.py"
+  tf_lib_path <- system2(python, c("-c", shQuote("import tensorflow as tf; print(tf.__file__)")),
+                         stderr = FALSE, stdout = TRUE)
+  tf_lib_path <- dirname(tf_lib_path)
+
+  from <- sub("^.*/site-packages/", "../", cudnn_sos)
+  to <- file.path(tf_lib_path, basename(cudnn_sos))
+  writeLines("creating symlinks:")
+  writeLines(paste("-", shQuote(to), "->", shQuote(from)))
+  # creating symlinks:
+  # - '~/.virtualenvs/r-tensorflow/lib/python3.9/site-packages/tensorflow/libcudnn_adv_infer.so.8' -> '../nvidia/cudnn/lib/libcudnn_adv_infer.so.8'
+  # - '~/.virtualenvs/r-tensorflow/lib/python3.9/site-packages/tensorflow/libcudnn_adv_train.so.8' -> '../nvidia/cudnn/lib/libcudnn_adv_train.so.8'
+  # - '~/.virtualenvs/r-tensorflow/lib/python3.9/site-packages/tensorflow/libcudnn_cnn_infer.so.8' -> '../nvidia/cudnn/lib/libcudnn_cnn_infer.so.8'
+  # - '~/.virtualenvs/r-tensorflow/lib/python3.9/site-packages/tensorflow/libcudnn_cnn_train.so.8' -> '../nvidia/cudnn/lib/libcudnn_cnn_train.so.8'
+  # - '~/.virtualenvs/r-tensorflow/lib/python3.9/site-packages/tensorflow/libcudnn_ops_infer.so.8' -> '../nvidia/cudnn/lib/libcudnn_ops_infer.so.8'
+  # - '~/.virtualenvs/r-tensorflow/lib/python3.9/site-packages/tensorflow/libcudnn_ops_train.so.8' -> '../nvidia/cudnn/lib/libcudnn_ops_train.so.8'
+  # - '~/.virtualenvs/r-tensorflow/lib/python3.9/site-packages/tensorflow/libcudnn.so.8' -> '../nvidia/cudnn/lib/libcudnn.so.8'
+  file.symlink(from = from, to = to)
+
+}

R/package.R

@@ -118,6 +118,9 @@ tf_v2 <- function() {
     "a Python installation where the tensorflow module is installed.", call. = FALSE)
   })
 
+  # if keras3 (r pkg) is not loaded, but "r-keras" (venv) exists and
+  # "r-tensorflow" (venv) doesn't, use "r-keras".
+  use_virtualenv("r-keras", required = FALSE)
 
   # provide a common base S3 class for tensors
   reticulate::register_class_filter(function(classes) {