big new image restructure

.github/workflows/build.yml

@@ -28,30 +28,3 @@ jobs:
           VALIDATE_BASH_EXEC: true
           VALIDATE_DOCKERFILE_HADOLINT: true
           VALIDATE_YAML: true
-
-#  build_python_kernel:
-#    permissions:
-#      id-token: write
-#      contents: read
-#      packages: write
-#      actions: write
-#    uses: ./.github/workflows/reusable-docker-build.yml
-#    strategy:
-#      matrix:
-#        # Must be a supported version by jupyter/datascience-notebook
-#        # https://hub.docker.com/r/jupyter/datascience-notebook/tags?page=1&name=python-
-#        version: [ "3.9.13", "3.8.13" ]
-#    secrets: inherit
-#    with:
-#      dockerfile: ./kernels/python/Dockerfile
-#      context: ./kernels/python
-#      images: |
-#        ghcr.io/${{ github.repository }}/python
-#      tags: |
-#        type=ref,event=branch,prefix=${{ matrix.version }}
-#        type=ref,event=pr,prefix=${{ matrix.version }}
-#        type=sha,format=long,prefix=${{ matrix.version }}
-#        type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }},prefix=${{ matrix.version }}
-#      build_args: |
-#        PYTHON_VERSION=${{ matrix.version }}
-#      platforms: "linux/amd64"

.gitignore

@@ -0,0 +1,46 @@
+# ignore these everywhere
+.pythonrc
+.Rprofile
+apt-install
+Aptfile
+environment.txt
+git_credential_helper.py
+git-wrapper.sh
+gpu.Aptfile
+gpu.requirements.in
+initial-condarc
+ipython_config.py
+secrets_helper.sh
+requirements.in
+requirements.R
+run.sh
+
+# ...except for these places where we care about changes happening
+# (NOTE: this is because the tasks should copy the files down into the build directories)
+!scripts/apt-install
+!scripts/secrets_helper.sh
+
+!python/base/Aptfile
+!python/datascience/Aptfile
+!python/noteable/Aptfile
+
+!python/base-gpu/gpu.Aptfile
+!python/base-gpu/environment.txt
+
+!python/base-gpu/initial-condarc
+
+!python/base/requirements.in
+!python/datascience/requirements.in
+!python/noteable/requirements.in
+
+!python/run.sh
+!python/base-gpu/run.sh
+!r/run.sh
+
+!python/noteable/.pythonrc
+!python/noteable/ipython_config.py
+!python/noteable/git_credential_helper.py
+!python/noteable/git-wrapper.sh
+
+!r/noteable/.Rprofile
+!r/noteable/requirements.R

R/Aptfile

@@ -0,0 +1,13 @@
+build-essential
+ca-certificates
+cmake
+curl
+bzip2
+gnupg2
+wget
+g++
+git
+jq
+libudunits2-dev
+procps
+unixodbc-dev

R/base/4.3.0/Dockerfile

@@ -0,0 +1,76 @@
+# syntax = docker/dockerfile:1.2.1
+# ---
+# Bare minimum R 4.3.x image with IRkernel installed
+# - no R packages aside from builtins and IRkernel
+# - no git, secrets, SQL, extensions, etc
+# ---
+ARG NBL_R_VERSION=4.3.0
+FROM r-base:${NBL_R_VERSION}
+
+# User/group setup
+USER root
+
+ENV NB_USER="noteable" \
+  NB_UID=4004 \
+  NB_GID=4004
+
+RUN groupadd --gid 4004 noteable && \
+  useradd --uid 4004 \
+  --shell /bin/false \
+  --create-home \
+  --no-log-init \
+  --gid noteable noteable \
+  --home-dir /srv/noteable && \
+  chown --recursive noteable:noteable /srv/noteable && \
+  mkdir -p /etc/noteable && chown noteable:noteable /etc/noteable
+
+# Install tini to manage passing signals to the child kernel process
+ENV TINI_VERSION v0.19.0
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini
+RUN chmod +x /tini
+
+# Use micromamba and set up a virtual environment so we can install packages without root
+COPY apt-install /usr/bin/
+# hadolint ignore=DL3045
+COPY Aptfile .
+RUN /usr/bin/apt-install Aptfile
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN wget -qO- https://micromamba.snakepit.net/api/micromamba/linux-64/latest | tar -xvj bin/micromamba && \
+  ./bin/micromamba shell init -s bash -p ~/micromamba
+
+USER noteable
+RUN micromamba create --name noteable-venv \
+  -c conda-forge \
+  -y \
+  r="${NBL_R_VERSION}"
+# make subsequent RUN commands use the virtualenv:
+SHELL ["micromamba", "run", "-n", "noteable-venv", "/bin/bash", "-c"]
+
+# hadolint ignore=SC2239
+RUN R -e "install.packages('IRkernel', repos='http://cran.us.r-project.org')"
+
+COPY secrets_helper.sh /tmp/secrets_helper.sh
+COPY run.sh /usr/local/bin
+
+ENV HOME="/srv/noteable" \
+  XDG_CACHE_HOME="/srv/noteable/.cache/" \
+  GOOGLE_APPLICATION_CREDENTIALS="/vault/secrets/gcp-credentials"
+
+WORKDIR /etc/noteable/project
+EXPOSE 50001-50005
+
+ENTRYPOINT ["/tini", "-g", "--"]
+CMD ["run.sh"]
+
+ARG NBL_ARG_BUILD_TIMESTAMP="undefined"
+ARG NBL_ARG_REVISION="undefined"
+ARG NBL_ARG_BUILD_URL="undefined"
+ARG NBL_ARG_VERSION="undefined"
+LABEL org.opencontainers.image.created="${NBL_ARG_BUILD_TIMESTAMP}" \
+  org.opencontainers.image.revision="${NBL_ARG_REVISION}" \
+  org.opencontainers.image.source="https://github.com/noteable-io/polymorph" \
+  org.opencontainers.image.title="noteable-R-${NBL_R_VERSION}" \
+  org.opencontainers.image.url="${NBL_ARG_BUILD_URL}" \
+  org.opencontainers.image.vendor="Noteable" \
+  org.opencontainers.image.version="${NBL_ARG_VERSION}"

R/noteable/.Rprofile

@@ -0,0 +1,99 @@
+library(IRdisplay)
+library(repr)
+library(reticulate)
+
+prepare_dex_content <- function(df) {
+  #' 
+  #' Create schema and data structure for data frame to be rendered by DEX
+  #' 
+
+  # create a schema for a dataframe, which DEX uses to determine column dtypes.
+  # R data frames don't have this functionality, so we have to use reticulate 
+  # to call into the python pandas library
+  pandas <- import("pandas")
+
+  # If df is a matrix, convert it to a data frame
+  if (is.matrix(df)) {
+    # In R, a matrix is a 2D vector, not a data frame. When reticulate converts an R matrix to Python,
+    # it becomes a numpy array, not a pandas DataFrame. The pandas function we're using requires a DataFrame,
+    # so we need to convert the matrix to a data frame first.
+    #
+    # We use stringsAsFactors = FALSE to prevent R from converting strings to factors. This is a feature of R
+    # that can be confusing for people used to Python, where there's no direct equivalent of factors.
+    #
+    # We use row.names = FALSE to prevent R from using the first column of the data as row names. This is
+    # because R matrices don't have row names in the same way that data frames do, and we want to keep the
+    # structure of the data consistent when we convert it to a DataFrame.
+    df <- as.data.frame(df, stringsAsFactors = FALSE, row.names = FALSE)
+  }
+  df_py <- r_to_py(df)
+  schema <- pandas$io$json$build_table_schema(df_py, index=FALSE)
+
+  # vectorized format (list of lists)
+  #data = as.matrix.data.frame(t(df))
+  # pandas df.to_dict("records") format
+  data = as.data.frame.list(df)
+
+  list(
+    schema = schema,
+    data = data
+  )
+}
+
+prepare_dex_metadata <- function(df) {
+  #' 
+  #' Create metadata for data frame to be rendered by DEX
+  #' 
+  list(
+    default_index_used=TRUE,
+    dataframe_info = list(
+      orig_num_rows = dim(df)[0],
+      orig_num_cols = dim(df)[1]
+    )
+  )
+}
+
+repr_dex <- function(obj, ...) {
+  if (is(obj, "data.frame") || is(obj, "matrix")) {
+    data <- prepare_dex_content(obj)
+    metadata <- prepare_dex_metadata(obj)
+    bundle_data <- list("application/vnd.dataresource+json"=data)
+    bundle_metadata <- list("application/vnd.dataresource+json"=metadata)
+    # we could use publish_mimebundle() to provide the data and metadata,
+    # but that doesn't return anything, which triggers repr_html/repr_markdown, etc
+    #publish_mimebundle(bundle_data, metadata=bundle_metadata)
+    return(data)
+  } else {
+    # if it's not a matrix or data.frame, return NULL to let other repr_* functions handle it.
+    return(NULL)
+  }
+}
+
+enable_dex_formatter <- function() {
+  # Add custom display formatter to newly added mimetype
+  IRkernel:::replace_in_package('repr', 'mime2repr', c(repr::mime2repr, list(`application/vnd.dataresource+json` = repr_dex)))
+
+  # Add dataresource mimetype to list of recognized mimetypes
+  mimetypes <- c(getOption('jupyter.display_mimetypes'), "application/vnd.dataresource+json")
+  options(jupyter.display_mimetypes = mimetypes)
+
+  # Register custom formatter for matrix and data.frame
+  registerS3method("repr_html", "matrix", repr_dex)
+  registerS3method("repr_html", "data.frame", repr_dex)
+}
+
+disable_dex_formatter <- function() {
+  # Remove custom display formatter
+  IRkernel:::replace_in_package('repr', 'mime2repr', repr::mime2repr)
+
+  # Remove dataresource mimetype from list of recognized mimetypes
+  mimetypes <- setdiff(getOption('jupyter.display_mimetypes'), "application/vnd.dataresource+json")
+  options(jupyter.display_mimetypes = mimetypes)
+
+  # Reset the formatter for matrix and data.frame to the default
+  registerS3method("repr_html", "matrix", repr:::repr_html.matrix)
+  registerS3method("repr_html", "data.frame", repr:::repr_html.data.frame)
+}
+
+# enable by default
+enable_dex_formatter()

R/noteable/4.3.0/Dockerfile

@@ -0,0 +1,18 @@
+# syntax = docker/dockerfile:1.2.1
+# Noteable build: adds packages to enable Noteable-specific functionality:
+# - DEX support (via .Rprofile)
+ARG BASE_IMAGE
+# hadolint ignore=DL3006
+FROM ${BASE_IMAGE} as base
+
+USER noteable
+
+# Install python to use with Reticulate
+RUN micromamba install python=3.9 -y -c conda-forge
+
+# R package dependencies and py_install
+COPY requirements.R /tmp/requirements.R
+RUN R -e "source('/tmp/requirements.R')"
+
+# similarly, copy any R commands that need to run on startup
+COPY .Rprofile /srv/noteable/.Rprofile

R/noteable/requirements.R

@@ -0,0 +1,5 @@
+install.packages('reticulate', repos='http://cran.us.r-project.org')
+library(reticulate)
+# Python packages to be used in R via reticulate
+# ref: https://rstudio.github.io/reticulate/articles/python_packages.html
+py_install('pandas==1.5.3', pip=TRUE)

R/run.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -o pipefail
+set -o nounset
+set -o errexit
+
+echo "Local time: $(date)"
+
+set -x
+
+connection_file=/tmp/connection_file.json
+cp /etc/noteable/connections/connection_file.json ${connection_file}
+
+# Inject Secrets into environment (see script docstring for more info)
+# set +x to avoid echoing the Secrets in plaintext to logs
+set +x
+echo "Injecting Secrets into environment, echoing is turned off"
+# shellcheck disable=SC1091
+source /tmp/secrets_helper.sh
+echo "Done injecting Secrets, turning echoing back on"
+set -x
+
+echo "Starting R kernel"
+micromamba run -n noteable-venv R --slave -e "IRkernel::main()" --args  ${connection_file}

Taskfile.R.yaml

@@ -0,0 +1,38 @@
+version: 3
+
+# https://hub.docker.com/_/r-base/tags
+vars:
+  NBL_R_VERSION: 4.3.0
+  IDENTIFIER: base
+
+# NOTE: When using `deps: []`, variables are inherited from the current task, but when calling them
+# directly in `cmds: []`, the variables have to be passed in explicitly.
+
+tasks:
+  core:build:
+    desc: Build the R 4.x image
+    cmds:
+      - >-
+        docker build R/{{.IDENTIFIER}}/{{.NBL_R_VERSION}} \
+          --build-arg "NBL_R_VERSION={{.NBL_R_VERSION}}" \
+          --build-arg "BASE_IMAGE={{.BASE_IMAGE}}" \
+          --tag "local/kernel-r-{{.NBL_R_VERSION}}-{{.IDENTIFIER}}:dev"
+
+  base:copy-files:
+    desc: Copy files from the R directory to the build directories
+    cmds:
+      - task copy-root-files LANGUAGE=R IDENTIFIER={{.IDENTIFIER}} NBL_LANGUAGE_VERSION={{.NBL_R_VERSION}}
+      - task copy-language-files LANGUAGE=R IDENTIFIER={{.IDENTIFIER}} NBL_LANGUAGE_VERSION={{.NBL_R_VERSION}}
+
+  base:build:
+    desc: Build the R 4.x base image after copying required files
+    cmds:
+      - task r:base:copy-files IDENTIFIER=base NBL_LANGUAGE_VERSION={{.NBL_R_VERSION}}
+      - task r:core:build IDENTIFIER=base NBL_R_VERSION={{.NBL_R_VERSION}}
+
+  noteable:build:
+    desc: Build the R 4.3.x image with data frame -> DEX support
+    cmds:
+      - cp R/noteable/.Rprofile R/noteable/{{.NBL_R_VERSION}}/.Rprofile
+      - cp R/noteable/requirements.R R/noteable/{{.NBL_R_VERSION}}/requirements.R
+      - task r:core:build IDENTIFIER=noteable NBL_R_VERSION={{.NBL_R_VERSION}} BASE_IMAGE=local/kernel-r-{{.NBL_R_VERSION}}-base:dev