From 526ed7ae93fa26b22dac0ba94dc814218a0e5cb3 Mon Sep 17 00:00:00 2001 From: Verdi March Date: Wed, 22 Dec 2021 16:39:32 +0000 Subject: [PATCH 1/8] Upgrade to jlab-3.x: initial commit --- initsmnb/upgrade-jupyter.sh | 45 +++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 initsmnb/upgrade-jupyter.sh diff --git a/initsmnb/upgrade-jupyter.sh b/initsmnb/upgrade-jupyter.sh new file mode 100755 index 0000000..fe348fa --- /dev/null +++ b/initsmnb/upgrade-jupyter.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +FLAVOR=$(grep PRETTY_NAME /etc/os-release | cut -d'"' -f 2) +if [[ $FLAVOR != "Amazon Linux 2" ]]; then + echo ${BASH_SOURCE[0]} does not support alinux instance. + exit 1 +fi + +################################################################################ +# IMPLEMENTATION NOTES: upgrade in-place JupyterSystemEnv instead of createing a +# new conda env for the new Jupyter version, because the original conda +# environment has sagemaker nbi agents installed (and possibly other pypi stuffs +# needed to make notebook instance works). +################################################################################ + +CONDA_ENV_DIR=~/anaconda3/envs/JupyterSystemEnv/ +BIN_DIR=$CONDA_ENV_DIR/bin + +# Uninstall the old jupyter proxy +$BIN_DIR/jupyter serverextension disable --py nbserverproxy +sed -i \ + 's/"nbserverproxy": true/"nbserverproxy": false/g' \ + $CONDA_ENV_DIR/etc/jupyter/jupyter_notebook_config.json +$BIN_DIR/pip uninstall --yes nbserverproxy + +# Upgrade jlab & extensions +declare -a PKGS=( + notebook + jupyterlab + jupyter_bokeh + jupyter-server-proxy + nbdime + + # jupyterlab_code_formatter requires formatters in its venv. + # See: https://github.com/ryantam626/jupyterlab_code_formatter/issues/153 + jupyterlab_code_formatter + black + isort +) +$BIN_DIR/pip install --no-cache-dir --upgrade "${PKGS[@]}" + +# These are outdated by Jlab-3.x which has built-in versions of them. +for i in $CONDA_ENV_DIR/share/jupyter/lab/extensions/jupyterlab-{celltags,toc,git}-*.tgz; do + rm $i +done From 126383620fe730b3bd24df889ef20a6cbf9ab01d Mon Sep 17 00:00:00 2001 From: Verdi March Date: Wed, 22 Dec 2021 17:53:50 +0000 Subject: [PATCH 2/8] Apply JLab-3+ configs --- initsmnb/upgrade-jupyter.sh | 120 ++++++++++++++++++++++++++++++++++-- 1 file changed, 115 insertions(+), 5 deletions(-) diff --git a/initsmnb/upgrade-jupyter.sh b/initsmnb/upgrade-jupyter.sh index fe348fa..67485b5 100755 --- a/initsmnb/upgrade-jupyter.sh +++ b/initsmnb/upgrade-jupyter.sh @@ -6,13 +6,15 @@ if [[ $FLAVOR != "Amazon Linux 2" ]]; then exit 1 fi + ################################################################################ -# IMPLEMENTATION NOTES: upgrade in-place JupyterSystemEnv instead of createing a -# new conda env for the new Jupyter version, because the original conda -# environment has sagemaker nbi agents installed (and possibly other pypi stuffs -# needed to make notebook instance works). +# STEP-00: upgrade to JLab-3.x +# +# [IMPLEMENTATION NOTES] Upgrade in-place JupyterSystemEnv instead of a new +# dedicated conda env for the new JLab, because the existing conda environment +# has sagemaker nbi agents installed (and possibly other pypi stuffs needed to +# make notebook instance works). ################################################################################ - CONDA_ENV_DIR=~/anaconda3/envs/JupyterSystemEnv/ BIN_DIR=$CONDA_ENV_DIR/bin @@ -43,3 +45,111 @@ $BIN_DIR/pip install --no-cache-dir --upgrade "${PKGS[@]}" for i in $CONDA_ENV_DIR/share/jupyter/lab/extensions/jupyterlab-{celltags,toc,git}-*.tgz; do rm $i done + + +################################################################################ +# STEP-01: Apply jlab-3+ configs +################################################################################ +JUPYTER_CONFIG_ROOT=~/.jupyter/lab/user-settings/\@jupyterlab + +# Show traliling space is brand-new since JLab-3.2.0 +# See: https://jupyterlab.readthedocs.io/en/3.2.x/getting_started/changelog.html#id22 +mkdir -p $JUPYTER_CONFIG_ROOT/notebook-extension/ +cat << EOF > $JUPYTER_CONFIG_ROOT/notebook-extension/tracker.jupyterlab-settings +{ + // Notebook + // @jupyterlab/notebook-extension:tracker + // Notebook settings. + // ************************************** + "codeCellConfig": { + "rulers": [80, 100], + "lineNumbers": true, + "showTrailingSpace": true + }, + "markdownCellConfig": { + "rulers": [80, 100], + "lineNumbers": true, + "showTrailingSpace": true + }, + "rawCellConfig": { + "rulers": [80, 100], + "lineNumbers": true, + "showTrailingSpace": true + } +} +EOF + +mkdir -p $JUPYTER_CONFIG_ROOT/fileeditor-extension/ +cat << EOF > $JUPYTER_CONFIG_ROOT/fileeditor-extension/plugin.jupyterlab-settings +{ + // Text Editor + // @jupyterlab/fileeditor-extension:plugin + // Text editor settings. + // *************************************** + "editorConfig": { + "rulers": [80, 100], + "codeFolding": true, + "lineNumbers": true, + "showTrailingSpace": true + } +} +EOF + +# macOptionIsMeta is brand-new since JLab-3.0. +# See: https://jupyterlab.readthedocs.io/en/3.0.x/getting_started/changelog.html#other +mkdir -p $JUPYTER_CONFIG_ROOT/terminal-extension/ +cat << EOF > $JUPYTER_CONFIG_ROOT/terminal-extension/plugin.jupyterlab-settings +{ + // Terminal + // @jupyterlab/terminal-extension:plugin + // Terminal settings. + // ************************************* + + // Font size + // The font size used to render text. + "fontSize": 11, + + // Theme + // The theme for the terminal. + "theme": "dark", + + // Treat option as meta key on macOS (new in JLab-3.0) + // Option key on macOS can be used as meta key. This enables to use shortcuts such as option + f + // to move cursor forward one word + "macOptionIsMeta": true +} +EOF + +# Undo the old "mac-option-is-meta" mechanism designed for jlab<3.0. +echo "# JLab-3 + macOptionIsMeta deprecates fix-osx-keymap.sh" > ~/.inputrc +rm ~/.ipython/profile_default/startup/01-osx-jupyterlab-keys.py + +# Show command palette on lhs navbar, similar behavior to smnb. +mkdir -p $JUPYTER_CONFIG_ROOT/apputils-extension/ +cat << EOF > $JUPYTER_CONFIG_ROOT/apputils-extension/palette.jupyterlab-settings +{ + // Command Palette + // @jupyterlab/apputils-extension:palette + // Command palette settings. + // ************************************** + + // Modal Command Palette + // Whether the command palette should be modal or in the left panel. + "modal": false +} +EOF + +# Auto-apply black & isort when saving on notebook editor (but sadly, not on text editor). +mkdir -p $JUPYTER_CONFIG_ROOT/jupyterlab_code_formatter-extension/ +cat << EOF > $JUPYTER_CONFIG_ROOT/jupyterlab_code_formatter-extension/settings.jupyterlab-settings +{ + // Jupyterlab Code Formatter + // @ryantam626/jupyterlab_code_formatter:settings + // Jupyterlab Code Formatter settings. + // ********************************************** + + // Auto format config + // Auto format code when save the notebook. + "formatOnSave": true, +} +EOF From 526327997ad1aa58c5dea39d4278ffc3b1294a08 Mon Sep 17 00:00:00 2001 From: Verdi March Date: Wed, 22 Dec 2021 18:23:58 +0000 Subject: [PATCH 3/8] jlab-3.x ui: code folding, no line wrap --- initsmnb/upgrade-jupyter.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/initsmnb/upgrade-jupyter.sh b/initsmnb/upgrade-jupyter.sh index 67485b5..da27d9d 100755 --- a/initsmnb/upgrade-jupyter.sh +++ b/initsmnb/upgrade-jupyter.sh @@ -63,17 +63,22 @@ cat << EOF > $JUPYTER_CONFIG_ROOT/notebook-extension/tracker.jupyterlab-settings // ************************************** "codeCellConfig": { "rulers": [80, 100], + "codeFolding": true, "lineNumbers": true, + "lineWrap": "off", "showTrailingSpace": true }, "markdownCellConfig": { "rulers": [80, 100], + "codeFolding": true, "lineNumbers": true, + "lineWrap": "off", "showTrailingSpace": true }, "rawCellConfig": { "rulers": [80, 100], "lineNumbers": true, + "lineWrap": "off", "showTrailingSpace": true } } @@ -90,6 +95,7 @@ cat << EOF > $JUPYTER_CONFIG_ROOT/fileeditor-extension/plugin.jupyterlab-setting "rulers": [80, 100], "codeFolding": true, "lineNumbers": true, + "lineWrap": "off", "showTrailingSpace": true } } From f8e43e606a61db7cfc46202513a4cd966eb3d22c Mon Sep 17 00:00:00 2001 From: Verdi March Date: Wed, 22 Dec 2021 18:35:59 +0000 Subject: [PATCH 4/8] Added adventurous level to install and setup scripts --- initsmnb/TEMPLATE-setup-my-sagemaker.sh | 5 ++++- initsmnb/install-initsmnb.sh | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/initsmnb/TEMPLATE-setup-my-sagemaker.sh b/initsmnb/TEMPLATE-setup-my-sagemaker.sh index 97926e9..98055db 100755 --- a/initsmnb/TEMPLATE-setup-my-sagemaker.sh +++ b/initsmnb/TEMPLATE-setup-my-sagemaker.sh @@ -18,6 +18,7 @@ get_bin_dir() { BIN_DIR=$(get_bin_dir) ENABLE_EXPERIMENTAL=0 +ADVENTUROUS=0 # Ensure that we run only on a SageMaker classic notebook instance. ${BIN_DIR}/ensure-smnb.sh @@ -32,7 +33,6 @@ mkdir -p ~/.local/bin ${BIN_DIR}/install-cli.sh ${BIN_DIR}/adjust-sm-git.sh 'Firstname Lastname' first.last@email.abc -${BIN_DIR}/change-jlab-ui.sh ${BIN_DIR}/fix-osx-keymap.sh ${BIN_DIR}/patch-bash-config.sh ${BIN_DIR}/fix-ipython.sh @@ -43,6 +43,7 @@ ${BIN_DIR}/mount-efs-accesspoint.sh fsid,fsapid,mountpoint # These require jupyter lab restarted and browser reloaded, to see the changes. ${BIN_DIR}/patch-jupyter-config.sh +${BIN_DIR}/change-jlab-ui.sh if [[ $ENABLE_EXPERIMENTAL == 1 ]]; then # NOTE: comment or uncomment tweaks in this stanza as necessary. @@ -65,6 +66,8 @@ if [[ $ENABLE_EXPERIMENTAL == 1 ]]; then ${BIN_DIR}/change-docker-tmp-dir.sh ${BIN_DIR}/restart-docker.sh + + [[ $ADVENTUROUS == 1 ]] && ${BIN_DIR}/upgrade-jupyter.sh fi # Final checks and next steps to see the changes in-effect diff --git a/initsmnb/install-initsmnb.sh b/initsmnb/install-initsmnb.sh index f5f68f2..fca9084 100755 --- a/initsmnb/install-initsmnb.sh +++ b/initsmnb/install-initsmnb.sh @@ -76,6 +76,7 @@ declare -a SCRIPTS=( change-docker-data-root.sh change-docker-tmp-dir.sh restart-docker.sh + upgrade-jupyter.sh ) declare -a NESTED_FILES=( From 1303e2768f81f6e8229d3436f6f8a0c3293a6213 Mon Sep 17 00:00:00 2001 From: Verdi March Date: Mon, 27 Dec 2021 09:48:30 +0800 Subject: [PATCH 5/8] Updated readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index af0a03a..a976a29 100644 --- a/README.md +++ b/README.md @@ -286,5 +286,5 @@ This library is licensed under the MIT-0 License. See the LICENSE file. # 9. Acknowledgements -[@yapweiyih](https://github.com/yapweiyih) (EFS), [@josiahdavis](https://github.com/josiahdavis) and [@kianho](https://github.com/kianho) (vim), [@theoldfather](https://github.com/theoldfather) (docker relocation), [@aws/amazon-sagemaker-examples](https://github.com/aws/amazon-sagemaker-examples) (SageMaker local mode), [@yinsong1986](https://github.com/yinsong1986) (persistent custom conda environment), [@verdimrc](https://github.com/verdimrc) (misc.), the originator of git lol & lola (earlier traceable could be this +[@yapweiyih](https://github.com/yapweiyih) (EFS, Streamlit), [@josiahdavis](https://github.com/josiahdavis) and [@kianho](https://github.com/kianho) (vim), [@theoldfather](https://github.com/theoldfather) (docker relocation), [@aws/amazon-sagemaker-examples](https://github.com/aws/amazon-sagemaker-examples) (SageMaker local mode), [@yinsong1986](https://github.com/yinsong1986) (persistent custom conda environment), [@verdimrc](https://github.com/verdimrc) (misc.), the originator of git lol & lola (earlier traceable could be this [blog](http://blog.kfish.org/2010/04/git-lola.html). From 62daf40ca41f7a6546cc5b6c367b3ac1536248ca Mon Sep 17 00:00:00 2001 From: Verdi March Date: Mon, 3 Jan 2022 08:32:28 +0000 Subject: [PATCH 6/8] Adventurous mode: remove outdated sagemaker_examples extension --- initsmnb/upgrade-jupyter.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/initsmnb/upgrade-jupyter.sh b/initsmnb/upgrade-jupyter.sh index da27d9d..6382ff5 100755 --- a/initsmnb/upgrade-jupyter.sh +++ b/initsmnb/upgrade-jupyter.sh @@ -45,6 +45,7 @@ $BIN_DIR/pip install --no-cache-dir --upgrade "${PKGS[@]}" for i in $CONDA_ENV_DIR/share/jupyter/lab/extensions/jupyterlab-{celltags,toc,git}-*.tgz; do rm $i done +rm $CONDA_ENV_DIR/share/jupyter/lab/extensions/sagemaker_examples-*.tgz ################################################################################ From 7de0372fc74402e3a2da5226832037fefe3d2062 Mon Sep 17 00:00:00 2001 From: Verdi March Date: Mon, 3 Jan 2022 17:35:07 +0000 Subject: [PATCH 7/8] jlab-3.x: more packages to update and remove --- initsmnb/upgrade-jupyter.sh | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/initsmnb/upgrade-jupyter.sh b/initsmnb/upgrade-jupyter.sh index 6382ff5..4bc9a03 100755 --- a/initsmnb/upgrade-jupyter.sh +++ b/initsmnb/upgrade-jupyter.sh @@ -18,19 +18,41 @@ fi CONDA_ENV_DIR=~/anaconda3/envs/JupyterSystemEnv/ BIN_DIR=$CONDA_ENV_DIR/bin -# Uninstall the old jupyter proxy +# Uninstall the old jupyter proxy, to unblock Streamlit. $BIN_DIR/jupyter serverextension disable --py nbserverproxy sed -i \ 's/"nbserverproxy": true/"nbserverproxy": false/g' \ $CONDA_ENV_DIR/etc/jupyter/jupyter_notebook_config.json $BIN_DIR/pip uninstall --yes nbserverproxy +# Completely remove these unused or outdated Python packages. +$BIN_DIR/pip uninstall --yes nbdime jupyterlab-git + +# These will be outdated by Jlab-3.x which has built-in versions of them. +declare -a EXTS_TO_DEL=( + jupyterlab-celltags + jupyterlab-toc + jupyterlab-git + nbdime-jupyterlab + + sagemaker_examples # Won't work on jlab-3.x anyway. +) +for i in "${EXTS_TO_DEL[@]}"; do + rm $CONDA_ENV_DIR/share/jupyter/lab/extensions/$i-*.tgz +done + # Upgrade jlab & extensions declare -a PKGS=( notebook jupyterlab - jupyter_bokeh jupyter-server-proxy + + jupyter + jupyter_client + jupyter_console + jupyter_core + + jupyter_bokeh nbdime # jupyterlab_code_formatter requires formatters in its venv. @@ -41,12 +63,6 @@ declare -a PKGS=( ) $BIN_DIR/pip install --no-cache-dir --upgrade "${PKGS[@]}" -# These are outdated by Jlab-3.x which has built-in versions of them. -for i in $CONDA_ENV_DIR/share/jupyter/lab/extensions/jupyterlab-{celltags,toc,git}-*.tgz; do - rm $i -done -rm $CONDA_ENV_DIR/share/jupyter/lab/extensions/sagemaker_examples-*.tgz - ################################################################################ # STEP-01: Apply jlab-3+ configs From 0881bcfd5b6790645e8223c63fd04832f3b2a6dd Mon Sep 17 00:00:00 2001 From: Verdi March Date: Tue, 4 Jan 2022 15:28:08 +0000 Subject: [PATCH 8/8] Added install flags for jlab-3x (aka adventurous) mode install-initsmnb.sh recognizes two new flags: --enable-experimental and --adventurous. --- initsmnb/install-initsmnb.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/initsmnb/install-initsmnb.sh b/initsmnb/install-initsmnb.sh index fca9084..cdcfe54 100755 --- a/initsmnb/install-initsmnb.sh +++ b/initsmnb/install-initsmnb.sh @@ -88,6 +88,9 @@ CURL_OPTS="--fail-early -fL" FROM_LOCAL=0 GIT_USER='' GIT_EMAIL='' +ENABLE_EXPERIMENTAL=0 +ADVENTUROUS=0 + declare -a EFS=() declare -a HELP=( @@ -96,6 +99,8 @@ declare -a HELP=( "[--git-user 'First Last']" "[--git-email me@abc.def]" "[--efs 'fsid,fsap,mp' [--efs ...]]" + "[--enable-experimental]" + "[--adventurous]" ) ################################################################################ @@ -132,6 +137,15 @@ parse_args() { [[ "$2" != "" ]] && EFS+=("$2") shift 2 ;; + --enable-experimental) + ENABLE_EXPERIMENTAL=1 + shift + ;; + --adventurous) + ENABLE_EXPERIMENTAL=1 + ADVENTUROUS=1 + shift + ;; *) error_and_exit "Unknown argument: $key" ;; @@ -213,6 +227,8 @@ sed \ -e "s/Firstname Lastname/$GIT_USER/" \ -e "s/first.last@email.abc/$GIT_EMAIL/" \ -e "s/fsid,fsapid,mountpoint/$(efs2str ' ')/" \ + -e "s/^ENABLE_EXPERIMENTAL=0/ENABLE_EXPERIMENTAL=$ENABLE_EXPERIMENTAL/" \ + -e "s/^ADVENTUROUS=0$/ADVENTUROUS=$ADVENTUROUS/" \ TEMPLATE-setup-my-sagemaker.sh >> setup-my-sagemaker.sh chmod ugo+x setup-my-sagemaker.sh