Merge pull request #87 from darribas/mono_restructure

Mono restructure

Makefile

@@ -1,24 +1,25 @@
 export GDS_VERSION=10.0
+# make command [image=image_name]
 #DOCKERRUN = docker run --rm --user root -e GRANT_SUDO=yes -e NB_UID=1002 -e NB_GID=100 -v `pwd`:/home/jovyan/test
 DOCKERRUN = docker run -v `pwd`:/home/jovyan/test
 test: test_py test_r
 test_py:
-	$(DOCKERRUN) darribas/gds_py:${GDS_VERSION} start.sh jupyter nbconvert --to html --execute /home/jovyan/test/gds_py/check_py_stack.ipynb
+	$(DOCKERRUN) $(image) start.sh /opt/conda/envs/gds/bin/jupyter nbconvert --to html --execute /home/jovyan/test/env/py/check_py_stack.ipynb
 test_r:
-	$(DOCKERRUN) darribas/gds:${GDS_VERSION} start.sh jupyter nbconvert --to html --execute /home/jovyan/test/gds/check_r_stack.ipynb
+	$(DOCKERRUN) $(image) start.sh /opt/conda/envs/gds/bin/jupyter nbconvert --to html --execute /home/jovyan/test/env/r/check_r_stack.ipynb
 write_stacks: write_py_stack write_r_stack
 write_py_stack:
-	$(DOCKERRUN) darribas/gds_py:${GDS_VERSION} start.sh bash -c "conda list -n gds > /home/jovyan/test/gds_py/stack_py.txt"
-	$(DOCKERRUN) darribas/gds_py:${GDS_VERSION} start.sh sed -i '1iGDS version: ${GDS_VERSION}' /home/jovyan/test/gds_py/stack_py.txt
-	$(DOCKERRUN) darribas/gds_py:${GDS_VERSION} start.sh python -c "import subprocess, pandas; fo=open('/home/jovyan/test/gds_py/stack_py.md', 'w'); fo.write(pandas.read_json(subprocess.check_output(['conda', 'list', '-n', 'gds', '--json']).decode())[['name', 'version', 'build_string', 'channel']].to_markdown());fo.close()"
-	$(DOCKERRUN) darribas/gds_py:${GDS_VERSION} start.sh sed -i "1s/^/\n/" /home/jovyan/test/gds_py/stack_py.md
+	$(DOCKERRUN) $(image) start.sh bash -c "conda list -n gds > /home/jovyan/test/env/py/stack_py.txt"
+	$(DOCKERRUN) $(image) start.sh sed -i '1iGDS image: $(image)' /home/jovyan/test/env/py/stack_py.txt
+	$(DOCKERRUN) $(image) start.sh /opt/conda/envs/gds/bin/python -c "import subprocess, pandas; fo=open('/home/jovyan/test/env/py/stack_py.md', 'w'); fo.write(pandas.read_json(subprocess.check_output(['conda', 'list', '-n', 'gds', '--json']).decode())[['name', 'version', 'build_string', 'channel']].to_markdown());fo.close()"
+	$(DOCKERRUN) $(image) start.sh sed -i "1s/^/\n/" /home/jovyan/test/env/py/stack_py.md
 write_r_stack:
-	$(DOCKERRUN) darribas/gds:${GDS_VERSION} start.sh sh -c 'Rscript -e "ip <- as.data.frame(installed.packages()[,c(1,3:4)]); print(ip)" > /home/jovyan/test/gds/stack_r.txt'
-	$(DOCKERRUN) darribas/gds:${GDS_VERSION} start.sh sed -i '1iGDS version: ${GDS_VERSION}' /home/jovyan/test/gds/stack_r.txt
-	$(DOCKERRUN) darribas/gds:${GDS_VERSION} start.sh Rscript -e "library(knitr); ip <- as.data.frame(installed.packages()[,c(1,3:4)]); fc <- file('/home/jovyan/test/gds/stack_r.md'); writeLines(kable(ip, format = 'markdown'), fc); close(fc);"
-	$(DOCKERRUN) darribas/gds:${GDS_VERSION} start.sh sed -i "1s/^/\n/" /home/jovyan/test/gds/stack_r.md
+	$(DOCKERRUN) $(image) start.sh sh -c 'Rscript -e "ip <- as.data.frame(installed.packages()[,c(1,3:4)]); print(ip)" > /home/jovyan/test/env/r/stack_r.txt'
+	$(DOCKERRUN) $(image) start.sh sed -i '1iGDS image: $(image)' /home/jovyan/test/env/r/stack_r.txt
+	$(DOCKERRUN) $(image) start.sh Rscript -e "library(knitr); ip <- as.data.frame(installed.packages()[,c(1,3:4)]); fc <- file('/home/jovyan/test/env/r/stack_r.md'); writeLines(kable(ip, format = 'markdown'), fc); close(fc);"
+	$(DOCKERRUN) $(image) start.sh sed -i "1s/^/\n/" /home/jovyan/test/env/r/stack_r.md
 write_py_explicit:
-	$(DOCKERRUN) darribas/gds_py:${GDS_VERSION} start.sh sh -c "conda list -n gds --explicit > /home/jovyan/test/gds_py/stack_py_explicit.txt"
+	$(DOCKERRUN) $(image) start.sh sh -c "conda list -n gds --explicit > /home/jovyan/test/env/py/stack_py_explicit.txt"
 website_build:
 	cd website && \
 	rm -rf _includes && \

env/Dockerfile

@@ -0,0 +1,133 @@
+FROM quay.io/jupyter/minimal-notebook
+
+LABEL maintainer="Dani Arribas-Bel <D.Arribas-Bel@liverpool.ac.uk>"
+
+# Set version
+ENV GDS_ENV_VERSION "11.0alpha"
+
+# https://github.com/ContinuumIO/docker-images/blob/master/miniconda3/Dockerfile
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+
+ADD ./*/*.sh $HOME/scripts/
+USER root
+RUN chmod +x $HOME/scripts/*.sh
+USER $NB_UID
+
+                ##############
+                ### Python ###
+                ##############
+
+#--- Jupyter Dev ---#
+RUN $HOME/scripts/install_jupyter_dev.sh 
+
+#--- Python ---#
+
+ADD ./py/gds_py.yml /home/${NB_USER}/
+RUN $HOME/scripts/install_py_stack.sh
+ENV PATH="/opt/conda/envs/gds/bin/:${PATH}"
+
+#--- Clean up ---#
+
+USER root
+
+# Clean cache up
+RUN jupyter lab clean -y \
+ && conda clean --yes --all --force-pkgs-dirs \
+ && npm cache clean --force \
+ && rm -rf $CONDA_DIR/share/jupyter/lab/staging \
+ && rm -rf "/home/${NB_USER}/.node-gyp" \
+ && rm -rf /home/$NB_USER/.cache/yarn \
+ && fix-permissions "${CONDA_DIR}" \
+ && fix-permissions "/home/${NB_USER}"
+# Build mpl font cache
+# https://github.com/jupyter/docker-stacks/blob/c3d5df67c8b158b0aded401a647ea97ada1dd085/scipy-notebook/Dockerfile#L59
+USER $NB_UID
+ENV XDG_CACHE_HOME="/home/${NB_USER}/.cache/"
+RUN source activate gds \
+ && MPLBACKEND=Agg python -c "import matplotlib.pyplot"
+
+                #########
+                ### R ###
+                #########
+
+USER root
+
+# Remove Conda from path to not interfere with R install
+RUN echo ${PATH}
+ENV PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+RUN echo ${PATH}
+
+# install R
+RUN $HOME/scripts/install_R.sh
+
+# Install Tidyverse + Geospatial
+RUN $HOME/scripts/install_R_stack.sh
+
+# install R GDS stack
+RUN $HOME/scripts/install_R_gds.sh
+
+# re-attach conda to path
+ENV PATH="/opt/conda/envs/gds/bin/:/opt/conda/condabin:/opt/conda/envs/gds/bin:/opt/conda/bin:${PATH}"
+
+#--- R/Python ---#
+
+RUN $HOME/scripts/setup_py-r.sh
+
+#--- Quarto ---#
+RUN $HOME/scripts/install_quarto.sh
+
+                ###########
+                ### Dev ###
+                ###########
+
+#--- jekyll ---#
+RUN $HOME/scripts/install_jekyll.sh 
+
+#--- tippecanoe ---#
+RUN $HOME/scripts/install_tippecanoe.sh 
+
+#--- htop ---#
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends htop
+
+#--- Decktape ---#
+RUN $HOME/scripts/install_decktape.sh 
+
+#--- LaTeX tools ---#
+ADD ./dev/texBuild.py $HOME/
+ADD ./dev/install_texbuild.py $HOME/
+RUN $HOME/scripts/install_latex_tools.sh 
+
+#---    Vim   ---#
+ADD ./dev/vimrc $HOME/.vimrc
+RUN $HOME/scripts/install_vim.sh 
+
+#---    GPQ   ---#
+RUN $HOME/scripts/install_gpq.sh 
+
+#--- Clean up ---#
+RUN fix-permissions $HOME \
+  && fix-permissions $CONDA_DIR
+
+USER $NB_UID
+
+RUN find /opt/conda/ -follow -type f -name '*.a' -delete \
+ && find /opt/conda/ -follow -type f -name '*.pyc' -delete \
+ && find /opt/conda/ -follow -type f -name '*.js.map' -delete \
+ && find /opt/conda/envs/gds//lib/python*/site-packages/bokeh/server/static \
+    -follow -type f -name '*.js' ! -name '*.min.js' -delete \
+ && pip cache purge \
+ && rm -rf $HOME/.cache/pip
+
+USER root
+RUN cd $NB_HOME \
+ && rm -rf /tmp/downloaded_packages/ /tmp/*.rds \
+ && rm -rf $GEM_HOME/cache \
+ && rm -rf /usr/local/bundle/cache \
+ && apt-get autoclean \
+ && apt-get autoremove \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/* \
+ && rm -rf $HOME/scripts
+
+USER $NB_UID

gds_dev/install_decktape.sh → env/dev/install_decktape.sh

@@ -63,6 +63,7 @@ apt-get update -qq \
 	libcups2 \
 	libxss-dev \
 	libxrandr-dev \
+ && rm -rf /var/lib/apt/lists/* \
  && apt-get autoclean \
  && apt-get autoremove \
  && apt-get clean 

env/dev/install_jekyll.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+apt-get update \
+ && apt-get install -y --no-install-recommends \
+        ruby-full \
+        build-essential \
+        zlib1g-dev
+# https://github.com/sass-contrib/sass-embedded-host-ruby/issues/130#issuecomment-1588245011
+gem install sass-embedded
+gem install sass --force sass-embedded
+gem install jekyll bundler github-pages jekyll-scholar just-the-docs
+
+rm -rf /var/lib/gems/3.0.0/cache/*
+rm -rf /var/lib/apt/lists/* \
+ && apt-get autoclean \
+ && apt-get autoremove \
+ && apt-get clean
+

gds_dev/install_jupyter_dev.sh → env/dev/install_jupyter_dev.sh

@@ -18,4 +18,6 @@ pip install dask-labextension bash_kernel jupyterlab_vim \
 sed -i "s/c.KernelSpecManager.whitelist = {'gds', 'ir'}/c.KernelSpecManager.whitelist = {'gds', 'ir', 'bash'}/g" \
  /home/${NB_USER}/.jupyter/jupyter_lab_config.py
 
+#--- Pypeteer ---#
+pyppeteer-install
 

env/dev/install_vim.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+#################
+### Vim setup ###
+#################
+
+apt-get update \
+ && apt-get install -y --no-install-recommends vim \
+ && curl -fLo $HOME/.vim/autoload/plug.vim --create-dirs \
+    https://raw.githubusercontent.com/junegunn/vim-plug/master/plug.vim \
+ && vim +PlugInstall +qall \
+ && rm -rf /var/lib/apt/lists/* \
+ && apt-get autoclean \
+ && apt-get autoremove \
+ && apt-get clean
+

gds_py/gds_py.yml → env/py/gds_py.yml

@@ -29,7 +29,6 @@ dependencies:
   - ipympl
   - ipywidgets
   - jupyterlab>=4.0
-  - jupyter_bokeh
   - jupyter-book
   - jupytext
   - legendgram
@@ -46,8 +45,6 @@ dependencies:
   - pandana
   - pip
   - pip:
-    - jupyterlab-geojson
-    - jupyterlab_myst
     - matplotlib-scalebar
     - polars
 #    - pygeoda

env/py/install_jupyter_dev.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+#--- [Run under $NB_USER] ---#
+
+#--- Text editor / default Vim ---#
+# https://github.com/jupyterlab/jupyterlab/issues/14599
+mkdir -p /home/$NB_USER/.jupyter/lab/user-settings/\@jupyterlab/fileeditor-extension/ \
+ && echo '{"editorConfig": {"codeFolding": true, "highlightActiveLine": true, "highlightTrailingWhitespace": true}}' \
+ >> /home/$NB_USER/.jupyter/lab/user-settings/\@jupyterlab/fileeditor-extension/plugin.jupyterlab-settings
+
+#--- JupyterLab extensions & Bash kernel ---#
+
+jupyter labextension disable "@jupyterlab/apputils-extension:announcements" \
+ && pip install \
+         bash_kernel \
+         jupyterlab-geojson \
+         jupyterlab_myst \
+         jupyterlab-quarto \
+         jupyterlab_vim \
+         jupyterlab_widgets \
+         jupytext
+# Bash kernel
+python -m bash_kernel.install
+# Required for dask extension
+mamba install python-graphviz
+# Clean
+pip cache purge \
+ && conda clean --all --yes --force-pkgs-dirs \
+ && jupyter lab clean -y \
+ && npm cache clean --force
+

env/py/install_py_stack.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+#--- [Run under $NB_USER] ---#
+
+# Install GDS conda environment
+mamba env create -f gds_py.yml \
+ && source activate gds \
+ && python -m ipykernel install --user --name gds --display-name "GDS-$GDS_ENV_VERSION" \
+ && conda deactivate \
+ && rm ./gds_py.yml \
+ && conda clean --yes --all --force-pkgs-dirs \
+ && find /opt/conda/ -follow -type f -name '*.a' -delete \
+ && find /opt/conda/ -follow -type f -name '*.pyc' -delete \
+ && find /opt/conda/ -follow -type f -name '*.js.map' -delete \
+ && find /opt/conda/envs/gds//lib/python*/site-packages/bokeh/server/static \
+    -follow -type f -name '*.js' ! -name '*.min.js' -delete \
+ && pip cache purge \
+ && rm -rf $HOME/.cache/pip
+
+# Pypeteer
+pyppeteer-install
+
+# Make GDS default
+# https://github.com/jupyter/notebook/issues/3674#issuecomment-397212982
+jupyter lab --generate-config \
+ && echo "c.MultiKernelManager.default_kernel_name='gds'" >> \
+ /home/${NB_USER}/.jupyter/jupyter_lab_config.py \
+ && echo "conda activate gds" >> /home/${NB_USER}/.bashrc \
+ && echo "c.KernelSpecManager.ensure_native_kernel = False" >> \
+ /home/${NB_USER}/.jupyter/jupyter_lab_config.py \
+ && echo "c.KernelSpecManager.whitelist = {'gds', 'bash'}" >> \
+ /home/${NB_USER}/.jupyter/jupyter_lab_config.py
+jupyter kernelspec remove -y python3 
+

gds/install_R.sh → env/r/install_R.sh

@@ -14,6 +14,7 @@ apt-get update -qq \
   && apt-get update -qq \
   && apt-get install -y --no-install-recommends \
     ca-certificates \
+    libcurl4-openssl-dev \
     less \
     libopenblas-base \
     locales \
@@ -56,6 +57,8 @@ apt-get update \
 && ln -s /usr/lib/R/site-library/littler/examples/testInstalled.r /usr/local/bin/testInstalled.r \
 && ln -s /usr/lib/R/site-library/littler/examples/update.r /usr/local/bin/update.r \
 && rm -rf /tmp/downloaded_packages/ /tmp/*.rds \
-&& rm -rf /var/lib/apt/lists/*
-
+&& rm -rf /var/lib/apt/lists/* \
+&& apt-get autoclean \
+&& apt-get autoremove \
+&& apt-get clean
 

gds/install_R_gds.sh → env/r/install_R_gds.sh

@@ -26,21 +26,6 @@ apt-get update -qq \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/*
 
-#--- R ---#
-# https://github.com/rocker-org/rocker-versioned/blob/master/r-ver/Dockerfile
-# Look at: http://sites.psu.edu/theubunturblog/installing-r-in-ubuntu/
-
-echo "deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/" >> /etc/apt/sources.list \
- && sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 \
- && apt-get update \
- && apt-get install -y \
-    r-base \
-    r-base-dev \
- && apt-get autoclean \
- && apt-get autoremove \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/*
-
 #--- R libs ---#
 # SAN requirements (on top of Rocker Geospatial)
 # https://github.com/GDSL-UL/san/blob/master/01-overview.Rmd

gds/install_R_stack.sh → env/r/install_R_stack.sh

@@ -32,6 +32,7 @@ apt-get update -qq \
 R -e "install.packages(c( \
             'tidyverse', \
             'devtools', \
+            'curl', \
             'rmarkdown', \
             'BiocManager', \
             'vroom', \
@@ -68,6 +69,7 @@ add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable \
     libhdf4-alt-dev \
     libhdf5-dev \
     libjq-dev \
+    liblwgeom-dev \
     libpq-dev \
     libproj-dev \
     libprotobuf-dev \

gds/setup_py-r.sh → env/r/setup_py-r.sh

@@ -1,3 +1,5 @@
+# Set up links between Python/R (IRkernel, rpy2) and R/Python (reticulate)
+
 ln -s /opt/conda/envs/gds/bin/jupyter /usr/local/bin
 R -e "install.packages(c( \
         'IRkernel', 'reticulate' \
@@ -8,8 +10,8 @@ LD_LIBRARY_PATH /usr/local/lib/R/lib/:${LD_LIBRARY_PATH}
 fix-permissions $HOME
 fix-permissions $CONDA_DIR
 
-sed -i "s/c.KernelSpecManager.whitelist = {'gds'}/c.KernelSpecManager.whitelist = {'gds', 'ir'}/g" \
-    me/${NB_USER}/.jupyter/jupyter_lab_config.py
+sed -i "s/c.KernelSpecManager.whitelist = {'gds', 'bash'}/c.KernelSpecManager.whitelist = {'gds', 'ir', 'bash'}/g" \
+  /home/${NB_USER}/.jupyter/jupyter_lab_config.py
 
 pip install -U --no-deps \
         rpy2 \
@@ -18,7 +20,8 @@ pip install -U --no-deps \
         pytz_deprecation_shim \
         jinja2 \
         'cffi>=1.10.0' \
-        tzlocal
+        tzlocal \
+ && pip cache purge
 
 rm -rf /home/$NB_USER/.cache/pip
 rm -rf /tmp/downloaded_packages/ /tmp/*.rds

gds_code/Dockerfile → frontend_code/Dockerfile

@@ -1,4 +1,4 @@
-FROM darribas/gds_dev:10.0
+FROM gds:11.0alpha
 
 USER root