Update to Heasoft 6.33

heasarc/CHANGES.md

@@ -39,3 +39,23 @@
 - Upgrade node in sciserver-jupyter to v20
 - Update ds9 to v8.5
 - Added (this) Changelog.
+
+# Version 0.5
+---
+- Add navo-workshop Dockerfile. This is independent of the heasoft and other builds.
+
+# Version 0.6
+---
+- Update heasoft to v6.33
+- Update CIAO to v4.16
+- Update SAS to v21.0
+- Switch to running jupyter in the base environment, not heasoft
+- Add jupytext, and make it a default reader for .md files.
+- User conda envs are in persistent/users_conda_envs (defined in condarc).
+- Add nb_conda_kernels to handle conda environments (builtin and from the user).
+- Fix ghostscript conflict and tcsh initialization in xmmsas
+- update ds9 download to use archive links so they persist when new versions are added
+- Add back csh initializations
+- Add jupyterlab-execute-time
+- Add support for a user's bashrc under /home/${user}/workspace/Storage/*/persistent/.bashrc.
+- Add JS9

heasarc/README.md

@@ -110,21 +110,21 @@ Starts from Ubuntu v22.04, creates `idies` user and install some base linux tool
 ### heasarc:
 This is the final image that will be used by the enduser.
 
-- Here we are using the heasoft as our default environment, and we have notejs already installed system-wide in `sciserver-jupyter` above.
 - Install any additional linux software: e.g. `vim`, `unzip` etc.
+- Add ipykernel/sciserver to all conda evironments so they can run as kernels.
 - Install the WWT jupyterlab extension.
-- Add all the conda environment created so far as kernels. Remove the default python3 kernel as we are setting heasoft as the default.
+- Install nb_conda_kernels (show conda envs as kernels) jupytext (Run markdown files as notebooks).
 - Install extra python libraries that the user may find useful from `requirements.txt`. Also install the sciserver python package.
-- Install any additional jupyterlab extensions: e.g. `jupyterlab-git`
-- Rebuild jupyterlab activate extension.
-- As root, copy some useful files to their location.
 - Clone sciserver_cookbooks git repo, and setup the default landing page.
 
+- As root, copy some useful files to their location.
+- Install ds9 and jdaviz
+
 
 ### Development:
 The code in the docker files was mostly developed by Ed Sabol and updated by
 Abdu Zoghbi.
 
 
 ---
-Last updated: 2023/10/30 by Abdu Zoghbi.
+Last updated: 2023/12/08 by Abdu Zoghbi.

heasarc/build.json

@@ -1,21 +1,21 @@
 {
     "heasoft": {
-        "version": "6.32.1"
+        "version": "6.33.1"
     },
 
     "ciao": {
-        "version": "4.15.1",
-        "heasoft_version": "6.32.1"
+        "version": "4.16",
+        "heasoft_version": "6.33.1"
     },
 
     "fermi": {
         "version": "2.2.0"
     },
 
     "xmmsas": {
-        "version": "20.0.0",
-        "ubuntu_version": "20.04",
-        "pyversion": "3.9"
+        "version": "21.0.0",
+        "ubuntu_version": "22.04",
+        "pyversion": "3.10"
     },
 
     "spex": {
@@ -24,13 +24,13 @@
     },
 
     "heasarc": {
-        "version": "0.4",
+        "version": "0.6",
         "NOVNC_VERSION": "1.3.0",
         "DS9_VERSION": "8.5"
     },
 
     "sciserver-jupyter": {
-        "version": "3.1"
+        "version": "3.2"
     },
 
     "sciserver-base": {

heasarc/heasarc/Dockerfile

@@ -2,7 +2,6 @@ FROM xmmsas:latest
 
 ARG headata=/home/idies/workspace/headata/FTP
 ARG user=idies
-ARG pythonenv=heasoft
 
 
 ## ----------------------- ##
@@ -26,46 +25,46 @@ WORKDIR /home/$user
 
 ## ------------------------------ ##
 ## A few additions to the kernels ##
-COPY --chown=$user kernel.json kernel-additions.sh ./
-COPY kernel-run.sh /opt/
+COPY --chown=1000:1000 kernel-additions.sh ./
 RUN bash kernel-additions.sh && \
-    rm kernel.json kernel-additions.sh && \ 
+    rm kernel-additions.sh && \ 
     mamba clean -y --all && rm -rf /home/$user/miniconda3/pkgs/*
 ## ------------------------------ ##
 
 
 ## ------------------------ ##
 ## Install WWT labextension ##
-RUN mamba run -n ${pythonenv} pip install pywwt wwt_kernel_data_relay && \
-    mamba run -n ${pythonenv} jupyter labextension install @wwtelescope/jupyterlab && \
+RUN pip install pywwt wwt_kernel_data_relay && \
+    jupyter labextension install @wwtelescope/jupyterlab && \
     pip cache purge
 ## ------------------------ ##
 
+# Custom condarc; install nb_conda_kernels to add conda env to jupyter
+# and also handle user environments
+COPY --chown=1000:1000 condarc /home/$user/miniconda3/.condarc
+COPY --chown=1000:1000 nb_conda_kernels.json /home/$user/miniconda3/etc/jupyter/jupyter_config.json
+RUN mamba install nb_conda_kernels jupytext=1.16.0 \
+ && mamba clean -y --all && rm -rf /home/$user/miniconda3/pkgs/*
+
 
 
 ## ------------------------------ ##
 ## Install extra python libraries ##
 ADD requirements.txt requirements.txt
-RUN mamba install -n ${pythonenv} -y -c conda-forge --file requirements.txt \
+RUN mamba install -n heasoft -y -c conda-forge --file requirements.txt \
  && mamba clean -y --all && rm requirements.txt \
- && rm -rf /home/$user/miniconda3/pkgs/*
+ && rm -rf /home/$user/miniconda3/pkgs/* \
+ && pip install jupyterlab-git jupyterlab-execute-time \
+ && pip cache purge
 ## ------------------------------ ##
 
 
 
-
-## copy some setting files ##
-COPY overrides.json /home/$user/miniconda3/envs/${pythonenv}/share/jupyter/lab/settings/overrides.json
-## ----------------------- ##
-
-
 ## -------------------------- ##
 ## other setup things as root ##
 ## -------------------------- ##
 
-# copy out startup.sh to /opt/
 USER root
-
 # fix yarn & clean cache
 RUN apt-get -y remove cmdtest yarn \
  && npm install -g yarn
@@ -74,23 +73,9 @@ RUN yarn cache clean
 USER root
 
 
-# Copy jupyter-lab startup script
-COPY startup.sh /opt/startup.sh
-RUN chmod +x /opt/startup.sh
-
 # create symlink for Hesarc data volume
 RUN ln -s $headata /FTP
 
-## update heasoftpy ##
-# ARG SHA=7e410150f6259a3bba894ed25841d33c346076bb
-# RUN wget https://github.com/HEASARC/heasoftpy/archive/$SHA.zip \
-#  && unzip $SHA.zip && rm -f $SHA.zip \
-#  && cd heasoftpy-$SHA \
-#  && conda run -n $pythonenv python setup.py build \
-#  && cp -r build/lib/heasoftpy/* /opt/heasoft/x86_64-*/lib/python/heasoftpy/ \
-#  && cd .. && rm -rf heasoftpy-$SHA 
-## ---------------- ##
-
 
 USER $user
 ## -------------------------- ##
@@ -103,18 +88,6 @@ RUN cd /home/$user/workspace \
  && git clone https://github.com/HEASARC/sciserver_cookbooks.git \
  && cd sciserver_cookbooks 
 
-# add entry.sh as an entrypoint script, which calls .bashrc before execution; 
-# Needed for running batch jobs.
-# When the container is run without starting jupyter-lab (i.e. without calling /opt/startup.sh), 
-# all commands are executed though this entry.sh script, which sets the default conda environment to heasoft
-# e.g. 'docker run --rm heasarc which python' should give: /home/idies/miniconda3/envs/heasoft/bin/python
-USER root
-COPY entry.sh /opt/entry.sh
-RUN chmod +x /opt/entry.sh
-ENTRYPOINT ["/opt/entry.sh"]
-CMD ["/bin/bash"]
-USER $user
-
 
 ## ds9 and vnc ##
 USER root
@@ -131,19 +104,19 @@ RUN curl -L -O https://github.com/novnc/noVNC/archive/refs/tags/v${NOVNC_VERSION
 
 # ds9; this needs libxss1 that should've been installed above
 ARG DS9_VERSION=8.4.1
-RUN wget -q https://ds9.si.edu/download/ubuntu20/ds9.ubuntu20.${DS9_VERSION}.tar.gz && \
+RUN wget -q https://ds9.si.edu/archive/ubuntu20/ds9.ubuntu20.${DS9_VERSION}.tar.gz && \
     tar -zxf ds9.ubuntu20.${DS9_VERSION}.tar.gz && \
     chmod +x ds9 && \
     mv ds9 /usr/local/bin && \
     rm ds9.ubuntu20.${DS9_VERSION}.tar.gz
-ADD ds9/config config
+ADD --chown=1000:1000 ds9/config config
 RUN cd config && bash distribute.sh && cd .. && rm -rf config
 ENV DISPLAY=:1
 
 USER $user
-RUN mamba install -n $pythonenv -y -c conda-forge websockify supervisor -y && \
+RUN mamba install -y -c conda-forge websockify supervisor -y && \
     mamba clean -y --all && \
-    mamba run -n $pythonenv pip install jupyter-server-proxy && \
+    pip install jupyter-server-proxy && \
     pip cache purge
 ## ----------- ##
 
@@ -154,10 +127,35 @@ RUN mamba install -n $pythonenv -y -c conda-forge websockify supervisor -y && \
 ## jupyter_server_terminals | error adding extension (enabled: True): Jupyter Server Terminals requires Jupyter Server 2.0+
 ## The terminal seems to be working fine despite that. Once voila is updated to use jupyter-server=2, this
 ## should disappear
-RUN conda run -n $pythonenv pip install jdaviz && pip cache purge \
- && conda run -n $pythonenv jupyter lab build
+RUN mamba run -n heasoft pip install jdaviz && mamba run -n heasoft pip cache purge \
+ && pip install jdaviz && pip cache purge
 ## -------------- ##
 
+## JS9 from Duy ##
+USER root
+RUN JS9_SHA=be9a644 \
+  && wget -q https://github.com/duytnguyendtn/js9/archive/${JS9_SHA}.zip \
+  && unzip ${JS9_SHA}.zip \
+  && rm ${JS9_SHA}.zip \
+  && mv js9-${JS9_SHA}* /opt/js9 \
+  # Temporary fix for require (REMOVE when fixed upstream)
+  && sed -i 's/require=(/var require=(/' /opt/js9/js9plugins.js
+
+
+USER $user
+# 2bfcb136802f23f1fab7d259a791b6e5ee5322bf
+ENV INSTALL_JS9=false
+RUN JPYJS9_SHA=2bfcb136 \
+ && mkdir -p ~/.local/share/jupyterjs9 \
+ && ln -s /opt/js9 /home/$user/.local/share/jupyterjs9/ \
+ && git clone -b jupyterjs9 https://github.com/duytnguyendtn/jpyjs9 \
+ && cd jpyjs9 && git checkout ${JPYJS9_SHA} && cd .. \
+ && pip install ./jpyjs9 \
+ && for cenv in heasoft ciao xmmsas fermi; do conda run -n $cenv pip install ./jpyjs9; done \
+ && rm -rf jpyjs9 && pip cache purge
+## ------------ ##
+
+
 # create a script to print the version; the version arg comes from the variable
 # passed when building the image
 ARG version=0.1
@@ -166,10 +164,19 @@ RUN printf "#!/usr/bin/bash\necho HEASARC Sciserver version: $version\necho Buil
     > /usr/local/bin/heasarc-sciserver-version && \
     chmod +x /usr/local/bin/heasarc-sciserver-version
 ## Fix permissions ##
-RUN chown $user:$user -R /opt/heasoft /home/$user
+RUN chown $user:$user -R /opt/heasoft /home/$user /opt/js9
 ## --------------- ##
 USER $user
 
+# Disable the annoucements extension
+RUN jupyter labextension disable "@jupyterlab/apputils-extension:announcements"
+
+# Copy our startup script to set landing page and default kernel
+COPY startup.sh /opt/startup.sh
+## copy some setting files
+COPY --chown=1000:1000 overrides.json /home/$user/miniconda3/share/jupyter/lab/settings/overrides.json
+## ----------------------- ##
+
 
 # add .bashrc from the user's persistent space
 RUN printf "\n\

heasarc/heasarc/condarc

@@ -0,0 +1,4 @@
+envs_dirs:
+  - /home/idies/workspace/Storage/*/persistent/users_conda_envs
+channels:
+- conda-forge

heasarc/heasarc/ds9/config/fluxbox/menu

@@ -1,6 +1,4 @@
 [begin] (Fluxbox)
-
-
    [submenu] (Applications) {}
         [exec] (ds9) {ds9} <>
         [exec] (xterm) {xterm} <>

heasarc/heasarc/extra_packages.txt

@@ -3,4 +3,5 @@ unzip
 ghostscript
 vim
 screen
-rename
+rename
+emacs

heasarc/heasarc/kernel-additions.sh

@@ -12,11 +12,7 @@ for env in  heasoft ciao fermi spex xmmsas; do
     echo "-- Working on conda environment $env --"
 
     # Add conda envs to jupyter
-    mamba install -y -n $env ipykernel jupyterlab=3.5.0 ipywidgets -c conda-forge
-    mamba run -n $env python -m ipykernel install --user --name=$env --display-name="($env)"
-
-    # A fix that ensures notebooks load the scripts in /env/etc/conda/activate.d
-    sed "s/heasoft/$env/" kernel.json > ~/.local/share/jupyter/kernels/$env/kernel.json
+    mamba install -y -n $env ipykernel -c conda-forge
 
     # Install sciserver python package
     # https://github.com/sciserver/sciserver-compute-images/blob/master/essentials/3.0/sciserver-essentials/Dockerfile#L154

heasarc/heasarc/nb_conda_kernels.json

@@ -0,0 +1,9 @@
+{
+    "JupyterApp": {
+        "kernel_spec_manager_class": "nb_conda_kernels.CondaKernelSpecManager"
+    },
+    "CondaKernelSpecManager": {
+        "kernelspec_path": "--user",
+        "name_format": "({environment})"
+    }
+}

heasarc/heasarc/overrides.json

@@ -1,8 +1,7 @@
 {
   "@jupyterlab/docmanager-extension:plugin": {
     "defaultViewers": {
-      "markdown": "Markdown Preview"
+      "markdown": "Jupytext Notebook"
     }
   }
-
 }

heasarc/heasarc/requirements.txt

@@ -1,4 +1,3 @@
-ipykernel
 wurlitzer
 pyvo
 scikit-learn
@@ -12,4 +11,3 @@ seaborn
 specutils
 aplpy
 pygments
-jupyterlab-git